@memberjunction/ai-agents 5.29.0 → 5.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,6 +4,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
4
4
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
5
  return c > 3 && r && Object.defineProperty(target, key, r), r;
6
6
  };
7
+ var MemoryManagerAgent_1;
7
8
  import { BaseAgent } from './base-agent.js';
8
9
  import { RegisterClass, CleanAndParseJSON } from '@memberjunction/global';
9
10
  import { Metadata, RunView, RunQuery, LogError, LogStatus } from '@memberjunction/core';
@@ -35,16 +36,93 @@ const CONSOLIDATION_CONFIG = {
35
36
  * - 'daily': Run once per day
36
37
  * - number: Run every N executions (e.g., 4 = every 4th run)
37
38
  */
38
- frequency: 'disabled',
39
+ frequency: 'daily',
39
40
  minClusterSize: 3, // Minimum notes in a cluster to trigger consolidation
40
- similarityThreshold: 0.60 // Semantic similarity threshold — 60% captures topically related notes while excluding dissimilar ones
41
+ maxClusterSize: 7, // Maximum cluster size (Miller's 7±2) — split larger clusters
42
+ similarityThreshold: 0.60, // Semantic similarity threshold — 60% captures topically related notes
43
+ maxConsolidationCount: 3, // Cap re-summarization depth to prevent drift (research: 25.5% semantic loss after 5 iterations)
44
+ noteCountTrigger: 100 // Event-driven trigger: consolidate when an agent accumulates this many new notes since last run
41
45
  };
46
+ /**
47
+ * Configuration for contradiction detection.
48
+ * Finds semantically similar note pairs and evaluates whether they contain conflicting facts.
49
+ */
50
+ const CONTRADICTION_CONFIG = {
51
+ similarityThreshold: 0.70, // Higher than consolidation — targets same-topic pairs
52
+ maxPairsPerRun: 50, // Limit LLM calls per cycle
53
+ highImportanceThreshold: 7.0 // Notes above this get importance-weighted protection (both preserved with flag)
54
+ };
55
+ /**
56
+ * Configuration for composite importance scoring.
57
+ * 7-signal formula replacing raw AccessCount for authority, retention, and consolidation decisions.
58
+ * Based on Park et al.'s Generative Agents (2023) and neuroscience research on memory importance.
59
+ */
60
+ const IMPORTANCE_CONFIG = {
61
+ weights: {
62
+ recencyDecay: 0.20, // Exponential decay: 0.995^hours
63
+ llmImportance: 0.25, // Confidence from extraction prompt
64
+ relevance: 0.15, // Cosine similarity to recent conversation inputs
65
+ uniqueness: 0.15, // Min distance to nearest neighbor (rarity signal)
66
+ correctionBoost: 0.10, // Notes that corrected/superseded existing notes
67
+ goalAlignment: 0.10, // Alignment with agent goals (Phase 2 enrichment, 0.5 default)
68
+ userMark: 0.05 // Manual creation flag (IsAutoGenerated=false)
69
+ },
70
+ correctionBonusPoints: 1.5, // Flat additive bonus for correction-provenance notes
71
+ autoPromoteThreshold: 8.0, // Auto-promote Standard notes to Protected above this score
72
+ };
73
+ /**
74
+ * Configuration for Ebbinghaus-inspired decay function.
75
+ * Replaces fixed 90/180-day retention windows with continuous importance-weighted decay.
76
+ * Formula: strength = normalizedImportance * e^(-lambdaEff * daysSinceAccess) * (1 + accessCount * accessBoostFactor)
77
+ * where lambdaEff = baseLambda * (1 - normalizedImportance * importanceDampening)
78
+ */
79
+ const DECAY_CONFIG = {
80
+ baseLambda: 0.16, // Base decay rate (Ebbinghaus-derived)
81
+ importanceDampening: 0.8, // How much high importance slows decay (0-1)
82
+ accessBoostFactor: 0.2, // Each access adds 20% to retention strength
83
+ accessRetentionCap: 1.5, // Hard cap on the access-retention multiplier to prevent frequently-accessed notes from becoming immortal
84
+ protectedInactivityDays: 365, // Protected tier extended retention before archival
85
+ ephemeralDecayMultiplier: 2.0, // Ephemeral tier decays 2x faster
86
+ archivalFloor: 0.5, // Score below this after proportional decay → archive (multiple cycles to reach)
87
+ };
88
+ /**
89
+ * Configuration for stale reference pruning.
90
+ * Caps how many orphaned notes the pruning phase will archive per cycle to keep
91
+ * Memory Manager runtime bounded even on large corpora.
92
+ */
93
+ const STALE_PRUNING_CONFIG = {
94
+ maxNotesPerRun: 200,
95
+ };
96
+ /**
97
+ * Minimum number of scored notes required to compute a 95th-percentile uniqueness
98
+ * threshold. Below this the percentile is meaningless (a 5-note cohort has a p95
99
+ * at index 4 which is just "the max"), so outlier auto-protection is disabled.
100
+ */
101
+ const OUTLIER_PROMOTION_MIN_COHORT = 20;
102
+ /**
103
+ * Valid Type values for MJAIAgentNote records. Must match the typed union in the
104
+ * generated entity class (MJCoreEntities/src/generated/entity_subclasses.ts).
105
+ * Used to validate LLM-returned values before assigning to entity properties.
106
+ */
107
+ const VALID_NOTE_TYPES = ['Constraint', 'Context', 'Example', 'Issue', 'Preference'];
108
+ function isValidNoteType(value) {
109
+ return typeof value === 'string' && VALID_NOTE_TYPES.includes(value);
110
+ }
111
+ /** Reusable parameter object for deterministic (temperature=0) LLM prompt calls. */
112
+ const DETERMINISTIC_PROMPT_PARAMS = { temperature: 0 };
113
+ /**
114
+ * Hard cap on `resolveOriginalSources()` recursion depth.
115
+ * The `visited` set already prevents cycles, but a depth limit protects against
116
+ * runaway recursion if `DerivedFromNoteIDs` chains are corrupted in a way that
117
+ * passes the visited check (e.g., extremely deep legitimate chains from a bug).
118
+ */
119
+ const MAX_SOURCE_RESOLUTION_DEPTH = 10;
42
120
  /**
43
121
  * Memory Manager Agent - automatically extracts notes and examples from conversations.
44
122
  * This agent runs on a schedule (every 15 minutes) and analyzes high-quality conversations
45
123
  * and agent runs to extract learnings and example interactions.
46
124
  */
47
- let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
125
+ let MemoryManagerAgent = MemoryManagerAgent_1 = class MemoryManagerAgent extends BaseAgent {
48
126
  constructor() {
49
127
  super(...arguments);
50
128
  /** Verbose logging flag from params.verbose */
@@ -56,6 +134,12 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
56
134
  /** Context user for step operations */
57
135
  this._contextUser = null;
58
136
  }
137
+ /** Check if a string is a valid UUID format. Filters out LLM-generated placeholders like "user-uuid-here". */
138
+ static isValidUUID(id) {
139
+ if (!id)
140
+ return false;
141
+ return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);
142
+ }
59
143
  /**
60
144
  * Create an agent run step record for observability.
61
145
  * Returns null if agentRunID is not set (defensive check).
@@ -312,33 +396,67 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
312
396
  return this.executeNoteExtraction(promptData, existingNotes, contextUser);
313
397
  }
314
398
  /**
315
- * Common extraction logic for note extraction.
316
- * Applies sparsity controls and deduplication.
399
+ * Common extraction logic for note extraction. Thin orchestrator that coordinates:
400
+ * (1) running the extraction prompt,
401
+ * (2) normalizing merge-field shape from the LLM,
402
+ * (3) applying confidence/length filters,
403
+ * (4) enforcing per-message/per-conversation sparsity caps,
404
+ * (5) collapsing duplicate merge candidates,
405
+ * (6) deduplicating against existing notes,
406
+ * (7) enriching with conversation context (userId, agentRunId).
407
+ *
408
+ * Each step is its own private helper so the pipeline can be read top-to-bottom.
317
409
  */
318
410
  async executeNoteExtraction(promptData, existingNotes, contextUser) {
319
- const conversationThreads = promptData.conversationThreads;
320
- // Find extraction prompt
321
- const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Extract Notes' && p.Category === 'MJ: System');
322
- if (!prompt) {
323
- LogError('Memory Manager note extraction prompt not found');
324
- if (this._verbose) {
325
- LogStatus(`Memory Manager: Available prompts in MJ: System category: ${AIEngine.Instance.Prompts.filter(p => p.Category === 'MJ: System').map(p => p.Name).join(', ')}`);
326
- }
411
+ // Step 1: run the extraction prompt and parse out the raw notes
412
+ const rawNotes = await this.runNoteExtractionPrompt(promptData, existingNotes.length, contextUser);
413
+ if (!rawNotes)
327
414
  return [];
328
- }
415
+ // Step 2: normalize the LLM's legacy singular mergeWithExistingId → plural mergeWithExistingIds
416
+ this.normalizeMergeCandidates(rawNotes);
329
417
  if (this._verbose) {
330
- LogStatus(`Memory Manager: Found extraction prompt "${prompt.Name}" (ID: ${prompt.ID})`);
331
- LogStatus(`Memory Manager: Sending ${conversationThreads.length} conversation threads with ${conversationThreads.reduce((sum, t) => sum + t.messages.length, 0)} total messages for note extraction`);
332
- if (conversationThreads.length > 0) {
333
- const firstThread = conversationThreads[0];
334
- LogStatus(`Memory Manager: Sample thread (conv ${firstThread.conversationId}): ${firstThread.messages.map(m => `[${m.role}] ${m.message?.substring(0, 80)}...`).join(' | ')}`);
418
+ LogStatus(`Memory Manager: LLM returned ${rawNotes.length} raw notes before filtering`);
419
+ for (const note of rawNotes) {
420
+ LogStatus(`Memory Manager: Raw note: [${note.type}] "${note.content}" (confidence: ${note.confidence})${note.mergeWithExistingIds?.length ? ` mergeWith: ${note.mergeWithExistingIds.join(', ')}` : ''}`);
335
421
  }
336
422
  }
337
- // Step 3: Execute AI extraction
338
- const step3 = await this.CreateRunStep('Prompt', 'Extract Notes from Conversations', {
423
+ // Step 3: filter by confidence and content length
424
+ const candidateNotes = this.filterByConfidenceAndLength(rawNotes);
425
+ if (candidateNotes.length === 0) {
426
+ if (this._verbose)
427
+ LogStatus('Memory Manager: No candidates passed confidence/length thresholds');
428
+ return [];
429
+ }
430
+ // Step 4: apply per-message and per-conversation sparsity caps
431
+ const sparseCandidates = this.applySparsityLimits(candidateNotes);
432
+ // Step 5: collapse merge candidates with identical content
433
+ const collapsedCandidates = this.collapseMergeCandidates(sparseCandidates);
434
+ if (this._verbose && collapsedCandidates.length < sparseCandidates.length) {
435
+ LogStatus(`Memory Manager: Collapsed ${sparseCandidates.length} candidates to ${collapsedCandidates.length} after merging duplicate merge targets`);
436
+ }
437
+ // Step 6: deduplicate against existing notes (creates and finalizes Step 4 run step)
438
+ const approvedNotes = await this.deduplicateCandidates(collapsedCandidates, existingNotes, contextUser);
439
+ if (this._verbose)
440
+ LogStatus(`Memory Manager: Final approved notes: ${approvedNotes.length}`);
441
+ // Step 7: enrich with conversation context (userId, agentRunId)
442
+ this.enrichWithConversationContext(approvedNotes, promptData.conversationThreads);
443
+ return approvedNotes;
444
+ }
445
+ /**
446
+ * Execute the extraction prompt, wrap it in an observability run step, and
447
+ * parse the raw result into a typed array. Returns null on any failure so the
448
+ * caller short-circuits cleanly.
449
+ */
450
+ async runNoteExtractionPrompt(promptData, existingNoteCount, contextUser) {
451
+ const prompt = this.findNoteExtractionPrompt();
452
+ if (!prompt)
453
+ return null;
454
+ const conversationThreads = promptData.conversationThreads;
455
+ this.logExtractionPromptContext(prompt, conversationThreads);
456
+ const step = await this.CreateRunStep('Prompt', 'Extract Notes from Conversations', {
339
457
  conversationCount: conversationThreads.length,
340
458
  messageCount: conversationThreads.reduce((sum, t) => sum + t.messages.length, 0),
341
- existingNoteCount: existingNotes.length
459
+ existingNoteCount
342
460
  }, prompt.ID);
343
461
  const runner = new AIPromptRunner();
344
462
  const params = new AIPromptParams();
@@ -346,93 +464,111 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
346
464
  params.data = promptData;
347
465
  params.contextUser = contextUser;
348
466
  params.attemptJSONRepair = true;
467
+ params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
349
468
  const result = await runner.ExecutePrompt(params);
350
- // Finalize Step 3 after extraction
351
- await this.FinalizeRunStep(step3, result.success, {
469
+ await this.FinalizeRunStep(step, result.success, {
352
470
  success: result.success,
353
471
  rawNoteCount: result.result && typeof result.result !== 'string' ? (result.result.notes?.length || 0) : 0
354
472
  }, result.promptRun?.ID, result.errorMessage || undefined);
473
+ return this.parseNoteExtractionResult(result);
474
+ }
475
+ /** Look up the extraction prompt, logging available prompts on miss. */
476
+ findNoteExtractionPrompt() {
477
+ const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Extract Notes' && p.Category === 'MJ: System');
478
+ if (!prompt) {
479
+ LogError('Memory Manager note extraction prompt not found');
480
+ if (this._verbose) {
481
+ LogStatus(`Memory Manager: Available prompts in MJ: System category: ${AIEngine.Instance.Prompts.filter(p => p.Category === 'MJ: System').map(p => p.Name).join(', ')}`);
482
+ }
483
+ }
484
+ return prompt;
485
+ }
486
+ /** Verbose-only logging block for extraction prompt context. */
487
+ logExtractionPromptContext(prompt, conversationThreads) {
488
+ if (!this._verbose)
489
+ return;
490
+ LogStatus(`Memory Manager: Found extraction prompt "${prompt.Name}" (ID: ${prompt.ID})`);
491
+ LogStatus(`Memory Manager: Sending ${conversationThreads.length} conversation threads with ${conversationThreads.reduce((sum, t) => sum + t.messages.length, 0)} total messages for note extraction`);
492
+ if (conversationThreads.length > 0) {
493
+ const firstThread = conversationThreads[0];
494
+ LogStatus(`Memory Manager: Sample thread (conv ${firstThread.conversationId}): ${firstThread.messages.map(m => `[${m.role}] ${m.message?.substring(0, 80)}...`).join(' | ')}`);
495
+ }
496
+ }
497
+ /**
498
+ * Parse an extraction prompt result into an ExtractedNote array, tolerating JSON-fenced
499
+ * strings from models like Gemini. Returns null on failure.
500
+ */
501
+ parseNoteExtractionResult(result) {
355
502
  if (this._verbose) {
356
503
  LogStatus(`Memory Manager: Extraction prompt result - success: ${result.success}, hasResult: ${!!result.result}`);
357
- if (result.errorMessage) {
504
+ if (result.errorMessage)
358
505
  LogStatus(`Memory Manager: Extraction error: ${result.errorMessage}`);
359
- }
360
- if (result.result) {
506
+ if (result.result)
361
507
  LogStatus(`Memory Manager: Raw extraction result: ${JSON.stringify(result.result)}`);
362
- }
363
508
  }
364
509
  if (!result.success || !result.result) {
365
510
  LogError('Failed to extract notes:', result.errorMessage);
366
- return [];
511
+ return null;
367
512
  }
368
- // Parse result if it's a string (AI sometimes returns JSON as string)
369
- // Some models (e.g., Gemini) wrap JSON in ```json fences
370
- let parsedResult;
371
513
  if (typeof result.result === 'string') {
372
- const parsed = CleanAndParseJSON(result.result, true);
373
- if (!parsed) {
514
+ const reparsed = CleanAndParseJSON(result.result, true);
515
+ if (!reparsed) {
374
516
  LogError('Failed to parse extraction result as JSON');
375
- return [];
517
+ return null;
376
518
  }
377
- parsedResult = parsed;
378
519
  if (this._verbose) {
379
- LogStatus(`Memory Manager: Parsed string result into object with ${parsedResult.notes?.length || 0} notes`);
380
- }
381
- }
382
- else {
383
- parsedResult = result.result;
384
- }
385
- // Normalize singular mergeWithExistingId → plural mergeWithExistingIds
386
- // The LLM still outputs the singular field; we convert it here.
387
- // Notes are plain objects from JSON.parse, so the runtime cast is safe.
388
- if (parsedResult.notes) {
389
- for (const note of parsedResult.notes) {
390
- if (note.mergeWithExistingId && !note.mergeWithExistingIds) {
391
- note.mergeWithExistingIds = [note.mergeWithExistingId];
392
- }
520
+ LogStatus(`Memory Manager: Parsed string result into object with ${reparsed.notes?.length || 0} notes`);
393
521
  }
522
+ return reparsed.notes || [];
394
523
  }
395
- // Log raw notes before filtering (for debugging)
396
- if (this._verbose && parsedResult.notes) {
397
- LogStatus(`Memory Manager: LLM returned ${parsedResult.notes.length} raw notes before filtering`);
398
- for (const note of parsedResult.notes) {
399
- LogStatus(`Memory Manager: Raw note: [${note.type}] "${note.content}" (confidence: ${note.confidence})${note.mergeWithExistingIds?.length ? ` mergeWith: ${note.mergeWithExistingIds.join(', ')}` : ''}`);
524
+ return result.result.notes || [];
525
+ }
526
+ /**
527
+ * Normalize the legacy singular `mergeWithExistingId` field into the plural
528
+ * `mergeWithExistingIds` array. The extraction prompt still emits the singular
529
+ * field in some paths; downstream code only reads the plural form.
530
+ */
531
+ normalizeMergeCandidates(notes) {
532
+ for (const note of notes) {
533
+ if (note.mergeWithExistingId && !note.mergeWithExistingIds) {
534
+ note.mergeWithExistingIds = [note.mergeWithExistingId];
400
535
  }
401
536
  }
402
- // Filter by confidence threshold (use EXTRACTION_CONFIG)
403
- const candidateNotes = (parsedResult.notes || []).filter(n => n.confidence >= EXTRACTION_CONFIG.minConfidenceThreshold &&
537
+ }
538
+ /**
539
+ * Filter raw extracted notes against the confidence and content-length thresholds
540
+ * from EXTRACTION_CONFIG. Notes below either threshold are silently dropped.
541
+ */
542
+ filterByConfidenceAndLength(notes) {
543
+ const filtered = notes.filter(n => n.confidence >= EXTRACTION_CONFIG.minConfidenceThreshold &&
404
544
  n.content && n.content.length >= EXTRACTION_CONFIG.minContentLength);
405
- if (candidateNotes.length === 0) {
406
- if (this._verbose) {
407
- LogStatus('Memory Manager: No candidates passed confidence/length thresholds');
408
- }
409
- return [];
410
- }
411
- if (this._verbose) {
412
- LogStatus(`Memory Manager: ${candidateNotes.length} candidates passed confidence threshold (>=${EXTRACTION_CONFIG.minConfidenceThreshold})`);
413
- // Log each extracted note for debugging
414
- for (const note of candidateNotes) {
545
+ if (this._verbose && filtered.length > 0) {
546
+ LogStatus(`Memory Manager: ${filtered.length} candidates passed confidence threshold (>=${EXTRACTION_CONFIG.minConfidenceThreshold})`);
547
+ for (const note of filtered) {
415
548
  LogStatus(`Memory Manager: Extracted note: [${note.type}] "${note.content}" (confidence: ${note.confidence})`);
416
549
  }
417
550
  }
418
- // Apply sparsity: limit notes per message AND per conversation
551
+ return filtered;
552
+ }
553
+ /**
554
+ * Enforce per-message and per-conversation note caps from EXTRACTION_CONFIG.
555
+ * Drops any candidates that would exceed either cap.
556
+ */
557
+ applySparsityLimits(candidates) {
419
558
  const notesByMessage = new Map();
420
559
  const noteCountByConversation = new Map();
421
- for (const note of candidateNotes) {
560
+ for (const note of candidates) {
422
561
  const msgId = note.sourceConversationDetailId || 'unknown';
423
562
  const convId = note.sourceConversationId || 'unknown';
424
563
  const existingForMsg = notesByMessage.get(msgId) || [];
425
564
  const convCount = noteCountByConversation.get(convId) || 0;
426
- // Check both per-message and per-conversation limits
427
565
  if (existingForMsg.length >= EXTRACTION_CONFIG.maxNotesPerMessage) {
428
- if (this._verbose) {
566
+ if (this._verbose)
429
567
  LogStatus(`Memory Manager: Skipping note (max ${EXTRACTION_CONFIG.maxNotesPerMessage} per message reached)`);
430
- }
431
568
  }
432
569
  else if (convCount >= EXTRACTION_CONFIG.maxNotesPerConversation) {
433
- if (this._verbose) {
570
+ if (this._verbose)
434
571
  LogStatus(`Memory Manager: Skipping note (max ${EXTRACTION_CONFIG.maxNotesPerConversation} per conversation reached)`);
435
- }
436
572
  }
437
573
  else {
438
574
  existingForMsg.push(note);
@@ -440,110 +576,123 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
440
576
  noteCountByConversation.set(convId, convCount + 1);
441
577
  }
442
578
  }
443
- const sparseCandidates = Array.from(notesByMessage.values()).flat();
444
- if (this._verbose) {
445
- LogStatus(`Memory Manager: ${sparseCandidates.length} candidates after sparsity filter`);
446
- }
447
- // Collapse merge candidates with identical content into single candidates
448
- const collapsedCandidates = this.collapseMergeCandidates(sparseCandidates);
449
- if (this._verbose && collapsedCandidates.length < sparseCandidates.length) {
450
- LogStatus(`Memory Manager: Collapsed ${sparseCandidates.length} candidates to ${collapsedCandidates.length} after merging duplicate merge targets`);
451
- }
452
- // Step 4: Apply deduplication (summary step)
453
- const step4 = await this.CreateRunStep('Decision', 'Deduplicate Note Candidates', {
454
- candidateCount: collapsedCandidates.length,
579
+ const sparse = Array.from(notesByMessage.values()).flat();
580
+ if (this._verbose)
581
+ LogStatus(`Memory Manager: ${sparse.length} candidates after sparsity filter`);
582
+ return sparse;
583
+ }
584
+ /**
585
+ * Deduplicate candidate notes against existing notes in the database using:
586
+ * 1. Exact text match (fast reject),
587
+ * 2. Semantic similarity lookup (top-N with ≥85% threshold),
588
+ * 3. Dedup prompt to resolve genuinely-similar candidates.
589
+ *
590
+ * Candidates flagged as merge/replacement by the extraction LLM are auto-approved
591
+ * (dedup would reject them as "too similar" to the note they are intentionally
592
+ * replacing). Creates and finalizes its own observability run step.
593
+ */
594
+ async deduplicateCandidates(candidates, existingNotes, contextUser) {
595
+ const step = await this.CreateRunStep('Decision', 'Deduplicate Note Candidates', {
596
+ candidateCount: candidates.length,
455
597
  existingNoteCount: existingNotes.length
456
598
  });
457
- const approvedNotes = [];
458
- let dedupeRejectedCount = 0;
459
- let dedupeLlmCallCount = 0;
460
- // Find deduplication prompt
599
+ const approved = [];
600
+ const stats = { rejectedCount: 0, llmCallCount: 0 };
601
+ const runner = new AIPromptRunner();
461
602
  const dedupePrompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Deduplicate Note' && p.Category === 'MJ: System');
462
- for (const candidate of collapsedCandidates) {
463
- // Check global max notes per run
464
- if (approvedNotes.length >= EXTRACTION_CONFIG.maxNotesPerRun) {
465
- if (this._verbose) {
603
+ for (const candidate of candidates) {
604
+ if (approved.length >= EXTRACTION_CONFIG.maxNotesPerRun) {
605
+ if (this._verbose)
466
606
  LogStatus(`Memory Manager: Stopping - max notes per run (${EXTRACTION_CONFIG.maxNotesPerRun}) reached`);
467
- }
468
607
  break;
469
608
  }
470
- // Notes with mergeWithExistingIds are intentional replacements —
471
- // the extraction LLM already determined this is a contradiction.
472
- // Dedup would reject them as "too similar" to the note they're replacing.
473
- if (candidate.mergeWithExistingIds?.length) {
474
- approvedNotes.push(candidate);
475
- if (this._verbose) {
476
- LogStatus(`Memory Manager: Auto-approved merge note targeting ${candidate.mergeWithExistingIds.join(', ')}: "${candidate.content.substring(0, 50)}..."`);
477
- }
478
- continue;
479
- }
480
- // Step 1: Check for exact content match FIRST (case-insensitive, trimmed)
481
- const normalizedContent = candidate.content.toLowerCase().trim();
482
- const exactMatch = existingNotes.find(n => n.Note && n.Note.toLowerCase().trim() === normalizedContent);
483
- if (exactMatch) {
484
- dedupeRejectedCount++;
485
- if (this._verbose) {
486
- LogStatus(`Memory Manager: Skipping exact duplicate: "${candidate.content.substring(0, 50)}..."`);
487
- }
488
- continue;
489
- }
490
- // Step 2: Find similar existing notes using semantic search with tighter threshold.
491
- // Cross-user dedup for org/global notes works naturally because those notes have
492
- // UserID=null, so FindSimilarAgentNotes won't filter them out for any user.
493
- const similarNotes = await AIEngine.Instance.FindSimilarAgentNotes(candidate.content, candidate.agentId, candidate.userId, candidate.companyId, 10, // Top 10 similar (increased from 5)
494
- 0.85 // 85% similarity threshold (increased from 70%)
495
- );
496
- // Step 3: If deduplication prompt exists and similar notes found, ask LLM
497
- if (dedupePrompt && similarNotes.length > 0) {
498
- const dedupeParams = new AIPromptParams();
499
- dedupeParams.prompt = dedupePrompt;
500
- dedupeParams.data = {
501
- candidateNote: candidate,
502
- similarNotes: similarNotes.map(s => ({
503
- type: s.note.Type,
504
- content: s.note.Note,
505
- agentId: s.note.AgentID,
506
- userId: s.note.UserID,
507
- companyId: s.note.CompanyID,
508
- similarity: s.similarity
509
- }))
510
- };
511
- dedupeParams.contextUser = contextUser;
512
- dedupeParams.attemptJSONRepair = true;
513
- const dedupeResult = await runner.ExecutePrompt(dedupeParams);
514
- dedupeLlmCallCount++;
515
- if (dedupeResult.success && dedupeResult.result && dedupeResult.result.shouldAdd) {
516
- approvedNotes.push(candidate);
517
- if (this._verbose) {
518
- LogStatus(`Memory Manager: Approved note - ${dedupeResult.result.reason}`);
519
- }
520
- }
521
- else {
522
- dedupeRejectedCount++;
523
- if (this._verbose) {
524
- LogStatus(`Memory Manager: Skipped duplicate note - ${dedupeResult.result?.reason || 'too similar to existing notes'}`);
525
- }
526
- }
527
- }
528
- else {
529
- // No similar notes found or no deduplication prompt, add the note
530
- approvedNotes.push(candidate);
531
- if (this._verbose) {
532
- LogStatus(`Memory Manager: Approved note (no similar notes found): "${candidate.content.substring(0, 50)}..."`);
533
- }
609
+ const decision = await this.evaluateCandidateForApproval(candidate, existingNotes, dedupePrompt, runner, contextUser, stats);
610
+ if (decision) {
611
+ approved.push(candidate);
534
612
  }
535
613
  }
536
- // Finalize Step 4 after deduplication loop
537
- await this.FinalizeRunStep(step4, true, {
538
- approvedCount: approvedNotes.length,
539
- rejectedCount: dedupeRejectedCount,
540
- llmCallCount: dedupeLlmCallCount
614
+ await this.FinalizeRunStep(step, true, {
615
+ approvedCount: approved.length,
616
+ rejectedCount: stats.rejectedCount,
617
+ llmCallCount: stats.llmCallCount
541
618
  });
542
- if (this._verbose) {
543
- LogStatus(`Memory Manager: Final approved notes: ${approvedNotes.length}`);
619
+ return approved;
620
+ }
621
+ /**
622
+ * Evaluate a single candidate: merge markers auto-approve, exact-text duplicates are
623
+ * rejected, otherwise run semantic similarity + LLM dedup. Returns true if the candidate
624
+ * should be kept. Mutates `stats` for rejection + LLM call counts as a side effect.
625
+ */
626
+ async evaluateCandidateForApproval(candidate, existingNotes, dedupePrompt, runner, contextUser, stats) {
627
+ // Merge/replacement candidates are auto-approved — the extraction LLM already
628
+ // determined they supersede an existing note. Running dedup on them would reject
629
+ // them as "too similar" to the very note they are replacing.
630
+ if (candidate.mergeWithExistingIds?.length) {
631
+ if (this._verbose) {
632
+ LogStatus(`Memory Manager: Auto-approved merge note targeting ${candidate.mergeWithExistingIds.join(', ')}: "${candidate.content.substring(0, 50)}..."`);
633
+ }
634
+ return true;
635
+ }
636
+ // Fast path: exact text match (case-insensitive, trimmed)
637
+ const normalizedContent = candidate.content.toLowerCase().trim();
638
+ if (existingNotes.some(n => n.Note && n.Note.toLowerCase().trim() === normalizedContent)) {
639
+ stats.rejectedCount++;
640
+ if (this._verbose)
641
+ LogStatus(`Memory Manager: Skipping exact duplicate: "${candidate.content.substring(0, 50)}..."`);
642
+ return false;
643
+ }
644
+ // Semantic similarity lookup. Cross-user dedup for org/global notes works naturally
645
+ // because those notes have UserID=null — FindSimilarAgentNotes won't filter them out.
646
+ const similarNotes = await AIEngine.Instance.FindSimilarAgentNotes(candidate.content, candidate.agentId, candidate.userId, candidate.companyId, 10, 0.85);
647
+ if (!dedupePrompt || similarNotes.length === 0) {
648
+ if (this._verbose)
649
+ LogStatus(`Memory Manager: Approved note (no similar notes found): "${candidate.content.substring(0, 50)}..."`);
650
+ return true;
651
+ }
652
+ const decision = await this.runDedupePromptForCandidate(runner, dedupePrompt, candidate, similarNotes, contextUser);
653
+ stats.llmCallCount++;
654
+ if (decision.shouldAdd) {
655
+ if (this._verbose)
656
+ LogStatus(`Memory Manager: Approved note - ${decision.reason}`);
657
+ return true;
658
+ }
659
+ stats.rejectedCount++;
660
+ if (this._verbose)
661
+ LogStatus(`Memory Manager: Skipped duplicate note - ${decision.reason}`);
662
+ return false;
663
+ }
664
+ /**
665
+ * Invoke the dedupe LLM prompt for a single candidate note against its similar
666
+ * neighbors. Returns the decision plus reason string — callers handle logging.
667
+ */
668
+ async runDedupePromptForCandidate(runner, dedupePrompt, candidate, similarNotes, contextUser) {
669
+ const params = new AIPromptParams();
670
+ params.prompt = dedupePrompt;
671
+ params.data = {
672
+ candidateNote: candidate,
673
+ similarNotes: similarNotes.map(s => ({
674
+ type: s.note.Type,
675
+ content: s.note.Note,
676
+ agentId: s.note.AgentID,
677
+ userId: s.note.UserID,
678
+ companyId: s.note.CompanyID,
679
+ similarity: s.similarity
680
+ }))
681
+ };
682
+ params.contextUser = contextUser;
683
+ params.attemptJSONRepair = true;
684
+ params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
685
+ const result = await runner.ExecutePrompt(params);
686
+ if (result.success && result.result) {
687
+ return { shouldAdd: result.result.shouldAdd, reason: result.result.reason || '' };
544
688
  }
545
- // Enrich notes with userId and agentRunId from conversation context
546
- // Build a map of conversationId -> context for quick lookup
689
+ return { shouldAdd: false, reason: result.result?.reason || 'too similar to existing notes' };
690
+ }
691
+ /**
692
+ * Populate userId and sourceAgentRunId on approved notes from the source
693
+ * conversation's context. Only fills fields the LLM didn't already set.
694
+ */
695
+ enrichWithConversationContext(notes, conversationThreads) {
547
696
  const conversationContext = new Map();
548
697
  for (const thread of conversationThreads) {
549
698
  conversationContext.set(thread.conversationId, {
@@ -551,23 +700,17 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
551
700
  agentRunId: thread.agentRunId || null
552
701
  });
553
702
  }
554
- // Enrich each note with conversation context
555
- for (const note of approvedNotes) {
556
- if (note.sourceConversationId) {
557
- const ctx = conversationContext.get(note.sourceConversationId);
558
- if (ctx) {
559
- // Set userId from conversation if not already set
560
- if (!note.userId && ctx.userId) {
561
- note.userId = ctx.userId;
562
- }
563
- // Set sourceAgentRunId from conversation if not already set
564
- if (!note.sourceAgentRunId && ctx.agentRunId) {
565
- note.sourceAgentRunId = ctx.agentRunId;
566
- }
567
- }
568
- }
703
+ for (const note of notes) {
704
+ if (!note.sourceConversationId)
705
+ continue;
706
+ const ctx = conversationContext.get(note.sourceConversationId);
707
+ if (!ctx)
708
+ continue;
709
+ if (!note.userId && ctx.userId)
710
+ note.userId = ctx.userId;
711
+ if (!note.sourceAgentRunId && ctx.agentRunId)
712
+ note.sourceAgentRunId = ctx.agentRunId;
569
713
  }
570
- return approvedNotes;
571
714
  }
572
715
  /**
573
716
  * Extract examples from conversation details with high ratings.
@@ -601,6 +744,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
601
744
  extractParams.data = { qaPairs };
602
745
  extractParams.contextUser = contextUser;
603
746
  extractParams.attemptJSONRepair = true;
747
+ extractParams.additionalParameters = { temperature: 0 };
604
748
  const extractResult = await runner.ExecutePrompt(extractParams);
605
749
  if (!extractResult.success || !extractResult.result) {
606
750
  await this.FinalizeRunStep(step5, false, {
@@ -662,6 +806,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
662
806
  };
663
807
  dedupeParams.contextUser = contextUser;
664
808
  dedupeParams.attemptJSONRepair = true;
809
+ dedupeParams.additionalParameters = { temperature: 0 };
665
810
  const dedupeResult = await runner.ExecutePrompt(dedupeParams);
666
811
  exampleDedupeLlmCallCount++;
667
812
  if (dedupeResult.success && dedupeResult.result && dedupeResult.result.shouldAdd) {
@@ -690,13 +835,6 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
690
835
  });
691
836
  return approvedExamples;
692
837
  }
693
- /**
694
- * Check if a string is a valid UUID format.
695
- */
696
- isValidUUID(str) {
697
- const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
698
- return uuidRegex.test(str);
699
- }
700
838
  /**
701
839
  * Create note records from extracted data.
702
840
  * Inherits scope from source agent run and applies scopeLevel to determine scope specificity.
@@ -746,6 +884,23 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
746
884
  for (const mergeTargetId of extracted.mergeWithExistingIds) {
747
885
  const existingNote = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
748
886
  if (await existingNote.Load(mergeTargetId)) {
887
+ // Honor protection tiers — Immutable and Protected notes must NEVER be
888
+ // auto-revoked even when the extraction LLM flags them as merge targets.
889
+ // The contradiction-detection phase enforces the same rule via tier-based
890
+ // filtering, but this earlier merge path bypassed it. Skip the revocation
891
+ // and just annotate the existing note so the conflict is observable.
892
+ const tier = existingNote.ProtectionTier || 'Standard';
893
+ if (tier === 'Immutable' || tier === 'Protected') {
894
+ const flagComment = `[Merge-target conflict: extraction LLM flagged this ${tier} note for merge from conversation ${extracted.sourceConversationId || 'unknown'} but tier prevents auto-revocation]`;
895
+ existingNote.Comments = `${existingNote.Comments || ''} ${flagComment}`.trim();
896
+ if (!await existingNote.Save()) {
897
+ LogError(`Memory Manager: Failed to annotate ${tier} merge target ${existingNote.ID}`);
898
+ }
899
+ if (this._verbose) {
900
+ LogStatus(`Memory Manager: Skipped revocation of ${tier} note ${existingNote.ID} flagged as merge target`);
901
+ }
902
+ continue;
903
+ }
749
904
  existingNote.Status = 'Revoked';
750
905
  existingNote.Comments = `Superseded: contradiction detected, replaced by new note from conversation ${extracted.sourceConversationId || 'unknown'}`;
751
906
  if (await existingNote.Save()) {
@@ -765,28 +920,25 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
765
920
  if (createNewNote) {
766
921
  // Create new note
767
922
  const note = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
768
- // Only use valid UUIDs, filter out placeholders like "user-uuid-here"
769
- const isValidUUID = (id) => {
770
- if (!id)
771
- return false;
772
- return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);
773
- };
774
923
  // Determine AgentID - prefer extracted, then inherit from source run
775
- let agentId = isValidUUID(extracted.agentId) ? extracted.agentId : null;
924
+ let agentId = MemoryManagerAgent_1.isValidUUID(extracted.agentId) ? extracted.agentId : null;
776
925
  if (!agentId && sourceRun?.AgentID) {
777
926
  agentId = sourceRun.AgentID;
778
927
  }
779
928
  note.AgentID = agentId;
780
- note.CompanyID = isValidUUID(extracted.companyId) ? extracted.companyId : null;
929
+ note.CompanyID = sourceRun?.CompanyID || null;
781
930
  note.AgentNoteTypeID = aiNoteTypeId; // "AI" type for AI-generated notes
782
931
  note.Type = extracted.type; // Category: Preference, Constraint, Context, Issue, Example
783
932
  note.Note = extracted.content;
784
933
  note.IsAutoGenerated = true;
785
934
  note.Status = 'Active'; // Auto-approve high-confidence notes
786
935
  note.AccessCount = 1; // Required field
936
+ // Consolidation tracking fields
937
+ note.ConsolidationCount = 0; // Raw extraction, never consolidated
938
+ note.ProtectionTier = note.IsAutoGenerated ? 'Standard' : 'Protected';
787
939
  note.SourceConversationID = extracted.sourceConversationId || null;
788
940
  // Only use if it's a valid UUID (LLM now sees message IDs in the prompt)
789
- note.SourceConversationDetailID = isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
941
+ note.SourceConversationDetailID = MemoryManagerAgent_1.isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
790
942
  note.SourceAIAgentRunID = extracted.sourceAgentRunId || null;
791
943
  // Apply scope: Lean towards user-specific memories by default
792
944
  const scopeLevel = extracted.scopeLevel || 'user';
@@ -808,9 +960,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
808
960
  // UserID: Only set for user-scoped notes. Company/global notes belong to the
809
961
  // company, not a specific user. This also enables cross-user dedup — FindSimilarAgentNotes
810
962
  // filters by userId, so null UserID makes company notes visible to all users.
811
- note.UserID = scopeLevel === 'user'
812
- ? (isValidUUID(extracted.userId) ? extracted.userId : null)
813
- : null;
963
+ note.UserID = scopeLevel === 'user' ? (sourceRun?.UserID || null) : null;
814
964
  const saveResult = await note.Save();
815
965
  if (saveResult) {
816
966
  created++;
@@ -861,6 +1011,175 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
861
1011
  }
862
1012
  return result;
863
1013
  }
1014
+ /**
1015
+ * Compute composite importance scores for all active notes.
1016
+ * Uses a 7-signal weighted formula based on Park et al.'s Generative Agents (2023).
1017
+ * Signals not available for a given note use neutral defaults (0.5) with proportional weight redistribution.
1018
+ *
1019
+ * Auto-promotes Standard-tier notes to Protected via two paths:
1020
+ * 1. ImportanceScore >= IMPORTANCE_CONFIG.autoPromoteThreshold (default 8.0)
1021
+ * 2. Uniqueness at or above the 95th percentile of the cohort (outlier protection,
1022
+ * FR-094). The cohort must be at least OUTLIER_PROMOTION_MIN_COHORT notes for
1023
+ * the percentile to be meaningful.
1024
+ *
1025
+ * Also reports the score distribution (min/max/mean/median) so the observability
1026
+ * step can surface it in the agent run step output.
1027
+ */
1028
+ async computeImportanceScores(contextUser) {
1029
+ const allNotes = AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active');
1030
+ let notesScored = 0;
1031
+ let tierPromotions = 0;
1032
+ const md = new Metadata();
1033
+ const scoresForDistribution = [];
1034
+ // Pre-pass: compute uniqueness per note once, derive 95th-percentile threshold.
1035
+ // Passing uniqueness into gatherImportanceSignals also eliminates a redundant
1036
+ // FindSimilarAgentNotes call per note.
1037
+ const uniquenessByNoteId = await this.precomputeUniqueness(allNotes);
1038
+ const p95Threshold = this.computeUniquenessP95(uniquenessByNoteId);
1039
+ for (const note of allNotes) {
1040
+ const uniquenessInfo = uniquenessByNoteId.get(note.ID);
1041
+ const signals = this.gatherImportanceSignals(note, uniquenessInfo);
1042
+ const importanceScore = this.computeWeightedImportanceScore(signals);
1043
+ const noteToUpdate = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
1044
+ if (await noteToUpdate.Load(note.ID)) {
1045
+ noteToUpdate.ImportanceScore = importanceScore;
1046
+ // Promotion path 1: high ImportanceScore
1047
+ const currentTier = noteToUpdate.ProtectionTier || 'Standard';
1048
+ let promotedByScore = false;
1049
+ if (currentTier === 'Standard' && importanceScore >= IMPORTANCE_CONFIG.autoPromoteThreshold) {
1050
+ noteToUpdate.ProtectionTier = 'Protected';
1051
+ noteToUpdate.Comments = `${noteToUpdate.Comments || ''} [Auto-promoted to Protected: ImportanceScore ${importanceScore} >= ${IMPORTANCE_CONFIG.autoPromoteThreshold}]`.trim();
1052
+ tierPromotions++;
1053
+ promotedByScore = true;
1054
+ }
1055
+ // Promotion path 2: uniqueness outlier (p95+). Only apply when the cohort
1056
+ // is large enough for the percentile to be meaningful, the note has a
1057
+ // computed uniqueness, and we didn't already promote it via the score path.
1058
+ if (!promotedByScore &&
1059
+ currentTier === 'Standard' &&
1060
+ p95Threshold !== null &&
1061
+ uniquenessInfo?.available &&
1062
+ uniquenessInfo.value >= p95Threshold) {
1063
+ noteToUpdate.ProtectionTier = 'Protected';
1064
+ noteToUpdate.Comments = `${noteToUpdate.Comments || ''} [Auto-protected: uniqueness ${uniquenessInfo.value.toFixed(3)} at/above 95th percentile ${p95Threshold.toFixed(3)}]`.trim();
1065
+ tierPromotions++;
1066
+ }
1067
+ if (await noteToUpdate.Save()) {
1068
+ notesScored++;
1069
+ scoresForDistribution.push(importanceScore);
1070
+ }
1071
+ else {
1072
+ LogError(`Memory Manager: Failed to update ImportanceScore for note ${note.ID}`);
1073
+ }
1074
+ }
1075
+ }
1076
+ return {
1077
+ notesScored,
1078
+ tierPromotions,
1079
+ scoreDistribution: this.summarizeScores(scoresForDistribution)
1080
+ };
1081
+ }
1082
+ /**
1083
+ * Pre-compute uniqueness (1 - maxSimilarity to nearest neighbor) for each note in
1084
+ * one pass. Returns a map keyed by note ID. Notes without content get an
1085
+ * unavailable signal so downstream scoring falls back to the neutral default.
1086
+ */
1087
+ async precomputeUniqueness(notes) {
1088
+ const map = new Map();
1089
+ for (const note of notes) {
1090
+ if (!note.Note) {
1091
+ map.set(note.ID, { value: 0.5, available: false });
1092
+ continue;
1093
+ }
1094
+ const similar = await AIEngine.Instance.FindSimilarAgentNotes(note.Note, note.AgentID, undefined, undefined, 1, 0.0);
1095
+ const uniqueness = similar.length > 0 ? 1.0 - similar[0].similarity : 1.0;
1096
+ map.set(note.ID, { value: uniqueness, available: true });
1097
+ }
1098
+ return map;
1099
+ }
1100
+ /**
1101
+ * Derive the 95th-percentile uniqueness threshold across the cohort. Returns null
1102
+ * when the cohort is smaller than OUTLIER_PROMOTION_MIN_COHORT (percentile isn't
1103
+ * meaningful) or when no uniqueness was actually computable.
1104
+ */
1105
+ computeUniquenessP95(uniquenessMap) {
1106
+ const available = Array.from(uniquenessMap.values())
1107
+ .filter(u => u.available)
1108
+ .map(u => u.value);
1109
+ if (available.length < OUTLIER_PROMOTION_MIN_COHORT) {
1110
+ return null;
1111
+ }
1112
+ const sorted = available.slice().sort((a, b) => a - b);
1113
+ const index = Math.floor(sorted.length * 0.95);
1114
+ const clampedIndex = Math.min(sorted.length - 1, index);
1115
+ return sorted[clampedIndex];
1116
+ }
1117
+ /**
1118
+ * Summarize importance scores into min/max/mean/median. Returns null for an empty
1119
+ * input so the caller can serialize it as an explicit "no data" signal in the
1120
+ * observability step.
1121
+ */
1122
+ summarizeScores(scores) {
1123
+ if (scores.length === 0)
1124
+ return null;
1125
+ const sorted = scores.slice().sort((a, b) => a - b);
1126
+ const min = sorted[0];
1127
+ const max = sorted[sorted.length - 1];
1128
+ const mean = sorted.reduce((sum, v) => sum + v, 0) / sorted.length;
1129
+ const mid = sorted.length / 2;
1130
+ const median = sorted.length % 2 === 0
1131
+ ? (sorted[mid - 1] + sorted[mid]) / 2
1132
+ : sorted[Math.floor(mid)];
1133
+ return {
1134
+ min: Math.round(min * 100) / 100,
1135
+ max: Math.round(max * 100) / 100,
1136
+ mean: Math.round(mean * 100) / 100,
1137
+ median: Math.round(median * 100) / 100
1138
+ };
1139
+ }
1140
+ /**
1141
+ * Gather the 7 importance signals for a single note. Uniqueness is precomputed by
1142
+ * the caller so the full cohort can be examined in one pass for percentile
1143
+ * analysis. Other signals are derived from the note itself.
1144
+ */
1145
+ gatherImportanceSignals(note, precomputedUniqueness) {
1146
+ const signals = [];
1147
+ const hoursSinceCreation = (Date.now() - new Date(note.__mj_CreatedAt).getTime()) / (1000 * 60 * 60);
1148
+ const hasConfidence = note.Comments?.includes('confidence') || false;
1149
+ const isCorrection = note.Comments?.includes('Superseded') || note.Comments?.includes('contradiction');
1150
+ signals.push({ signal: 'recencyDecay', value: Math.pow(0.995, hoursSinceCreation), available: true });
1151
+ signals.push({ signal: 'llmImportance', value: hasConfidence ? 0.7 : 0.5, available: hasConfidence });
1152
+ signals.push({ signal: 'relevance', value: 0.5, available: false }); // Phase 2 enrichment
1153
+ signals.push({
1154
+ signal: 'uniqueness',
1155
+ value: precomputedUniqueness?.value ?? 0.5,
1156
+ available: precomputedUniqueness?.available ?? false
1157
+ });
1158
+ signals.push({ signal: 'correctionBoost', value: isCorrection ? 1.0 : 0.0, available: true });
1159
+ signals.push({ signal: 'goalAlignment', value: 0.5, available: false }); // Phase 2 enrichment
1160
+ signals.push({ signal: 'userMark', value: note.IsAutoGenerated ? 0.0 : 1.0, available: true });
1161
+ return signals;
1162
+ }
1163
+ /** Compute a weighted importance score (0-10) from signals, redistributing weight from unavailable signals. */
1164
+ computeWeightedImportanceScore(signals) {
1165
+ const weights = IMPORTANCE_CONFIG.weights;
1166
+ const availableSignals = signals.filter(s => s.available);
1167
+ const totalAvailableWeight = availableSignals.reduce((sum, s) => sum + (weights[s.signal] || 0), 0);
1168
+ const unavailableWeight = signals.filter(s => !s.available).reduce((sum, s) => sum + (weights[s.signal] || 0), 0);
1169
+ let rawScore = 0;
1170
+ for (const s of availableSignals) {
1171
+ if (totalAvailableWeight > 0) {
1172
+ const adjustedWeight = (weights[s.signal] || 0) + ((weights[s.signal] || 0) / totalAvailableWeight) * unavailableWeight;
1173
+ rawScore += s.value * adjustedWeight;
1174
+ }
1175
+ }
1176
+ let score = rawScore * 10.0;
1177
+ const isCorrection = signals.some(s => s.signal === 'correctionBoost' && s.value > 0);
1178
+ if (isCorrection) {
1179
+ score = Math.min(10.0, score + IMPORTANCE_CONFIG.correctionBonusPoints);
1180
+ }
1181
+ return Math.round(score * 100) / 100;
1182
+ }
864
1183
  /**
865
1184
  * Determine whether consolidation should run based on CONSOLIDATION_CONFIG.frequency.
866
1185
  * Uses AIAgentRun history instead of static in-memory state so that timing
@@ -871,45 +1190,90 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
871
1190
  * - 'hourly': Run if 1+ hour since last completed MM run
872
1191
  * - 'daily': Run if 24+ hours since last completed MM run
873
1192
  * - number: Run every N completed MM runs
1193
+ *
1194
+ * @param forceMaintenance When true, bypasses all gating and runs maintenance
1195
+ * regardless of frequency/history. Set by callers via `params.data.forceMaintenance`
1196
+ * (see `executeAgentInternal`). Intended for tests, manual admin triggers, and
1197
+ * the E2E verification script — NOT for production config. Previously this was
1198
+ * an env var (`FORCE_MAINTENANCE=1`), which was removed because an env-var
1199
+ * sidechannel could accidentally leak into production and cause massive DB/LLM
1200
+ * churn on every run.
1201
+ *
1202
+ * Returns both a boolean and the reason it fired so the observability layer can
1203
+ * record the trigger type on the parent consolidation run step.
874
1204
  */
875
- async shouldRunConsolidation(agentId, contextUser) {
1205
+ async shouldRunConsolidation(agentId, contextUser, forceMaintenance) {
1206
+ if (forceMaintenance)
1207
+ return { shouldRun: true, triggerType: 'forced' };
876
1208
  const freq = CONSOLIDATION_CONFIG.frequency;
877
- if (freq === 'disabled') {
878
- return false;
1209
+ if (freq === 'disabled')
1210
+ return { shouldRun: false, triggerType: 'disabled' };
1211
+ if (freq === 'every-run')
1212
+ return { shouldRun: true, triggerType: 'every-run' };
1213
+ if (freq === 'hourly' || freq === 'daily') {
1214
+ return this.shouldRunConsolidationByTimeWindow(agentId, contextUser, freq === 'hourly' ? 1 : 24);
879
1215
  }
880
- if (freq === 'every-run') {
881
- return true;
1216
+ if (typeof freq === 'number') {
1217
+ return this.shouldRunConsolidationByRunCount(agentId, contextUser, freq);
882
1218
  }
883
- if (freq === 'hourly' || freq === 'daily') {
884
- const thresholdHours = freq === 'hourly' ? 1 : 24;
885
- const lastRun = await this.GetLastRunTime(agentId, contextUser);
886
- if (!lastRun) {
887
- return true; // First run — consolidate
1219
+ return { shouldRun: true, triggerType: 'every-run' };
1220
+ }
1221
+ /**
1222
+ * Time-window frequency gate: returns shouldRun=true with triggerType='time' when
1223
+ * enough time has passed since the last memory-manager run, or triggerType='event'
1224
+ * when the event-driven new-note threshold has been crossed. Falls back to
1225
+ * triggerType='not-triggered' when neither condition fires.
1226
+ */
1227
+ async shouldRunConsolidationByTimeWindow(agentId, contextUser, thresholdHours) {
1228
+ const lastRun = await this.GetLastRunTime(agentId, contextUser);
1229
+ if (!lastRun)
1230
+ return { shouldRun: true, triggerType: 'time' };
1231
+ const hoursSinceLast = (Date.now() - lastRun.getTime()) / (1000 * 60 * 60);
1232
+ if (hoursSinceLast >= thresholdHours)
1233
+ return { shouldRun: true, triggerType: 'time' };
1234
+ // Event-driven trigger: consolidate when at least `noteCountTrigger` new notes
1235
+ // have been added system-wide since the last Memory Manager run.
1236
+ //
1237
+ // NOTE: this query intentionally does NOT filter by AgentID. The `agentId` parameter
1238
+ // here is the Memory Manager's own ID (passed from `params.agent.ID` at the call
1239
+ // site), not the agent that owns the notes. Filtering the count by MM's own agent
1240
+ // ID would never match anything because MM doesn't own notes — it processes notes
1241
+ // belonging to other (memory-enabled) agents.
1242
+ const rv = new RunView();
1243
+ const newNoteResult = await rv.RunView({
1244
+ EntityName: 'MJ: AI Agent Notes',
1245
+ ExtraFilter: `IsAutoGenerated=1 AND Status='Active' AND __mj_CreatedAt > '${lastRun.toISOString()}'`,
1246
+ ResultType: 'count_only'
1247
+ }, contextUser);
1248
+ if (newNoteResult.Success && (newNoteResult.TotalRowCount || 0) >= CONSOLIDATION_CONFIG.noteCountTrigger) {
1249
+ if (this._verbose) {
1250
+ LogStatus(`Memory Manager: Event-driven consolidation trigger fired (${newNoteResult.TotalRowCount} new notes >= ${CONSOLIDATION_CONFIG.noteCountTrigger} threshold since last MM run)`);
888
1251
  }
889
- const hoursSinceLast = (Date.now() - lastRun.getTime()) / (1000 * 60 * 60);
890
- return hoursSinceLast >= thresholdHours;
1252
+ return { shouldRun: true, triggerType: 'event' };
891
1253
  }
892
- // Numeric frequency: run every N completed executions
893
- if (typeof freq === 'number') {
894
- const rv = new RunView();
895
- const countResult = await rv.RunView({
896
- EntityName: 'MJ: AI Agent Runs',
897
- ExtraFilter: `AgentID='${agentId}' AND Status='Completed'`,
898
- ResultType: 'count_only'
899
- }, contextUser);
900
- const completedCount = countResult.Success ? (countResult.TotalRowCount || 0) : 0;
901
- return completedCount > 0 && completedCount % freq === 0;
902
- }
903
- // Default: run every time
904
- return true;
1254
+ return { shouldRun: false, triggerType: 'not-triggered' };
1255
+ }
1256
+ /** Run-count frequency gate: consolidate every N completed runs. */
1257
+ async shouldRunConsolidationByRunCount(agentId, contextUser, everyN) {
1258
+ const rv = new RunView();
1259
+ const countResult = await rv.RunView({
1260
+ EntityName: 'MJ: AI Agent Runs',
1261
+ ExtraFilter: `AgentID='${agentId}' AND Status='Completed'`,
1262
+ ResultType: 'count_only'
1263
+ }, contextUser);
1264
+ const completedCount = countResult.Success ? (countResult.TotalRowCount || 0) : 0;
1265
+ const shouldRun = completedCount > 0 && completedCount % everyN === 0;
1266
+ return { shouldRun, triggerType: shouldRun ? 'count' : 'not-triggered' };
905
1267
  }
906
1268
  /**
907
1269
  * Consolidate related notes into single, comprehensive notes.
908
1270
  * This method finds clusters of similar notes and synthesizes them.
909
1271
  *
1272
+ * Returns per-cluster verification results alongside the consolidation counts so
1273
+ * the phase-level "Verify Consolidation Output" run step can aggregate them.
1274
+ *
910
1275
  * @param agentId Optional - consolidate notes for a specific agent only
911
1276
  * @param contextUser The context user for database operations
912
- * @returns Number of notes consolidated (archived)
913
1277
  */
914
1278
  async consolidateRelatedNotes(agentId, contextUser) {
915
1279
  const allNotes = AIEngine.Instance.AgentNotes;
@@ -921,7 +1285,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
921
1285
  if (this._verbose) {
922
1286
  LogStatus(`Memory Manager: Only ${activeNotes.length} active notes - need at least ${CONSOLIDATION_CONFIG.minClusterSize} to consolidate`);
923
1287
  }
924
- return { consolidated: 0, archived: 0 };
1288
+ return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
925
1289
  }
926
1290
  if (this._verbose) {
927
1291
  LogStatus(`Memory Manager: Analyzing ${activeNotes.length} active notes for consolidation`);
@@ -931,7 +1295,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
931
1295
  if (this._verbose) {
932
1296
  LogStatus(`Memory Manager: No clusters found with ${CONSOLIDATION_CONFIG.minClusterSize}+ similar notes`);
933
1297
  }
934
- return { consolidated: 0, archived: 0 };
1298
+ return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
935
1299
  }
936
1300
  if (this._verbose) {
937
1301
  LogStatus(`Memory Manager: Found ${clusters.length} clusters to consolidate`);
@@ -940,15 +1304,17 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
940
1304
  const consolidatePrompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Consolidate Notes' && p.Category === 'MJ: System');
941
1305
  if (!consolidatePrompt) {
942
1306
  LogError('Memory Manager: Consolidation prompt not found');
943
- return { consolidated: 0, archived: 0 };
1307
+ return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
944
1308
  }
945
1309
  const aiNoteTypeId = AIEngine.Instance.AgenteNoteTypeIDByName('AI');
946
1310
  if (!aiNoteTypeId) {
947
1311
  LogError('Memory Manager: Could not find "AI" note type');
948
- return { consolidated: 0, archived: 0 };
1312
+ return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
949
1313
  }
950
1314
  let consolidated = 0;
951
1315
  let archived = 0;
1316
+ const newNoteIds = [];
1317
+ const verifications = [];
952
1318
  const runner = new AIPromptRunner();
953
1319
  const md = new Metadata();
954
1320
  for (const cluster of clusters) {
@@ -956,12 +1322,16 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
956
1322
  const result = await this.processConsolidationCluster(cluster, consolidatePrompt, aiNoteTypeId, runner, md, contextUser);
957
1323
  consolidated += result.consolidated;
958
1324
  archived += result.archived;
1325
+ if (result.newNoteId)
1326
+ newNoteIds.push(result.newNoteId);
1327
+ if (result.verification)
1328
+ verifications.push(result.verification);
959
1329
  }
960
1330
  catch (error) {
961
1331
  LogError('Memory Manager: Exception during consolidation:', error);
962
1332
  }
963
1333
  }
964
- return { consolidated, archived };
1334
+ return { consolidated, archived, newNoteIds, verifications };
965
1335
  }
966
1336
  /**
967
1337
  * Find clusters of semantically similar notes suitable for consolidation.
@@ -970,7 +1340,12 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
970
1340
  async findConsolidationClusters(activeNotes) {
971
1341
  const clusters = [];
972
1342
  const processedIds = new Set();
973
- for (const note of activeNotes) {
1343
+ // Protection tier filtering: exclude Immutable and Protected notes from consolidation clustering
1344
+ const eligibleNotes = activeNotes.filter(n => {
1345
+ const tier = n.ProtectionTier || 'Standard';
1346
+ return tier !== 'Immutable' && tier !== 'Protected';
1347
+ });
1348
+ for (const note of eligibleNotes) {
974
1349
  if (processedIds.has(note.ID)) {
975
1350
  continue;
976
1351
  }
@@ -979,101 +1354,527 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
979
1354
  processedIds.add(note.ID);
980
1355
  for (const similar of similarNotes) {
981
1356
  if (!processedIds.has(similar.note.ID)) {
982
- cluster.push(similar.note);
983
- processedIds.add(similar.note.ID);
1357
+ // Also check that similar notes pass protection tier filter
1358
+ const similarTier = similar.note.ProtectionTier || 'Standard';
1359
+ if (similarTier !== 'Immutable' && similarTier !== 'Protected') {
1360
+ cluster.push(similar.note);
1361
+ processedIds.add(similar.note.ID);
1362
+ }
984
1363
  }
985
1364
  }
986
- if (cluster.length >= CONSOLIDATION_CONFIG.minClusterSize) {
987
- clusters.push(cluster);
1365
+ // Determine minClusterSize based on tier: Ephemeral uses relaxed threshold of 2
1366
+ const noteTier = note.ProtectionTier || 'Standard';
1367
+ const minSize = noteTier === 'Ephemeral' ? 2 : CONSOLIDATION_CONFIG.minClusterSize;
1368
+ if (cluster.length >= minSize) {
1369
+ // Enforce maxClusterSize by splitting oversized clusters
1370
+ if (cluster.length > CONSOLIDATION_CONFIG.maxClusterSize) {
1371
+ const splitClusters = this.splitOversizedCluster(cluster);
1372
+ clusters.push(...splitClusters);
1373
+ }
1374
+ else {
1375
+ clusters.push(cluster);
1376
+ }
988
1377
  }
989
1378
  }
990
1379
  return clusters;
991
1380
  }
992
1381
  /**
993
- * Process a single consolidation cluster: run the LLM prompt, create the consolidated note,
994
- * and revoke source notes.
1382
+ * Split a cluster that exceeds maxClusterSize into sub-clusters.
1383
+ * Takes the first maxClusterSize notes (sorted by position in original cluster, which preserves
1384
+ * similarity ordering from FindSimilarAgentNotes), and forms a second cluster from the remainder
1385
+ * if it meets minClusterSize.
995
1386
  */
996
- async processConsolidationCluster(cluster, consolidatePrompt, aiNoteTypeId, runner, md, contextUser) {
997
- const promptData = {
998
- notesToConsolidate: cluster.map(n => ({
999
- id: n.ID,
1000
- type: n.Type,
1001
- content: n.Note,
1002
- createdAt: n.__mj_CreatedAt,
1003
- accessCount: n.AccessCount,
1004
- agentId: n.AgentID,
1387
+ splitOversizedCluster(cluster) {
1388
+ const result = [];
1389
+ const maxSize = CONSOLIDATION_CONFIG.maxClusterSize;
1390
+ const minSize = CONSOLIDATION_CONFIG.minClusterSize;
1391
+ // Split into chunks of maxClusterSize
1392
+ for (let i = 0; i < cluster.length; i += maxSize) {
1393
+ const chunk = cluster.slice(i, i + maxSize);
1394
+ if (chunk.length >= minSize) {
1395
+ result.push(chunk);
1396
+ }
1397
+ }
1398
+ return result;
1399
+ }
1400
+ /**
1401
+ * Walk DerivedFromNoteIDs chains breadth-first to find all generation-0 (original) source notes.
1402
+ * Batches each depth level into a single `ID IN (...)` query instead of one query per ID.
1403
+ */
1404
+ async resolveOriginalSources(noteIds, contextUser) {
1405
+ const originals = [];
1406
+ const visited = new Set();
1407
+ let currentIds = noteIds.filter(id => !visited.has(id));
1408
+ currentIds.forEach(id => visited.add(id));
1409
+ let depth = 0;
1410
+ while (currentIds.length > 0 && depth <= MAX_SOURCE_RESOLUTION_DEPTH) {
1411
+ if (depth > MAX_SOURCE_RESOLUTION_DEPTH) {
1412
+ LogError(`Memory Manager: resolveOriginalSources() exceeded max depth ${MAX_SOURCE_RESOLUTION_DEPTH} — treating remaining ${currentIds.length} IDs as terminal to prevent runaway recursion`);
1413
+ break;
1414
+ }
1415
+ const results = await this.loadResolvedNotes(currentIds, contextUser);
1416
+ if (results.length === 0)
1417
+ break;
1418
+ currentIds = this.collectNextLevelSourceIds(results, originals, visited);
1419
+ depth++;
1420
+ }
1421
+ return originals;
1422
+ }
1423
+ /** Load a batch of notes by ID with the fields needed for source resolution. */
1424
+ async loadResolvedNotes(noteIds, contextUser) {
1425
+ const resolvedFields = [
1426
+ 'ID', 'ConsolidationCount', 'DerivedFromNoteIDs', 'Note', 'Type',
1427
+ 'AccessCount', 'ImportanceScore', 'AgentID', 'UserID', 'CompanyID', '__mj_CreatedAt'
1428
+ ];
1429
+ const inClause = noteIds.map(id => `'${id}'`).join(',');
1430
+ const rv = new RunView();
1431
+ const result = await rv.RunView({
1432
+ EntityName: 'MJ: AI Agent Notes',
1433
+ ExtraFilter: `ID IN (${inClause})`,
1434
+ Fields: resolvedFields,
1435
+ }, contextUser);
1436
+ return result.Success && result.Results ? result.Results : [];
1437
+ }
1438
+ /**
1439
+ * Partition a batch of resolved notes into terminal originals (pushed into `originals`)
1440
+ * and parent IDs that need further resolution. Mutates `originals` and `visited`.
1441
+ */
1442
+ collectNextLevelSourceIds(notes, originals, visited) {
1443
+ const nextLevelIds = [];
1444
+ for (const note of notes) {
1445
+ const consolidationCount = note.ConsolidationCount || 0;
1446
+ if (consolidationCount === 0 || !note.DerivedFromNoteIDs) {
1447
+ originals.push(note);
1448
+ continue;
1449
+ }
1450
+ try {
1451
+ const parentIds = JSON.parse(note.DerivedFromNoteIDs);
1452
+ for (const parentId of parentIds) {
1453
+ if (!visited.has(parentId)) {
1454
+ visited.add(parentId);
1455
+ nextLevelIds.push(parentId);
1456
+ }
1457
+ }
1458
+ }
1459
+ catch {
1460
+ LogError(`Memory Manager: Malformed DerivedFromNoteIDs on note ${note.ID} — treating as terminal source`);
1461
+ originals.push(note);
1462
+ }
1463
+ }
1464
+ return nextLevelIds;
1465
+ }
1466
+ /**
1467
+ * Verify that a consolidated note preserves key entities from its source notes.
1468
+ * Extracts numbers, dates, and proper nouns from source notes and checks they appear in the output.
1469
+ * Non-blocking: returns verification results but does not prevent consolidation.
1470
+ */
1471
+ verifyConsolidationOutput(sourceNotes, consolidatedText) {
1472
+ const entities = new Set();
1473
+ const consolidatedLower = consolidatedText.toLowerCase();
1474
+ for (const note of sourceNotes) {
1475
+ this.extractVerifiableEntities(note.Note || '', entities);
1476
+ }
1477
+ const missing = [];
1478
+ for (const entity of entities) {
1479
+ if (!consolidatedLower.includes(entity.toLowerCase())) {
1480
+ missing.push(entity);
1481
+ }
1482
+ }
1483
+ return {
1484
+ entitiesChecked: entities.size,
1485
+ entitiesMissing: missing,
1486
+ passed: missing.length === 0
1487
+ };
1488
+ }
1489
+ /**
1490
+ * Harvest numbers, dates, proper nouns, and camelCase identifiers from a note's text
1491
+ * into the supplied set. Used only by verifyConsolidationOutput to assemble a
1492
+ * best-effort entity inventory for post-consolidation checks.
1493
+ */
1494
+ extractVerifiableEntities(text, entities) {
1495
+ const numbers = text.match(/\b\d+\.?\d*%?\b/g);
1496
+ if (numbers)
1497
+ numbers.forEach(n => entities.add(n));
1498
+ const dates = text.match(/\b\d{4}-\d{2}-\d{2}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\s+\d{1,2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b/gi);
1499
+ if (dates)
1500
+ dates.forEach(d => entities.add(d));
1501
+ const properNouns = text.match(/(?<=[.!?]\s+|^)(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)/g);
1502
+ if (properNouns) {
1503
+ const skipWords = new Set(['the', 'a', 'an', 'this', 'that', 'it', 'we', 'they', 'user', 'note']);
1504
+ properNouns
1505
+ .filter(pn => !skipWords.has(pn.toLowerCase()) && pn.length > 2)
1506
+ .forEach(pn => entities.add(pn));
1507
+ }
1508
+ const techTerms = text.match(/\b[a-z]+(?:[A-Z][a-z]+)+\b|\b(?:[A-Z][a-z]+){2,}\b/g);
1509
+ if (techTerms)
1510
+ techTerms.forEach(t => entities.add(t));
1511
+ }
1512
+ /**
1513
+ * Detect and resolve contradictions between semantically similar notes.
1514
+ * Uses a three-stage pipeline: embedding pre-filter, entity-attribute extraction, LLM judgment.
1515
+ * High-importance pairs (both >= highImportanceThreshold) are flagged but preserved.
1516
+ * Protected-tier notes are never auto-revoked.
1517
+ *
1518
+ * Aggregates the per-pair entity-attribute-value triple count so the observability
1519
+ * step can report how much structural reasoning the LLM produced.
1520
+ */
1521
+ async detectAndResolveContradictions(contextUser, consolidatedNoteIds) {
1522
+ const result = { contradictionsFound: 0, resolved: 0, flagged: 0, pairsAnalyzed: 0, entityTriplesExtracted: 0 };
1523
+ const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Detect Contradictions' && p.Category === 'MJ: System');
1524
+ if (!prompt) {
1525
+ LogError('Memory Manager: Contradiction detection prompt not found, skipping phase');
1526
+ return result;
1527
+ }
1528
+ const candidatePairs = await this.buildContradictionCandidatePairs(consolidatedNoteIds);
1529
+ if (candidatePairs.length === 0)
1530
+ return result;
1531
+ const runner = new AIPromptRunner();
1532
+ const md = new Metadata();
1533
+ for (const [noteA, noteB] of candidatePairs) {
1534
+ result.pairsAnalyzed++;
1535
+ const llmResult = await this.evaluateContradictionPair(noteA, noteB, prompt, runner, contextUser);
1536
+ if (!llmResult)
1537
+ continue;
1538
+ result.entityTriplesExtracted += llmResult.triplesExtracted;
1539
+ if (!llmResult.isContradiction)
1540
+ continue;
1541
+ result.contradictionsFound++;
1542
+ const resolved = await this.resolveContradiction(noteA, noteB, llmResult, md, contextUser);
1543
+ if (resolved === 'flagged')
1544
+ result.flagged++;
1545
+ else if (resolved === 'resolved')
1546
+ result.resolved++;
1547
+ }
1548
+ return result;
1549
+ }
1550
+ /** Build unique candidate pairs of semantically similar notes for contradiction analysis. */
1551
+ async buildContradictionCandidatePairs(consolidatedNoteIds) {
1552
+ const allNotes = AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active' && n.IsAutoGenerated && (n.ProtectionTier || 'Standard') !== 'Immutable');
1553
+ const processedPairs = new Set();
1554
+ const pairs = [];
1555
+ for (const note of allNotes) {
1556
+ if (!note.Note || pairs.length >= CONTRADICTION_CONFIG.maxPairsPerRun)
1557
+ break;
1558
+ const similarNotes = await AIEngine.Instance.FindSimilarAgentNotes(note.Note, note.AgentID, undefined, undefined, 5, CONTRADICTION_CONFIG.similarityThreshold);
1559
+ for (const similar of similarNotes) {
1560
+ const pairKey = [note.ID, similar.note.ID].sort().join('|');
1561
+ if (processedPairs.has(pairKey))
1562
+ continue;
1563
+ processedPairs.add(pairKey);
1564
+ if (consolidatedNoteIds.has(note.ID) && consolidatedNoteIds.has(similar.note.ID))
1565
+ continue;
1566
+ pairs.push([note, similar.note]);
1567
+ if (pairs.length >= CONTRADICTION_CONFIG.maxPairsPerRun)
1568
+ break;
1569
+ }
1570
+ }
1571
+ return pairs;
1572
+ }
1573
+ /** Evaluate a single note pair for contradiction using the LLM prompt. */
1574
+ async evaluateContradictionPair(noteA, noteB, prompt, runner, contextUser) {
1575
+ const params = new AIPromptParams();
1576
+ params.prompt = prompt;
1577
+ params.data = {
1578
+ noteA: { id: noteA.ID, type: noteA.Type, content: noteA.Note, createdAt: noteA.__mj_CreatedAt, accessCount: noteA.AccessCount, importanceScore: noteA.ImportanceScore || null, agentId: noteA.AgentID, userId: noteA.UserID, companyId: noteA.CompanyID },
1579
+ noteB: { id: noteB.ID, type: noteB.Type, content: noteB.Note, createdAt: noteB.__mj_CreatedAt, accessCount: noteB.AccessCount, importanceScore: noteB.ImportanceScore || null, agentId: noteB.AgentID, userId: noteB.UserID, companyId: noteB.CompanyID }
1580
+ };
1581
+ params.contextUser = contextUser;
1582
+ params.attemptJSONRepair = true;
1583
+ params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
1584
+ const llmResult = await runner.ExecutePrompt(params);
1585
+ if (!llmResult.success || !llmResult.result)
1586
+ return null;
1587
+ let parsed = llmResult.result;
1588
+ if (typeof parsed === 'string') {
1589
+ const cleaned = CleanAndParseJSON(parsed, true);
1590
+ if (!cleaned)
1591
+ return null;
1592
+ parsed = cleaned;
1593
+ }
1594
+ return {
1595
+ isContradiction: parsed.isContradiction,
1596
+ keepNoteId: parsed.keepNoteId,
1597
+ revokeNoteId: parsed.revokeNoteId,
1598
+ reason: parsed.reason,
1599
+ triplesExtracted: this.countEntityTriples(parsed.entityTriples)
1600
+ };
1601
+ }
1602
+ /**
1603
+ * Count entity-attribute-value triples returned by the contradiction detection prompt.
1604
+ * Accepts any shape the LLM may produce: object of arrays keyed by noteId, flat array,
1605
+ * or null/undefined. Unrecognized shapes return 0 so a malformed response can't poison
1606
+ * the aggregate counter.
1607
+ */
1608
+ countEntityTriples(entityTriples) {
1609
+ if (!entityTriples)
1610
+ return 0;
1611
+ if (Array.isArray(entityTriples))
1612
+ return entityTriples.length;
1613
+ if (typeof entityTriples === 'object') {
1614
+ let total = 0;
1615
+ for (const value of Object.values(entityTriples)) {
1616
+ if (Array.isArray(value))
1617
+ total += value.length;
1618
+ }
1619
+ return total;
1620
+ }
1621
+ return 0;
1622
+ }
1623
+ /** Resolve a detected contradiction: flag both notes if high-importance/Protected, otherwise revoke the loser. */
1624
+ async resolveContradiction(noteA, noteB, llmResult, md, contextUser) {
1625
+ if (this.shouldFlagContradiction(noteA, noteB)) {
1626
+ await this.flagContradictionPair(noteA, noteB, llmResult, md, contextUser);
1627
+ return 'flagged';
1628
+ }
1629
+ const { revokeNoteId: revokeId, keepNoteId: keepId } = llmResult;
1630
+ if (!revokeId || !keepId)
1631
+ return 'skipped';
1632
+ const noteToRevoke = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
1633
+ if (!(await noteToRevoke.Load(revokeId)))
1634
+ return 'skipped';
1635
+ noteToRevoke.Status = 'Revoked';
1636
+ noteToRevoke.ConsolidatedIntoNoteID = keepId;
1637
+ noteToRevoke.Comments = `[Contradiction resolved: superseded by note ${keepId}. ${llmResult.reason}]`;
1638
+ if (!(await noteToRevoke.Save()))
1639
+ return 'skipped';
1640
+ if (this._verbose) {
1641
+ LogStatus(`Memory Manager: Contradiction resolved — revoked note ${revokeId} in favor of ${keepId}`);
1642
+ }
1643
+ return 'resolved';
1644
+ }
1645
+ /** Decide whether a contradiction pair must be flagged rather than auto-resolved. */
1646
+ shouldFlagContradiction(noteA, noteB) {
1647
+ const bothHighImportance = (noteA.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold &&
1648
+ (noteB.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold;
1649
+ const hasProtected = (noteA.ProtectionTier || 'Standard') === 'Protected' || (noteB.ProtectionTier || 'Standard') === 'Protected';
1650
+ return bothHighImportance || hasProtected;
1651
+ }
1652
+ /** Flag both notes in a contradiction pair with a cross-referencing audit comment. */
1653
+ async flagContradictionPair(noteA, noteB, llmResult, md, contextUser) {
1654
+ const flagComment = `[Contradiction flagged: conflicts with note ${llmResult.revokeNoteId && UUIDsEqual(noteA.ID, llmResult.revokeNoteId) ? llmResult.keepNoteId : llmResult.revokeNoteId}. ${llmResult.reason}]`;
1655
+ for (const flagNote of [noteA, noteB]) {
1656
+ const noteToFlag = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
1657
+ if (await noteToFlag.Load(flagNote.ID)) {
1658
+ noteToFlag.Comments = `${noteToFlag.Comments || ''} ${flagComment}`.trim();
1659
+ await noteToFlag.Save();
1660
+ }
1661
+ }
1662
+ if (this._verbose) {
1663
+ const bothHighImportance = (noteA.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold &&
1664
+ (noteB.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold;
1665
+ const reason = bothHighImportance ? 'both high-importance' : 'Protected tier present';
1666
+ LogStatus(`Memory Manager: Contradiction flagged (${reason}): "${noteA.Note?.substring(0, 40)}..." vs "${noteB.Note?.substring(0, 40)}..."`);
1667
+ }
1668
+ }
1669
+ /**
1670
+ * Process a single consolidation cluster: run the LLM prompt, create the consolidated note,
1671
+ * and revoke source notes.
1672
+ *
1673
+ * Emits a per-cluster "Process Consolidation Cluster" observability step that runs
1674
+ * between the parent consolidation step's create/finalize calls, producing an
1675
+ * implicit parent-child relationship via step-number ordering.
1676
+ */
1677
+ async processConsolidationCluster(cluster, consolidatePrompt, aiNoteTypeId, runner, md, contextUser) {
1678
+ const maxGeneration = Math.max(...cluster.map(n => n.ConsolidationCount || 0));
1679
+ const clusterStep = await this.CreateRunStep('Prompt', 'Process Consolidation Cluster', {
1680
+ clusterSize: cluster.length,
1681
+ noteIds: cluster.map(n => n.ID),
1682
+ maxGeneration
1683
+ });
1684
+ const outcome = {
1685
+ consolidated: 0,
1686
+ archived: 0,
1687
+ newNoteId: null,
1688
+ verification: null
1689
+ };
1690
+ let stepOutput = { clusterSize: cluster.length };
1691
+ try {
1692
+ const { notesForPrompt, promptData } = await this.buildConsolidationPromptData(cluster, contextUser);
1693
+ const parsedResult = await this.runConsolidationPrompt(consolidatePrompt, promptData, runner, contextUser);
1694
+ if (!parsedResult) {
1695
+ stepOutput = { ...stepOutput, shouldConsolidate: false, skipReason: 'prompt-failed' };
1696
+ return outcome;
1697
+ }
1698
+ if (!parsedResult.shouldConsolidate) {
1699
+ if (this._verbose) {
1700
+ LogStatus(`Memory Manager: Skipping cluster consolidation - ${parsedResult.reason}`);
1701
+ }
1702
+ stepOutput = { ...stepOutput, shouldConsolidate: false, skipReason: parsedResult.reason };
1703
+ return outcome;
1704
+ }
1705
+ const consolidatedNoteData = parsedResult.consolidatedNote;
1706
+ const newNote = await this.createConsolidatedNote(cluster, consolidatedNoteData, parsedResult.reason, aiNoteTypeId, md, contextUser);
1707
+ if (!newNote) {
1708
+ stepOutput = { ...stepOutput, shouldConsolidate: true, skipReason: 'save-failed' };
1709
+ return outcome;
1710
+ }
1711
+ outcome.verification = await this.annotateConsolidationVerification(newNote, notesForPrompt, consolidatedNoteData.content);
1712
+ outcome.archived = await this.revokeSourceNotes(cluster, newNote.ID, parsedResult.reason, md, contextUser);
1713
+ outcome.consolidated = 1;
1714
+ outcome.newNoteId = newNote.ID;
1715
+ if (this._verbose) {
1716
+ LogStatus(`Memory Manager: Consolidated ${cluster.length} notes into: "${consolidatedNoteData.content.substring(0, 50)}..."`);
1717
+ }
1718
+ stepOutput = {
1719
+ ...stepOutput,
1720
+ shouldConsolidate: true,
1721
+ consolidatedNoteId: newNote.ID,
1722
+ sourceNotesArchived: outcome.archived,
1723
+ verificationPassed: outcome.verification?.passed ?? null,
1724
+ entitiesChecked: outcome.verification?.entitiesChecked ?? 0,
1725
+ entitiesMissing: outcome.verification?.entitiesMissing ?? []
1726
+ };
1727
+ return outcome;
1728
+ }
1729
+ finally {
1730
+ // Treat skip (shouldConsolidate: false) as a successful completion rather than a
1731
+ // failure — the phase intentionally chose not to consolidate this cluster.
1732
+ await this.FinalizeRunStep(clusterStep, outcome.consolidated === 1 || stepOutput.shouldConsolidate === false, stepOutput);
1733
+ }
1734
+ }
1735
+ /**
1736
+ * Prepare the prompt payload for consolidation. Handles drift prevention: if any source
1737
+ * note is at the consolidation count cap, resolve back to original generation-0 sources
1738
+ * to avoid re-summarizing summaries. Returns both the notes used (for verification) and
1739
+ * the prompt data.
1740
+ */
1741
+ async buildConsolidationPromptData(cluster, contextUser) {
1742
+ const maxConsolidationCount = Math.max(...cluster.map(n => n.ConsolidationCount || 0));
1743
+ let notesForPrompt = cluster;
1744
+ let anchoredMode = false;
1745
+ if (maxConsolidationCount >= CONSOLIDATION_CONFIG.maxConsolidationCount) {
1746
+ const originals = await this.resolveOriginalSources(cluster.map(n => n.ID), contextUser);
1747
+ if (originals.length > 0) {
1748
+ notesForPrompt = originals;
1749
+ if (this._verbose) {
1750
+ LogStatus(`Memory Manager: ConsolidationCount cap reached (${maxConsolidationCount}). Resolved ${cluster.length} notes to ${originals.length} original sources.`);
1751
+ }
1752
+ }
1753
+ }
1754
+ else if (maxConsolidationCount > 0) {
1755
+ anchoredMode = true;
1756
+ }
1757
+ const promptData = {
1758
+ anchoredMode,
1759
+ notesToConsolidate: notesForPrompt.map(n => ({
1760
+ id: n.ID,
1761
+ type: n.Type,
1762
+ content: n.Note,
1763
+ createdAt: n.__mj_CreatedAt,
1764
+ accessCount: n.AccessCount,
1765
+ importanceScore: n.ImportanceScore || null,
1766
+ consolidationCount: n.ConsolidationCount || 0,
1767
+ agentId: n.AgentID,
1005
1768
  userId: n.UserID,
1006
1769
  companyId: n.CompanyID
1007
1770
  }))
1008
1771
  };
1772
+ return { notesForPrompt, promptData };
1773
+ }
1774
+ /**
1775
+ * Execute the consolidation prompt and parse the result. Returns null on any failure
1776
+ * so the caller can short-circuit.
1777
+ */
1778
+ async runConsolidationPrompt(consolidatePrompt, promptData, runner, contextUser) {
1009
1779
  const params = new AIPromptParams();
1010
1780
  params.prompt = consolidatePrompt;
1011
1781
  params.data = promptData;
1012
1782
  params.contextUser = contextUser;
1013
1783
  params.attemptJSONRepair = true;
1784
+ params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
1014
1785
  const result = await runner.ExecutePrompt(params);
1015
1786
  if (!result.success || !result.result) {
1016
1787
  LogError(`Memory Manager: Consolidation prompt failed for cluster: ${result.errorMessage}`);
1017
- return { consolidated: 0, archived: 0 };
1788
+ return null;
1018
1789
  }
1019
- // Parse result if string (some models wrap JSON in ```json fences)
1020
- let parsedResult = result.result;
1021
1790
  if (typeof result.result === 'string') {
1022
1791
  const parsed = CleanAndParseJSON(result.result, true);
1023
1792
  if (!parsed) {
1024
1793
  LogError('Memory Manager: Failed to parse consolidation result');
1025
- return { consolidated: 0, archived: 0 };
1026
- }
1027
- parsedResult = parsed;
1028
- }
1029
- if (!parsedResult.shouldConsolidate) {
1030
- if (this._verbose) {
1031
- LogStatus(`Memory Manager: Skipping cluster consolidation - ${parsedResult.reason}`);
1794
+ return null;
1032
1795
  }
1033
- return { consolidated: 0, archived: 0 };
1796
+ return parsed;
1034
1797
  }
1035
- // Create consolidated note
1036
- const consolidatedNoteData = parsedResult.consolidatedNote;
1798
+ return result.result;
1799
+ }
1800
+ /**
1801
+ * Build, validate, and save the consolidated note. Returns the saved entity on success,
1802
+ * null on save failure. The LLM-returned Type is validated against the allowed union and
1803
+ * falls back to the template note's Type on hallucination — previously a blind `as` cast
1804
+ * silently wrote invalid values.
1805
+ */
1806
+ async createConsolidatedNote(cluster, consolidatedNoteData, reason, aiNoteTypeId, md, contextUser) {
1037
1807
  const newNote = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
1038
1808
  const templateNote = cluster[0];
1809
+ let validatedType;
1810
+ if (isValidNoteType(consolidatedNoteData.type)) {
1811
+ validatedType = consolidatedNoteData.type;
1812
+ }
1813
+ else {
1814
+ LogError(`Memory Manager: Consolidation LLM returned invalid note Type "${consolidatedNoteData.type}" — falling back to template note's Type "${templateNote.Type}"`);
1815
+ validatedType = templateNote.Type;
1816
+ }
1039
1817
  newNote.AgentID = templateNote.AgentID;
1040
1818
  newNote.UserID = templateNote.UserID;
1041
1819
  newNote.CompanyID = templateNote.CompanyID;
1042
1820
  newNote.AgentNoteTypeID = aiNoteTypeId;
1043
- newNote.Type = consolidatedNoteData.type;
1821
+ newNote.Type = validatedType;
1044
1822
  newNote.Note = consolidatedNoteData.content;
1045
1823
  newNote.IsAutoGenerated = true;
1046
1824
  newNote.Status = 'Active';
1047
1825
  newNote.AccessCount = cluster.reduce((sum, n) => sum + (n.AccessCount || 0), 0);
1048
- newNote.Comments = `Consolidated from ${cluster.length} notes: ${parsedResult.reason}`;
1826
+ newNote.Comments = `Consolidated from ${cluster.length} notes: ${reason}`;
1827
+ const maxSourceConsolidationCount = Math.max(...cluster.map(n => n.ConsolidationCount || 0));
1828
+ newNote.ConsolidationCount = maxSourceConsolidationCount + 1;
1829
+ newNote.DerivedFromNoteIDs = JSON.stringify(cluster.map(n => n.ID));
1830
+ newNote.ProtectionTier = 'Standard';
1049
1831
  this.applyScopeToConsolidatedNote(newNote, templateNote, consolidatedNoteData.scopeLevel);
1050
- const saveResult = await newNote.Save();
1051
- if (!saveResult) {
1832
+ if (!(await newNote.Save())) {
1052
1833
  LogError(`Memory Manager: Failed to save consolidated note: ${JSON.stringify(newNote.LatestResult)}`);
1053
- return { consolidated: 0, archived: 0 };
1834
+ return null;
1054
1835
  }
1055
- // Revoke source notes in parallel
1056
- // TODO: When a ConsolidatedIntoNoteID FK column is added to AI Agent Notes,
1057
- // set it here so decomposition can properly restore source notes.
1058
- const revokeResults = await Promise.all(cluster.map(async (sourceNote) => {
1836
+ return newNote;
1837
+ }
1838
+ /**
1839
+ * Run post-consolidation entity verification and, if entities are missing, append a
1840
+ * warning to the saved note's Comments (best-effort, non-blocking). Returns the
1841
+ * verification result so the caller can aggregate it into the phase-level
1842
+ * observability step.
1843
+ */
1844
+ async annotateConsolidationVerification(newNote, notesForPrompt, consolidatedContent) {
1845
+ const verification = this.verifyConsolidationOutput(notesForPrompt, consolidatedContent);
1846
+ if (verification.passed || verification.entitiesMissing.length === 0) {
1847
+ return verification;
1848
+ }
1849
+ const missingList = verification.entitiesMissing.slice(0, 10).join(', ');
1850
+ const warning = `[Verification: potentially missing entities: ${missingList}]`;
1851
+ if (this._verbose) {
1852
+ LogStatus(`Memory Manager: Post-consolidation verification warning — ${verification.entitiesMissing.length} entities not found in consolidated output (${verification.entitiesChecked} checked)`);
1853
+ }
1854
+ newNote.Comments = `${newNote.Comments || ''} ${warning}`.trim();
1855
+ await newNote.Save();
1856
+ return verification;
1857
+ }
1858
+ /**
1859
+ * Revoke source notes in parallel, linking each to the consolidated replacement.
1860
+ * Returns the number of notes successfully revoked.
1861
+ */
1862
+ async revokeSourceNotes(cluster, newNoteId, reason, md, contextUser) {
1863
+ const results = await Promise.all(cluster.map(async (sourceNote) => {
1059
1864
  const noteToRevoke = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
1060
- if (await noteToRevoke.Load(sourceNote.ID)) {
1061
- noteToRevoke.Status = 'Revoked';
1062
- noteToRevoke.Comments = 'Revoked during note consolidation';
1063
- if (await noteToRevoke.Save()) {
1064
- return true;
1065
- }
1066
- else {
1067
- LogError(`Memory Manager: Failed to revoke source note ${sourceNote.ID}`);
1068
- }
1865
+ if (!(await noteToRevoke.Load(sourceNote.ID))) {
1866
+ return false;
1069
1867
  }
1868
+ noteToRevoke.Status = 'Revoked';
1869
+ noteToRevoke.ConsolidatedIntoNoteID = newNoteId;
1870
+ noteToRevoke.Comments = `Consolidated into note ${newNoteId}: ${reason}`;
1871
+ if (await noteToRevoke.Save()) {
1872
+ return true;
1873
+ }
1874
+ LogError(`Memory Manager: Failed to revoke source note ${sourceNote.ID}`);
1070
1875
  return false;
1071
1876
  }));
1072
- const archived = revokeResults.filter(Boolean).length;
1073
- if (this._verbose) {
1074
- LogStatus(`Memory Manager: Consolidated ${cluster.length} notes into: "${consolidatedNoteData.content.substring(0, 50)}..."`);
1075
- }
1076
- return { consolidated: 1, archived };
1877
+ return results.filter(Boolean).length;
1077
1878
  }
1078
1879
  /**
1079
1880
  * Apply scope fields to a consolidated note based on the LLM's scope recommendation.
@@ -1098,6 +1899,577 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
1098
1899
  newNote.SecondaryScopes = templateNote.SecondaryScopes;
1099
1900
  }
1100
1901
  }
1902
+ /**
1903
+ * Prune notes that reference entities which no longer exist or are inactive.
1904
+ * Checks AgentID, UserID, CompanyID, and SourceConversationID against current database state.
1905
+ * Immutable-tier notes are never pruned. Caps at 200 notes per cycle.
1906
+ */
1907
+ async pruneStaleReferences(contextUser) {
1908
+ const maxPerCycle = STALE_PRUNING_CONFIG.maxNotesPerRun;
1909
+ const result = { notesArchived: 0, orphanedAgents: 0, orphanedUsers: 0, orphanedCompanies: 0, orphanedConversations: 0 };
1910
+ const notes = await this.loadPruneCandidateNotes(contextUser, maxPerCycle * 2);
1911
+ if (notes.length === 0) {
1912
+ return result;
1913
+ }
1914
+ const caches = await this.buildReferenceExistenceCache(notes, contextUser);
1915
+ const md = new Metadata();
1916
+ for (const note of notes) {
1917
+ if (result.notesArchived >= maxPerCycle)
1918
+ break;
1919
+ const orphanReasons = this.detectOrphanReasons(note, caches, result);
1920
+ if (orphanReasons.length > 0 && await this.archiveOrphanedNote(note.ID, orphanReasons, md, contextUser)) {
1921
+ result.notesArchived++;
1922
+ }
1923
+ }
1924
+ return result;
1925
+ }
1926
+ /**
1927
+ * Load auto-generated, non-immutable Active notes that are candidates for orphan pruning.
1928
+ */
1929
+ async loadPruneCandidateNotes(contextUser, maxRows) {
1930
+ const rv = new RunView();
1931
+ const notesResult = await rv.RunView({
1932
+ EntityName: 'MJ: AI Agent Notes',
1933
+ ExtraFilter: `IsAutoGenerated = 1 AND Status = 'Active' AND ProtectionTier != 'Immutable'`,
1934
+ Fields: ['ID', 'AgentID', 'UserID', 'CompanyID', 'SourceConversationID'],
1935
+ OrderBy: '__mj_CreatedAt ASC',
1936
+ MaxRows: maxRows,
1937
+ ResultType: 'simple'
1938
+ }, contextUser);
1939
+ if (!notesResult.Success || !notesResult.Results?.length) {
1940
+ return [];
1941
+ }
1942
+ return notesResult.Results;
1943
+ }
1944
+ /**
1945
+ * Build in-memory lookup caches of which referenced users/companies/conversations still
1946
+ * exist and which agents are still Active. Runs all three existence lookups in a single
1947
+ * RunViews batch to minimize round trips.
1948
+ */
1949
+ async buildReferenceExistenceCache(notes, contextUser) {
1950
+ const caches = {
1951
+ activeAgentIds: new Set(AIEngine.Instance.Agents.filter(a => a.Status === 'Active').map(a => a.ID?.toLowerCase())),
1952
+ existingUserIds: new Set(),
1953
+ existingCompanyIds: new Set(),
1954
+ existingConversationIds: new Set()
1955
+ };
1956
+ const { lookupQueries, queryMapping } = this.buildExistenceLookupQueries(notes);
1957
+ if (lookupQueries.length === 0)
1958
+ return caches;
1959
+ const rv = new RunView();
1960
+ const lookupResults = await rv.RunViews(lookupQueries, contextUser);
1961
+ for (let i = 0; i < lookupResults.length; i++) {
1962
+ if (!lookupResults[i].Success)
1963
+ continue;
1964
+ const targetSet = queryMapping[i] === 'users' ? caches.existingUserIds
1965
+ : queryMapping[i] === 'companies' ? caches.existingCompanyIds
1966
+ : caches.existingConversationIds;
1967
+ lookupResults[i].Results.forEach(r => targetSet.add(r.ID?.toLowerCase()));
1968
+ }
1969
+ return caches;
1970
+ }
1971
+ /**
1972
+ * Collect unique referenced IDs from the note batch and assemble a RunViews query list
1973
+ * for existence checks. Returns the query list alongside a parallel mapping array that
1974
+ * tells the caller which target cache each query result belongs in.
1975
+ */
1976
+ buildExistenceLookupQueries(notes) {
1977
+ const referencedUserIds = new Set();
1978
+ const referencedCompanyIds = new Set();
1979
+ const referencedConversationIds = new Set();
1980
+ for (const note of notes) {
1981
+ if (note.UserID)
1982
+ referencedUserIds.add(note.UserID);
1983
+ if (note.CompanyID)
1984
+ referencedCompanyIds.add(note.CompanyID);
1985
+ if (note.SourceConversationID)
1986
+ referencedConversationIds.add(note.SourceConversationID);
1987
+ }
1988
+ const lookupQueries = [];
1989
+ const queryMapping = [];
1990
+ const push = (entityName, ids, kind) => {
1991
+ if (ids.size === 0)
1992
+ return;
1993
+ lookupQueries.push({ EntityName: entityName, ExtraFilter: `ID IN (${Array.from(ids).map(id => `'${id}'`).join(',')})`, Fields: ['ID'] });
1994
+ queryMapping.push(kind);
1995
+ };
1996
+ push('MJ: Users', referencedUserIds, 'users');
1997
+ push('MJ: Companies', referencedCompanyIds, 'companies');
1998
+ push('MJ: Conversations', referencedConversationIds, 'conversations');
1999
+ return { lookupQueries, queryMapping };
2000
+ }
2001
+ /**
2002
+ * Evaluate a single note's foreign-key references against the existence caches, returning
2003
+ * human-readable reasons for any orphaned fields. Increments the supplied counter as a
2004
+ * side effect so aggregate totals are tracked without a second pass.
2005
+ */
2006
+ detectOrphanReasons(note, caches, counter) {
2007
+ const reasons = [];
2008
+ if (note.AgentID && !caches.activeAgentIds.has(note.AgentID.toLowerCase())) {
2009
+ reasons.push('referenced agent is inactive or deleted');
2010
+ counter.orphanedAgents++;
2011
+ }
2012
+ if (note.UserID && !caches.existingUserIds.has(note.UserID.toLowerCase())) {
2013
+ reasons.push('referenced user no longer exists');
2014
+ counter.orphanedUsers++;
2015
+ }
2016
+ if (note.CompanyID && !caches.existingCompanyIds.has(note.CompanyID.toLowerCase())) {
2017
+ reasons.push('referenced company no longer exists');
2018
+ counter.orphanedCompanies++;
2019
+ }
2020
+ if (note.SourceConversationID && !caches.existingConversationIds.has(note.SourceConversationID.toLowerCase())) {
2021
+ reasons.push('source conversation no longer exists');
2022
+ counter.orphanedConversations++;
2023
+ }
2024
+ return reasons;
2025
+ }
2026
+ /**
2027
+ * Archive a single orphaned note with an audit-trail Comments entry. Returns true on success.
2028
+ */
2029
+ async archiveOrphanedNote(noteId, reasons, md, contextUser) {
2030
+ const noteToArchive = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
2031
+ if (!(await noteToArchive.Load(noteId))) {
2032
+ return false;
2033
+ }
2034
+ noteToArchive.Status = 'Archived';
2035
+ noteToArchive.Comments = `[Archived by Memory Manager: stale reference pruning - ${reasons.join('; ')} on ${new Date().toISOString()}]`;
2036
+ if (!(await noteToArchive.Save())) {
2037
+ LogError(`Memory Manager: Failed to archive orphaned note ${noteId}`);
2038
+ return false;
2039
+ }
2040
+ return true;
2041
+ }
2042
+ /**
2043
+ * Compute Ebbinghaus-inspired proportional decay factor for a note.
2044
+ * Returns a multiplier (0-1) to apply to the note's ImportanceScore.
2045
+ * Per research: "decay should be proportional (multiply all strengths by a factor <1),
2046
+ * not threshold-based deletion — this preserves relative importance ordering."
2047
+ */
2048
+ computeDecayFactor(note) {
2049
+ const importanceScore = note.ImportanceScore || 5.0;
2050
+ const normalizedImportance = importanceScore / 10.0;
2051
+ let lambdaEff = DECAY_CONFIG.baseLambda * (1 - normalizedImportance * DECAY_CONFIG.importanceDampening);
2052
+ if (note.ProtectionTier === 'Ephemeral') {
2053
+ lambdaEff *= DECAY_CONFIG.ephemeralDecayMultiplier;
2054
+ }
2055
+ const lastAccess = note.LastAccessedAt || note.__mj_CreatedAt;
2056
+ const daysSinceAccess = (Date.now() - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
2057
+ // Decay factor: how much of the current score to retain.
2058
+ // Higher access count slows decay (well-used notes persist longer), but the
2059
+ // multiplier is CAPPED — without a cap, a note with AccessCount=1000 would get
2060
+ // accessRetention=201 and the `Math.min(1.0, ...)` clamp below would force the
2061
+ // decay factor to 1.0 forever, making frequently-accessed notes effectively
2062
+ // immortal. Capping at ~1.5 means access count can meaningfully slow decay but
2063
+ // cannot indefinitely prevent it.
2064
+ const rawAccessRetention = 1 + (note.AccessCount || 0) * DECAY_CONFIG.accessBoostFactor;
2065
+ const accessRetention = Math.min(DECAY_CONFIG.accessRetentionCap, rawAccessRetention);
2066
+ return Math.min(1.0, Math.exp(-lambdaEff * daysSinceAccess) * accessRetention);
2067
+ }
2068
+ /**
2069
+ * Decay-based archival replacing fixed 90/180-day retention windows.
2070
+ * Uses Ebbinghaus-inspired decay function weighted by ImportanceScore.
2071
+ * Respects protection tiers: Immutable=never, Protected=365d extended, Ephemeral=2x decay.
2072
+ * Handles both notes and examples. Respects per-agent AutoArchiveEnabled.
2073
+ */
2074
+ async decayBasedArchival(contextUser) {
2075
+ const archiveEnabledAgentIds = new Set(AIEngine.Instance.Agents
2076
+ .filter(a => a.AutoArchiveEnabled !== false)
2077
+ .map(a => a.ID?.toLowerCase()));
2078
+ const [expiredResult, decayResult] = await Promise.all([
2079
+ this.archiveExpiredItems(contextUser),
2080
+ this.applyProportionalDecay(contextUser, archiveEnabledAgentIds)
2081
+ ]);
2082
+ return {
2083
+ notesExpired: expiredResult.notesExpired,
2084
+ examplesExpired: expiredResult.examplesExpired,
2085
+ notesArchived: decayResult.notesArchived,
2086
+ examplesArchived: decayResult.examplesArchived,
2087
+ decayScoreDistribution: this.summarizeDecayFactors(decayResult.stats.decayFactors),
2088
+ protectedPreserved: decayResult.stats.protectedPreserved,
2089
+ ephemeralAccelerated: decayResult.stats.ephemeralAccelerated
2090
+ };
2091
+ }
2092
+ /**
2093
+ * Summarize raw decay factors (values in [0,1]) into a min/max/mean triple for
2094
+ * the observability step. Returns null when no factors were computed (empty
2095
+ * cohort or all notes skipped via tier/config).
2096
+ */
2097
+ summarizeDecayFactors(factors) {
2098
+ if (factors.length === 0)
2099
+ return null;
2100
+ let min = factors[0];
2101
+ let max = factors[0];
2102
+ let sum = 0;
2103
+ for (const f of factors) {
2104
+ if (f < min)
2105
+ min = f;
2106
+ if (f > max)
2107
+ max = f;
2108
+ sum += f;
2109
+ }
2110
+ return {
2111
+ min: Math.round(min * 1000) / 1000,
2112
+ max: Math.round(max * 1000) / 1000,
2113
+ mean: Math.round((sum / factors.length) * 1000) / 1000
2114
+ };
2115
+ }
2116
+ /** Archive notes and examples past their explicit ExpiresAt timestamp, regardless of decay score. */
2117
+ async archiveExpiredItems(contextUser) {
2118
+ const result = { notesExpired: 0, examplesExpired: 0 };
2119
+ const md = new Metadata();
2120
+ const nowISO = new Date().toISOString();
2121
+ const [expiredNotes, expiredExamples] = await this.loadExpiredItems(contextUser, nowISO);
2122
+ for (const note of expiredNotes) {
2123
+ if ((note.ProtectionTier || 'Standard') === 'Immutable')
2124
+ continue;
2125
+ if (await this.archiveExpiredNote(note, md, contextUser))
2126
+ result.notesExpired++;
2127
+ }
2128
+ for (const example of expiredExamples) {
2129
+ if (await this.archiveExpiredExample(example, md, contextUser))
2130
+ result.examplesExpired++;
2131
+ }
2132
+ return result;
2133
+ }
2134
+ async loadExpiredItems(contextUser, nowISO) {
2135
+ const rv = new RunView();
2136
+ const [notesResult, examplesResult] = await Promise.all([
2137
+ rv.RunView({
2138
+ EntityName: 'MJ: AI Agent Notes',
2139
+ ExtraFilter: `Status = 'Active' AND ExpiresAt IS NOT NULL AND ExpiresAt < '${nowISO}'`,
2140
+ Fields: ['ID', 'ExpiresAt', 'ProtectionTier'],
2141
+ MaxRows: 200,
2142
+ }, contextUser),
2143
+ rv.RunView({
2144
+ EntityName: 'MJ: AI Agent Examples',
2145
+ ExtraFilter: `Status = 'Active' AND ExpiresAt IS NOT NULL AND ExpiresAt < '${nowISO}'`,
2146
+ Fields: ['ID', 'ExpiresAt'],
2147
+ MaxRows: 200,
2148
+ }, contextUser)
2149
+ ]);
2150
+ return [
2151
+ notesResult.Success && notesResult.Results ? notesResult.Results : [],
2152
+ examplesResult.Success && examplesResult.Results ? examplesResult.Results : []
2153
+ ];
2154
+ }
2155
+ async archiveExpiredNote(note, md, contextUser) {
2156
+ const noteToArchive = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
2157
+ if (!(await noteToArchive.Load(note.ID)))
2158
+ return false;
2159
+ noteToArchive.Status = 'Archived';
2160
+ noteToArchive.Comments = `${noteToArchive.Comments || ''} [Archived by Memory Manager: expired on ${note.ExpiresAt}]`.trim();
2161
+ return await noteToArchive.Save();
2162
+ }
2163
+ async archiveExpiredExample(example, md, contextUser) {
2164
+ const exampleToArchive = await md.GetEntityObject('MJ: AI Agent Examples', contextUser);
2165
+ if (!(await exampleToArchive.Load(example.ID)))
2166
+ return false;
2167
+ exampleToArchive.Status = 'Archived';
2168
+ exampleToArchive.Comments = `${exampleToArchive.Comments || ''} [Archived by Memory Manager: expired on ${example.ExpiresAt}]`.trim();
2169
+ return await exampleToArchive.Save();
2170
+ }
2171
+ /**
2172
+ * Apply proportional decay to ImportanceScores, archiving only when score reaches the floor.
2173
+ * Per research: "decay should be proportional (multiply all strengths by a factor <1),
2174
+ * not threshold-based deletion — this preserves relative importance ordering while reducing overall load."
2175
+ * Notes without ImportanceScore (not yet scored) are skipped entirely.
2176
+ *
2177
+ * Threads a stats accumulator through decayOneNote/decayOneExample to collect
2178
+ * decay-factor distribution and tier-specific counts for observability.
2179
+ */
2180
+ async applyProportionalDecay(contextUser, archiveEnabledAgentIds) {
2181
+ const result = { notesDecayed: 0, notesArchived: 0, examplesDecayed: 0, examplesArchived: 0 };
2182
+ const stats = { decayFactors: [], protectedPreserved: 0, ephemeralAccelerated: 0 };
2183
+ const md = new Metadata();
2184
+ const [activeNotes, activeExamples] = await this.loadDecayCandidates(contextUser);
2185
+ for (const note of activeNotes) {
2186
+ await this.decayOneNote(note, archiveEnabledAgentIds, md, contextUser, result, stats);
2187
+ }
2188
+ for (const example of activeExamples) {
2189
+ await this.decayOneExample(example, archiveEnabledAgentIds, md, contextUser, result, stats);
2190
+ }
2191
+ return { ...result, stats };
2192
+ }
2193
+ /**
2194
+ * Fetch notes and examples in parallel — they're independent.
2195
+ */
2196
+ async loadDecayCandidates(contextUser) {
2197
+ const rv = new RunView();
2198
+ const [activeNotesResult, activeExamplesResult] = await Promise.all([
2199
+ rv.RunView({
2200
+ EntityName: 'MJ: AI Agent Notes',
2201
+ ExtraFilter: `IsAutoGenerated = 1 AND Status = 'Active'`,
2202
+ Fields: ['ID', 'ImportanceScore', 'ProtectionTier', 'LastAccessedAt', '__mj_CreatedAt', 'AccessCount', 'AgentID'],
2203
+ MaxRows: 1000,
2204
+ }, contextUser),
2205
+ rv.RunView({
2206
+ EntityName: 'MJ: AI Agent Examples',
2207
+ ExtraFilter: `IsAutoGenerated = 1 AND Status = 'Active' AND SuccessScore < 50`,
2208
+ Fields: ['ID', 'SuccessScore', 'LastAccessedAt', '__mj_CreatedAt', 'AccessCount', 'AgentID'],
2209
+ MaxRows: 500,
2210
+ }, contextUser)
2211
+ ]);
2212
+ return [
2213
+ activeNotesResult.Success && activeNotesResult.Results ? activeNotesResult.Results : [],
2214
+ activeExamplesResult.Success && activeExamplesResult.Results ? activeExamplesResult.Results : []
2215
+ ];
2216
+ }
2217
+ /**
2218
+ * Apply proportional decay to a single note. Archives if the decayed score drops below
2219
+ * the archival floor, otherwise writes the reduced score (skipping when the factor is
2220
+ * effectively 1.0 to avoid unnecessary saves).
2221
+ */
2222
+ async decayOneNote(note, archiveEnabledAgentIds, md, contextUser, result, stats) {
2223
+ if (note.AgentID && !archiveEnabledAgentIds.has(note.AgentID.toLowerCase()))
2224
+ return;
2225
+ if (note.ImportanceScore === null)
2226
+ return;
2227
+ const tier = note.ProtectionTier || 'Standard';
2228
+ if (tier === 'Immutable')
2229
+ return;
2230
+ if (tier === 'Protected') {
2231
+ const lastAccess = note.LastAccessedAt || note.__mj_CreatedAt;
2232
+ const daysSinceAccess = (Date.now() - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
2233
+ if (daysSinceAccess < DECAY_CONFIG.protectedInactivityDays) {
2234
+ // Protected note within the extended-retention window — evaluated but preserved.
2235
+ stats.protectedPreserved++;
2236
+ return;
2237
+ }
2238
+ }
2239
+ if (tier === 'Ephemeral') {
2240
+ stats.ephemeralAccelerated++;
2241
+ }
2242
+ const decayFactor = this.computeDecayFactor(note);
2243
+ stats.decayFactors.push(decayFactor);
2244
+ const newScore = Math.round(note.ImportanceScore * decayFactor * 100) / 100;
2245
+ const noteToUpdate = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
2246
+ if (!(await noteToUpdate.Load(note.ID)))
2247
+ return;
2248
+ if (newScore < DECAY_CONFIG.archivalFloor) {
2249
+ noteToUpdate.Status = 'Archived';
2250
+ noteToUpdate.ImportanceScore = newScore;
2251
+ noteToUpdate.Comments = `${noteToUpdate.Comments || ''} [Archived by Memory Manager: ImportanceScore decayed to ${newScore.toFixed(2)}, below floor ${DECAY_CONFIG.archivalFloor}]`.trim();
2252
+ if (await noteToUpdate.Save())
2253
+ result.notesArchived++;
2254
+ }
2255
+ else if (decayFactor < 0.99) {
2256
+ noteToUpdate.ImportanceScore = newScore;
2257
+ if (await noteToUpdate.Save())
2258
+ result.notesDecayed++;
2259
+ }
2260
+ }
2261
+ /**
2262
+ * Apply proportional decay to a single example, using SuccessScore as a proxy importance.
2263
+ * Archives if the effective score drops below the archival floor.
2264
+ */
2265
+ async decayOneExample(example, archiveEnabledAgentIds, md, contextUser, result, stats) {
2266
+ if (example.AgentID && !archiveEnabledAgentIds.has(example.AgentID.toLowerCase()))
2267
+ return;
2268
+ const normalizedImportance = (example.SuccessScore || 25) / 100.0;
2269
+ const lambdaEff = DECAY_CONFIG.baseLambda * (1 - normalizedImportance * DECAY_CONFIG.importanceDampening);
2270
+ const lastAccess = example.LastAccessedAt || example.__mj_CreatedAt;
2271
+ const daysSinceAccess = (Date.now() - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
2272
+ const decayFactor = Math.min(1.0, Math.exp(-lambdaEff * daysSinceAccess) * (1 + (example.AccessCount || 0) * DECAY_CONFIG.accessBoostFactor));
2273
+ stats.decayFactors.push(decayFactor);
2274
+ const effectiveScore = normalizedImportance * 10.0 * decayFactor;
2275
+ if (effectiveScore >= DECAY_CONFIG.archivalFloor)
2276
+ return;
2277
+ const exampleToArchive = await md.GetEntityObject('MJ: AI Agent Examples', contextUser);
2278
+ if (!(await exampleToArchive.Load(example.ID)))
2279
+ return;
2280
+ exampleToArchive.Status = 'Archived';
2281
+ exampleToArchive.Comments = `${exampleToArchive.Comments || ''} [Archived by Memory Manager: effective score ${effectiveScore.toFixed(2)} below floor]`.trim();
2282
+ if (await exampleToArchive.Save())
2283
+ result.examplesArchived++;
2284
+ }
2285
+ /**
2286
+ * Run all 5 maintenance phases in order: importance scoring, consolidation,
2287
+ * contradiction detection, stale-reference pruning, and decay-based archival.
2288
+ * Each phase is wrapped in try/catch so a failure in one does not abort the rest.
2289
+ *
2290
+ * `triggerType` flows through to the parent consolidation run step so observability
2291
+ * records *why* the maintenance cycle fired this run.
2292
+ */
2293
+ async runMaintenancePhases(contextUser, forceMaintenance, triggerType) {
2294
+ const r = {
2295
+ consolidatedCount: 0, consolidationArchived: 0,
2296
+ contradictionsFound: 0, contradictionsResolved: 0, contradictionsFlagged: 0,
2297
+ staleNotesArchived: 0, decayNotesArchived: 0, decayExamplesArchived: 0,
2298
+ importanceScored: 0, tierPromotions: 0
2299
+ };
2300
+ LogStatus(`Memory Manager: Running maintenance phases${forceMaintenance ? ' (forced via params.data.forceMaintenance)' : ''}...`);
2301
+ await this.runImportancePhase(r, contextUser);
2302
+ const consolidatedNoteIds = await this.runConsolidationPhase(r, contextUser, triggerType);
2303
+ await this.runContradictionPhase(r, contextUser, consolidatedNoteIds);
2304
+ await this.runPruneAndDecayPhases(r, contextUser);
2305
+ return r;
2306
+ }
2307
+ async runImportancePhase(r, contextUser) {
2308
+ const importanceStep = await this.CreateRunStep('Decision', 'Compute Importance Scores', {
2309
+ activeNoteCount: AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active').length
2310
+ });
2311
+ try {
2312
+ const scoringResult = await this.computeImportanceScores(contextUser);
2313
+ r.importanceScored = scoringResult.notesScored;
2314
+ r.tierPromotions = scoringResult.tierPromotions;
2315
+ if (this._verbose)
2316
+ LogStatus(`Memory Manager: Scored ${r.importanceScored} notes, ${r.tierPromotions} tier promotions`);
2317
+ await this.FinalizeRunStep(importanceStep, true, {
2318
+ notesScored: r.importanceScored,
2319
+ tierPromotions: r.tierPromotions,
2320
+ scoreDistribution: scoringResult.scoreDistribution
2321
+ });
2322
+ }
2323
+ catch (error) {
2324
+ LogError('Memory Manager: Importance scoring failed, continuing with other phases:', error);
2325
+ await this.FinalizeRunStep(importanceStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
2326
+ }
2327
+ }
2328
+ async runConsolidationPhase(r, contextUser, triggerType) {
2329
+ const consolidatedNoteIds = new Set();
2330
+ const consolidationStep = await this.CreateRunStep('Decision', 'Consolidate Related Notes', {
2331
+ frequency: CONSOLIDATION_CONFIG.frequency,
2332
+ triggerType,
2333
+ activeNoteCount: AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active' && n.IsAutoGenerated).length
2334
+ });
2335
+ let consolidationResult = null;
2336
+ try {
2337
+ consolidationResult = await this.consolidateRelatedNotes(null, contextUser);
2338
+ r.consolidatedCount = consolidationResult.consolidated;
2339
+ r.consolidationArchived = consolidationResult.archived;
2340
+ for (const id of consolidationResult.newNoteIds)
2341
+ consolidatedNoteIds.add(id);
2342
+ if (r.consolidatedCount > 0 && this._verbose) {
2343
+ LogStatus(`Memory Manager: Consolidated ${r.consolidatedCount} clusters, archived ${r.consolidationArchived} source notes`);
2344
+ }
2345
+ await this.FinalizeRunStep(consolidationStep, true, {
2346
+ consolidatedClusterCount: r.consolidatedCount,
2347
+ sourceNotesArchived: r.consolidationArchived,
2348
+ newConsolidatedNoteIds: Array.from(consolidatedNoteIds)
2349
+ });
2350
+ }
2351
+ catch (error) {
2352
+ LogError('Memory Manager: Consolidation failed, continuing with other phases:', error);
2353
+ await this.FinalizeRunStep(consolidationStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
2354
+ }
2355
+ // Emit the phase-level verification run step (spec Task 8c). Runs whether or not
2356
+ // any clusters were consolidated so "no items" runs still produce a visible step.
2357
+ await this.emitVerificationRunStep(consolidationResult?.verifications ?? []);
2358
+ return consolidatedNoteIds;
2359
+ }
2360
+ /** Aggregate per-cluster verification results into a single phase-level run step. */
2361
+ async emitVerificationRunStep(verifications) {
2362
+ const verificationStep = await this.CreateRunStep('Validation', 'Verify Consolidation Output', {
2363
+ clustersVerified: verifications.length
2364
+ });
2365
+ const verificationsFlagged = verifications.filter(v => !v.passed).length;
2366
+ const verificationsPassed = verifications.filter(v => v.passed).length;
2367
+ const entitiesChecked = verifications.reduce((sum, v) => sum + v.entitiesChecked, 0);
2368
+ const entitiesMissing = verifications.flatMap(v => v.entitiesMissing);
2369
+ await this.FinalizeRunStep(verificationStep, true, {
2370
+ clustersVerified: verifications.length,
2371
+ entitiesChecked,
2372
+ entitiesMissing,
2373
+ verificationsPassed,
2374
+ verificationsFlagged
2375
+ });
2376
+ }
2377
+ async runContradictionPhase(r, contextUser, consolidatedNoteIds) {
2378
+ const contradictionStep = await this.CreateRunStep('Decision', 'Detect Note Contradictions', {
2379
+ consolidatedNoteIdsCount: consolidatedNoteIds.size
2380
+ });
2381
+ try {
2382
+ const contradictionResult = await this.detectAndResolveContradictions(contextUser, consolidatedNoteIds);
2383
+ r.contradictionsFound = contradictionResult.contradictionsFound;
2384
+ r.contradictionsResolved = contradictionResult.resolved;
2385
+ r.contradictionsFlagged = contradictionResult.flagged;
2386
+ if (r.contradictionsFound > 0 && this._verbose) {
2387
+ LogStatus(`Memory Manager: Found ${r.contradictionsFound} contradictions — resolved ${r.contradictionsResolved}, flagged ${r.contradictionsFlagged}`);
2388
+ }
2389
+ await this.FinalizeRunStep(contradictionStep, true, {
2390
+ pairsAnalyzed: contradictionResult.pairsAnalyzed,
2391
+ contradictionsFound: r.contradictionsFound,
2392
+ contradictionsResolved: r.contradictionsResolved,
2393
+ contradictionsFlagged: r.contradictionsFlagged,
2394
+ entityTriplesExtracted: contradictionResult.entityTriplesExtracted
2395
+ });
2396
+ }
2397
+ catch (error) {
2398
+ LogError('Memory Manager: Contradiction detection failed, continuing with other phases:', error);
2399
+ await this.FinalizeRunStep(contradictionStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
2400
+ }
2401
+ }
2402
+ /**
2403
+ * Prune and decay are independent — run in parallel via Promise.allSettled so one failure
2404
+ * doesn't block the other.
2405
+ */
2406
+ async runPruneAndDecayPhases(r, contextUser) {
2407
+ const pruneStep = await this.CreateRunStep('Decision', 'Prune Stale References', {});
2408
+ const decayStep = await this.CreateRunStep('Decision', 'Decay-Based Archival', {});
2409
+ await Promise.allSettled([
2410
+ this.runPrunePhase(r, contextUser, pruneStep),
2411
+ this.runDecayPhase(r, contextUser, decayStep),
2412
+ ]);
2413
+ }
2414
+ async runPrunePhase(r, contextUser, pruneStep) {
2415
+ try {
2416
+ const pruneResult = await this.pruneStaleReferences(contextUser);
2417
+ r.staleNotesArchived = pruneResult.notesArchived;
2418
+ if (r.staleNotesArchived > 0 && this._verbose) {
2419
+ LogStatus(`Memory Manager: Pruned ${r.staleNotesArchived} orphaned notes`);
2420
+ }
2421
+ await this.FinalizeRunStep(pruneStep, true, {
2422
+ notesArchived: r.staleNotesArchived,
2423
+ orphanedAgents: pruneResult.orphanedAgents,
2424
+ orphanedUsers: pruneResult.orphanedUsers,
2425
+ orphanedCompanies: pruneResult.orphanedCompanies,
2426
+ orphanedConversations: pruneResult.orphanedConversations
2427
+ });
2428
+ }
2429
+ catch (error) {
2430
+ LogError('Memory Manager: Stale reference pruning failed:', error);
2431
+ await this.FinalizeRunStep(pruneStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
2432
+ }
2433
+ }
2434
+ async runDecayPhase(r, contextUser, decayStep) {
2435
+ try {
2436
+ const decayResult = await this.decayBasedArchival(contextUser);
2437
+ r.decayNotesArchived = decayResult.notesArchived;
2438
+ r.decayExamplesArchived = decayResult.examplesArchived;
2439
+ const totalDecayArchived = decayResult.notesArchived + decayResult.examplesArchived + decayResult.notesExpired + decayResult.examplesExpired;
2440
+ if (totalDecayArchived > 0 && this._verbose) {
2441
+ LogStatus(`Memory Manager: Decay archival — ${decayResult.notesArchived} notes, ${decayResult.examplesArchived} examples, ${decayResult.notesExpired} expired notes, ${decayResult.examplesExpired} expired examples`);
2442
+ }
2443
+ await this.FinalizeRunStep(decayStep, true, {
2444
+ notesArchived: decayResult.notesArchived,
2445
+ examplesArchived: decayResult.examplesArchived,
2446
+ notesExpired: decayResult.notesExpired,
2447
+ examplesExpired: decayResult.examplesExpired,
2448
+ decayScoreDistribution: decayResult.decayScoreDistribution,
2449
+ protectedPreserved: decayResult.protectedPreserved,
2450
+ ephemeralAccelerated: decayResult.ephemeralAccelerated
2451
+ });
2452
+ }
2453
+ catch (error) {
2454
+ LogError('Memory Manager: Decay-based archival failed:', error);
2455
+ await this.FinalizeRunStep(decayStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
2456
+ }
2457
+ }
2458
+ /** Build a human-readable summary string from maintenance phase results. */
2459
+ buildMaintenanceSummary(m) {
2460
+ const parts = [];
2461
+ if (m.consolidatedCount > 0)
2462
+ parts.push(`consolidated ${m.consolidatedCount} clusters`);
2463
+ if (m.contradictionsResolved > 0)
2464
+ parts.push(`resolved ${m.contradictionsResolved} contradictions`);
2465
+ if (m.contradictionsFlagged > 0)
2466
+ parts.push(`flagged ${m.contradictionsFlagged} high-importance contradictions`);
2467
+ if (m.staleNotesArchived > 0)
2468
+ parts.push(`pruned ${m.staleNotesArchived} stale notes`);
2469
+ if (m.decayNotesArchived + m.decayExamplesArchived > 0)
2470
+ parts.push(`decay-archived ${m.decayNotesArchived} notes + ${m.decayExamplesArchived} examples`);
2471
+ return parts.length > 0 ? ` Maintenance: ${parts.join(', ')}.` : '';
2472
+ }
1101
2473
  /**
1102
2474
  * Create example records from extracted data.
1103
2475
  * Inherits scope from source agent run and applies scopeLevel to determine scope specificity.
@@ -1150,7 +2522,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
1150
2522
  example.Status = 'Active'; // Auto-approve high-confidence examples
1151
2523
  example.SourceConversationID = extracted.sourceConversationId || null;
1152
2524
  // Only use if it's a valid UUID
1153
- example.SourceConversationDetailID = this.isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
2525
+ example.SourceConversationDetailID = MemoryManagerAgent_1.isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
1154
2526
  example.SourceAIAgentRunID = extracted.sourceAgentRunId || null;
1155
2527
  // Apply scope from source agent run based on scopeLevel hint
1156
2528
  if (sourceRun && sourceRun.PrimaryScopeEntityID) {
@@ -1201,7 +2573,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
1201
2573
  async executeAgentInternal(params, config) {
1202
2574
  try {
1203
2575
  // Use verbose flag from agent execution params or data payload (UI passes it via data)
1204
- this._verbose = params.verbose ?? params.data?.verbose ?? false;
2576
+ this._verbose = params.verbose || params.data?.verbose || false;
1205
2577
  // Initialize observability state for this run
1206
2578
  this._agentRunID = this.AgentRun?.ID || null;
1207
2579
  this._stepCounter = 0;
@@ -1252,112 +2624,90 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
1252
2624
  if (this._verbose) {
1253
2625
  LogStatus(`Memory Manager: Found ${agentRuns.length} high-value agent runs`);
1254
2626
  }
1255
- if (conversations.length === 0 && agentRuns.length === 0) {
1256
- if (this._verbose) {
1257
- LogStatus('Memory Manager: No data to process');
1258
- }
1259
- const finalStep = {
1260
- terminate: true,
1261
- step: 'Success',
1262
- message: 'No conversations with new activity or agent runs to process'
1263
- };
1264
- return { finalStep, stepCount: 1 };
1265
- }
2627
+ const hasNewData = conversations.length > 0 || agentRuns.length > 0;
1266
2628
  // Initialization guard: ensure vector services exist for semantic dedup.
1267
2629
  // If no notes existed at server startup, _noteVectorService is null and
1268
2630
  // AddOrUpdateSingleNoteEmbedding will throw when saving new notes.
1269
2631
  // The two post-creation/post-consolidation Config(true) calls were removed
1270
2632
  // because entity save hooks now update vectors incrementally.
1271
2633
  await AIEngine.Instance.Config(true, params.contextUser);
1272
- // Extract notes from conversations (with rating context)
1273
- const extractedNotes = await this.ExtractNotesFromConversations(conversations, params.contextUser);
1274
- if (this._verbose) {
1275
- LogStatus(`Memory Manager: Extracted ${extractedNotes.length} potential notes`);
1276
- }
1277
- // Extract examples from conversation details (legacy method for now)
1278
- // Convert conversations back to flat details for example extraction
1279
- const conversationDetails = conversations.flatMap(conv => conv.messages.map(msg => ({
1280
- ID: msg.id,
1281
- ConversationID: conv.conversationId,
1282
- Role: msg.role,
1283
- Message: msg.message,
1284
- Status: 'Complete',
1285
- __mj_CreatedAt: msg.createdAt
1286
- })));
1287
- const extractedExamples = await this.ExtractExamples(conversationDetails, params.contextUser);
1288
- if (this._verbose) {
1289
- LogStatus(`Memory Manager: Extracted ${extractedExamples.length} potential examples`);
1290
- }
1291
- // Enrich examples with userId and agentRunId from conversation context
1292
- // Build a map from conversationId to context (same as notes enrichment)
1293
- const conversationContextForExamples = new Map();
1294
- for (const conv of conversations) {
1295
- conversationContextForExamples.set(conv.conversationId, {
1296
- userId: conv.userId || null,
1297
- agentRunId: conv.agentRunId || null
1298
- });
1299
- }
1300
- for (const example of extractedExamples) {
1301
- if (example.sourceConversationId) {
1302
- const ctx = conversationContextForExamples.get(example.sourceConversationId);
1303
- if (ctx) {
1304
- // Enrich userId if not set
1305
- if (!example.userId && ctx.userId) {
1306
- example.userId = ctx.userId;
1307
- if (this._verbose) {
1308
- LogStatus(`Memory Manager: Enriched example with userId from conversation: ${ctx.userId}`);
2634
+ // Extract notes and examples only when there's new data to process
2635
+ let notesCreated = 0;
2636
+ let examplesCreated = 0;
2637
+ if (hasNewData) {
2638
+ const extractedNotes = await this.ExtractNotesFromConversations(conversations, params.contextUser);
2639
+ if (this._verbose) {
2640
+ LogStatus(`Memory Manager: Extracted ${extractedNotes.length} potential notes`);
2641
+ }
2642
+ // Convert conversations back to flat details for example extraction.
2643
+ // ExtractExamples accepts a ConversationDetailProjection, so we only
2644
+ // need the fields it reads — no entity cast.
2645
+ const conversationDetails = conversations.flatMap(conv => conv.messages.map(msg => ({
2646
+ ID: msg.id,
2647
+ ConversationID: conv.conversationId,
2648
+ Role: msg.role,
2649
+ Message: msg.message,
2650
+ __mj_CreatedAt: msg.createdAt
2651
+ })));
2652
+ const extractedExamples = await this.ExtractExamples(conversationDetails, params.contextUser);
2653
+ if (this._verbose) {
2654
+ LogStatus(`Memory Manager: Extracted ${extractedExamples.length} potential examples`);
2655
+ }
2656
+ // Enrich examples with userId and agentRunId from conversation context
2657
+ const conversationContextForExamples = new Map();
2658
+ for (const conv of conversations) {
2659
+ conversationContextForExamples.set(conv.conversationId, {
2660
+ userId: conv.userId || null,
2661
+ agentRunId: conv.agentRunId || null
2662
+ });
2663
+ }
2664
+ for (const example of extractedExamples) {
2665
+ if (example.sourceConversationId) {
2666
+ const ctx = conversationContextForExamples.get(example.sourceConversationId);
2667
+ if (ctx) {
2668
+ if (!example.userId && ctx.userId) {
2669
+ example.userId = ctx.userId;
2670
+ if (this._verbose) {
2671
+ LogStatus(`Memory Manager: Enriched example with userId from conversation: ${ctx.userId}`);
2672
+ }
1309
2673
  }
1310
- }
1311
- // Enrich agentRunId if not set
1312
- if (!example.sourceAgentRunId && ctx.agentRunId) {
1313
- example.sourceAgentRunId = ctx.agentRunId;
1314
- if (this._verbose) {
1315
- LogStatus(`Memory Manager: Enriched example with agentRunId from conversation: ${ctx.agentRunId}`);
2674
+ if (!example.sourceAgentRunId && ctx.agentRunId) {
2675
+ example.sourceAgentRunId = ctx.agentRunId;
2676
+ if (this._verbose) {
2677
+ LogStatus(`Memory Manager: Enriched example with agentRunId from conversation: ${ctx.agentRunId}`);
2678
+ }
1316
2679
  }
1317
2680
  }
1318
2681
  }
1319
- }
1320
- // Clear invalid agentId (LLM sometimes returns placeholder values like "agent-12345")
1321
- if (example.agentId && !this.isValidUUID(example.agentId)) {
1322
- if (this._verbose) {
1323
- LogStatus(`Memory Manager: Clearing invalid agentId "${example.agentId}" from example`);
2682
+ if (example.agentId && !MemoryManagerAgent_1.isValidUUID(example.agentId)) {
2683
+ if (this._verbose) {
2684
+ LogStatus(`Memory Manager: Clearing invalid agentId "${example.agentId}" from example`);
2685
+ }
2686
+ example.agentId = '';
1324
2687
  }
1325
- example.agentId = '';
1326
- }
1327
- }
1328
- // Create records
1329
- const notesCreated = await this.CreateNoteRecords(extractedNotes, params.contextUser);
1330
- const examplesCreated = await this.CreateExampleRecords(extractedExamples, params.contextUser);
1331
- LogStatus(`Memory Manager: Created ${notesCreated} notes and ${examplesCreated} examples`);
1332
- // Step 9: Consolidate related notes
1333
- // This finds clusters of similar notes and synthesizes them into single comprehensive notes
1334
- let consolidatedCount = 0;
1335
- let archivedCount = 0;
1336
- if (await this.shouldRunConsolidation(params.agent.ID, params.contextUser)) {
1337
- LogStatus(`Memory Manager: Running note consolidation...`);
1338
- const consolidationResult = await this.consolidateRelatedNotes(null, params.contextUser);
1339
- consolidatedCount = consolidationResult.consolidated;
1340
- archivedCount = consolidationResult.archived;
1341
- if (consolidatedCount > 0 || archivedCount > 0) {
1342
- LogStatus(`Memory Manager: Consolidated ${consolidatedCount} note clusters, archived ${archivedCount} source notes`);
1343
- }
1344
- else {
1345
- LogStatus(`Memory Manager: No notes to consolidate`);
1346
2688
  }
2689
+ notesCreated = await this.CreateNoteRecords(extractedNotes, params.contextUser);
2690
+ examplesCreated = await this.CreateExampleRecords(extractedExamples, params.contextUser);
2691
+ if (this._verbose)
2692
+ LogStatus(`Memory Manager: Created ${notesCreated} notes and ${examplesCreated} examples`);
1347
2693
  }
1348
- const consolidationSummary = consolidatedCount > 0 ? ` Consolidated ${consolidatedCount} clusters (${archivedCount} notes archived).` : '';
2694
+ const forceMaintenance = params.data?.forceMaintenance === true;
2695
+ const triggerDecision = await this.shouldRunConsolidation(params.agent.ID, params.contextUser, forceMaintenance);
2696
+ const maintenance = triggerDecision.shouldRun
2697
+ ? await this.runMaintenancePhases(params.contextUser, forceMaintenance, triggerDecision.triggerType)
2698
+ : { consolidatedCount: 0, consolidationArchived: 0, contradictionsFound: 0, contradictionsResolved: 0, contradictionsFlagged: 0, staleNotesArchived: 0, decayNotesArchived: 0, decayExamplesArchived: 0, importanceScored: 0, tierPromotions: 0 };
2699
+ const maintenanceSummary = this.buildMaintenanceSummary(maintenance);
1349
2700
  const finalStep = {
1350
2701
  terminate: true,
1351
2702
  step: 'Success',
1352
- message: `Processed ${conversations.length} conversations (${totalMessages} messages) and ${agentRuns.length} agent runs. Created ${notesCreated} notes and ${examplesCreated} examples.${consolidationSummary}`,
2703
+ message: `Processed ${conversations.length} conversations (${totalMessages} messages) and ${agentRuns.length} agent runs. Created ${notesCreated} notes and ${examplesCreated} examples.${maintenanceSummary}`,
1353
2704
  newPayload: {
1354
2705
  notesCreated,
1355
2706
  examplesCreated,
1356
2707
  conversationsProcessed: conversations.length,
1357
2708
  messagesProcessed: totalMessages,
1358
2709
  agentRunsProcessed: agentRuns.length,
1359
- notesConsolidated: consolidatedCount,
1360
- notesArchived: archivedCount
2710
+ ...maintenance
1361
2711
  }
1362
2712
  };
1363
2713
  return { finalStep, stepCount: 1 };
@@ -1373,7 +2723,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
1373
2723
  }
1374
2724
  }
1375
2725
  };
1376
- MemoryManagerAgent = __decorate([
2726
+ MemoryManagerAgent = MemoryManagerAgent_1 = __decorate([
1377
2727
  RegisterClass(BaseAgent, 'MemoryManagerAgent')
1378
2728
  ], MemoryManagerAgent);
1379
2729
  export { MemoryManagerAgent };