@memberjunction/ai-agents 5.29.0 → 5.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AgentRunner.js +1 -1
- package/dist/AgentRunner.js.map +1 -1
- package/dist/base-agent.d.ts +7 -0
- package/dist/base-agent.d.ts.map +1 -1
- package/dist/base-agent.js +68 -3
- package/dist/base-agent.js.map +1 -1
- package/dist/memory-cleanup-agent.d.ts +10 -3
- package/dist/memory-cleanup-agent.d.ts.map +1 -1
- package/dist/memory-cleanup-agent.js +10 -3
- package/dist/memory-cleanup-agent.js.map +1 -1
- package/dist/memory-manager-agent.d.ts +341 -7
- package/dist/memory-manager-agent.d.ts.map +1 -1
- package/dist/memory-manager-agent.js +1723 -373
- package/dist/memory-manager-agent.js.map +1 -1
- package/package.json +15 -15
|
@@ -4,6 +4,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
4
4
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
5
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
6
|
};
|
|
7
|
+
var MemoryManagerAgent_1;
|
|
7
8
|
import { BaseAgent } from './base-agent.js';
|
|
8
9
|
import { RegisterClass, CleanAndParseJSON } from '@memberjunction/global';
|
|
9
10
|
import { Metadata, RunView, RunQuery, LogError, LogStatus } from '@memberjunction/core';
|
|
@@ -35,16 +36,93 @@ const CONSOLIDATION_CONFIG = {
|
|
|
35
36
|
* - 'daily': Run once per day
|
|
36
37
|
* - number: Run every N executions (e.g., 4 = every 4th run)
|
|
37
38
|
*/
|
|
38
|
-
frequency: '
|
|
39
|
+
frequency: 'daily',
|
|
39
40
|
minClusterSize: 3, // Minimum notes in a cluster to trigger consolidation
|
|
40
|
-
|
|
41
|
+
maxClusterSize: 7, // Maximum cluster size (Miller's 7±2) — split larger clusters
|
|
42
|
+
similarityThreshold: 0.60, // Semantic similarity threshold — 60% captures topically related notes
|
|
43
|
+
maxConsolidationCount: 3, // Cap re-summarization depth to prevent drift (research: 25.5% semantic loss after 5 iterations)
|
|
44
|
+
noteCountTrigger: 100 // Event-driven trigger: consolidate when an agent accumulates this many new notes since last run
|
|
41
45
|
};
|
|
46
|
+
/**
|
|
47
|
+
* Configuration for contradiction detection.
|
|
48
|
+
* Finds semantically similar note pairs and evaluates whether they contain conflicting facts.
|
|
49
|
+
*/
|
|
50
|
+
const CONTRADICTION_CONFIG = {
|
|
51
|
+
similarityThreshold: 0.70, // Higher than consolidation — targets same-topic pairs
|
|
52
|
+
maxPairsPerRun: 50, // Limit LLM calls per cycle
|
|
53
|
+
highImportanceThreshold: 7.0 // Notes above this get importance-weighted protection (both preserved with flag)
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Configuration for composite importance scoring.
|
|
57
|
+
* 7-signal formula replacing raw AccessCount for authority, retention, and consolidation decisions.
|
|
58
|
+
* Based on Park et al.'s Generative Agents (2023) and neuroscience research on memory importance.
|
|
59
|
+
*/
|
|
60
|
+
const IMPORTANCE_CONFIG = {
|
|
61
|
+
weights: {
|
|
62
|
+
recencyDecay: 0.20, // Exponential decay: 0.995^hours
|
|
63
|
+
llmImportance: 0.25, // Confidence from extraction prompt
|
|
64
|
+
relevance: 0.15, // Cosine similarity to recent conversation inputs
|
|
65
|
+
uniqueness: 0.15, // Min distance to nearest neighbor (rarity signal)
|
|
66
|
+
correctionBoost: 0.10, // Notes that corrected/superseded existing notes
|
|
67
|
+
goalAlignment: 0.10, // Alignment with agent goals (Phase 2 enrichment, 0.5 default)
|
|
68
|
+
userMark: 0.05 // Manual creation flag (IsAutoGenerated=false)
|
|
69
|
+
},
|
|
70
|
+
correctionBonusPoints: 1.5, // Flat additive bonus for correction-provenance notes
|
|
71
|
+
autoPromoteThreshold: 8.0, // Auto-promote Standard notes to Protected above this score
|
|
72
|
+
};
|
|
73
|
+
/**
|
|
74
|
+
* Configuration for Ebbinghaus-inspired decay function.
|
|
75
|
+
* Replaces fixed 90/180-day retention windows with continuous importance-weighted decay.
|
|
76
|
+
* Formula: strength = normalizedImportance * e^(-lambdaEff * daysSinceAccess) * (1 + accessCount * accessBoostFactor)
|
|
77
|
+
* where lambdaEff = baseLambda * (1 - normalizedImportance * importanceDampening)
|
|
78
|
+
*/
|
|
79
|
+
const DECAY_CONFIG = {
|
|
80
|
+
baseLambda: 0.16, // Base decay rate (Ebbinghaus-derived)
|
|
81
|
+
importanceDampening: 0.8, // How much high importance slows decay (0-1)
|
|
82
|
+
accessBoostFactor: 0.2, // Each access adds 20% to retention strength
|
|
83
|
+
accessRetentionCap: 1.5, // Hard cap on the access-retention multiplier to prevent frequently-accessed notes from becoming immortal
|
|
84
|
+
protectedInactivityDays: 365, // Protected tier extended retention before archival
|
|
85
|
+
ephemeralDecayMultiplier: 2.0, // Ephemeral tier decays 2x faster
|
|
86
|
+
archivalFloor: 0.5, // Score below this after proportional decay → archive (multiple cycles to reach)
|
|
87
|
+
};
|
|
88
|
+
/**
|
|
89
|
+
* Configuration for stale reference pruning.
|
|
90
|
+
* Caps how many orphaned notes the pruning phase will archive per cycle to keep
|
|
91
|
+
* Memory Manager runtime bounded even on large corpora.
|
|
92
|
+
*/
|
|
93
|
+
const STALE_PRUNING_CONFIG = {
|
|
94
|
+
maxNotesPerRun: 200,
|
|
95
|
+
};
|
|
96
|
+
/**
|
|
97
|
+
* Minimum number of scored notes required to compute a 95th-percentile uniqueness
|
|
98
|
+
* threshold. Below this the percentile is meaningless (a 5-note cohort has a p95
|
|
99
|
+
* at index 4 which is just "the max"), so outlier auto-protection is disabled.
|
|
100
|
+
*/
|
|
101
|
+
const OUTLIER_PROMOTION_MIN_COHORT = 20;
|
|
102
|
+
/**
|
|
103
|
+
* Valid Type values for MJAIAgentNote records. Must match the typed union in the
|
|
104
|
+
* generated entity class (MJCoreEntities/src/generated/entity_subclasses.ts).
|
|
105
|
+
* Used to validate LLM-returned values before assigning to entity properties.
|
|
106
|
+
*/
|
|
107
|
+
const VALID_NOTE_TYPES = ['Constraint', 'Context', 'Example', 'Issue', 'Preference'];
|
|
108
|
+
function isValidNoteType(value) {
|
|
109
|
+
return typeof value === 'string' && VALID_NOTE_TYPES.includes(value);
|
|
110
|
+
}
|
|
111
|
+
/** Reusable parameter object for deterministic (temperature=0) LLM prompt calls. */
|
|
112
|
+
const DETERMINISTIC_PROMPT_PARAMS = { temperature: 0 };
|
|
113
|
+
/**
|
|
114
|
+
* Hard cap on `resolveOriginalSources()` recursion depth.
|
|
115
|
+
* The `visited` set already prevents cycles, but a depth limit protects against
|
|
116
|
+
* runaway recursion if `DerivedFromNoteIDs` chains are corrupted in a way that
|
|
117
|
+
* passes the visited check (e.g., extremely deep legitimate chains from a bug).
|
|
118
|
+
*/
|
|
119
|
+
const MAX_SOURCE_RESOLUTION_DEPTH = 10;
|
|
42
120
|
/**
|
|
43
121
|
* Memory Manager Agent - automatically extracts notes and examples from conversations.
|
|
44
122
|
* This agent runs on a schedule (every 15 minutes) and analyzes high-quality conversations
|
|
45
123
|
* and agent runs to extract learnings and example interactions.
|
|
46
124
|
*/
|
|
47
|
-
let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
125
|
+
let MemoryManagerAgent = MemoryManagerAgent_1 = class MemoryManagerAgent extends BaseAgent {
|
|
48
126
|
constructor() {
|
|
49
127
|
super(...arguments);
|
|
50
128
|
/** Verbose logging flag from params.verbose */
|
|
@@ -56,6 +134,12 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
56
134
|
/** Context user for step operations */
|
|
57
135
|
this._contextUser = null;
|
|
58
136
|
}
|
|
137
|
+
/** Check if a string is a valid UUID format. Filters out LLM-generated placeholders like "user-uuid-here". */
|
|
138
|
+
static isValidUUID(id) {
|
|
139
|
+
if (!id)
|
|
140
|
+
return false;
|
|
141
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);
|
|
142
|
+
}
|
|
59
143
|
/**
|
|
60
144
|
* Create an agent run step record for observability.
|
|
61
145
|
* Returns null if agentRunID is not set (defensive check).
|
|
@@ -312,33 +396,67 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
312
396
|
return this.executeNoteExtraction(promptData, existingNotes, contextUser);
|
|
313
397
|
}
|
|
314
398
|
/**
|
|
315
|
-
* Common extraction logic for note extraction.
|
|
316
|
-
*
|
|
399
|
+
* Common extraction logic for note extraction. Thin orchestrator that coordinates:
|
|
400
|
+
* (1) running the extraction prompt,
|
|
401
|
+
* (2) normalizing merge-field shape from the LLM,
|
|
402
|
+
* (3) applying confidence/length filters,
|
|
403
|
+
* (4) enforcing per-message/per-conversation sparsity caps,
|
|
404
|
+
* (5) collapsing duplicate merge candidates,
|
|
405
|
+
* (6) deduplicating against existing notes,
|
|
406
|
+
* (7) enriching with conversation context (userId, agentRunId).
|
|
407
|
+
*
|
|
408
|
+
* Each step is its own private helper so the pipeline can be read top-to-bottom.
|
|
317
409
|
*/
|
|
318
410
|
async executeNoteExtraction(promptData, existingNotes, contextUser) {
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
if (!prompt) {
|
|
323
|
-
LogError('Memory Manager note extraction prompt not found');
|
|
324
|
-
if (this._verbose) {
|
|
325
|
-
LogStatus(`Memory Manager: Available prompts in MJ: System category: ${AIEngine.Instance.Prompts.filter(p => p.Category === 'MJ: System').map(p => p.Name).join(', ')}`);
|
|
326
|
-
}
|
|
411
|
+
// Step 1: run the extraction prompt and parse out the raw notes
|
|
412
|
+
const rawNotes = await this.runNoteExtractionPrompt(promptData, existingNotes.length, contextUser);
|
|
413
|
+
if (!rawNotes)
|
|
327
414
|
return [];
|
|
328
|
-
|
|
415
|
+
// Step 2: normalize the LLM's legacy singular mergeWithExistingId → plural mergeWithExistingIds
|
|
416
|
+
this.normalizeMergeCandidates(rawNotes);
|
|
329
417
|
if (this._verbose) {
|
|
330
|
-
LogStatus(`Memory Manager:
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
const firstThread = conversationThreads[0];
|
|
334
|
-
LogStatus(`Memory Manager: Sample thread (conv ${firstThread.conversationId}): ${firstThread.messages.map(m => `[${m.role}] ${m.message?.substring(0, 80)}...`).join(' | ')}`);
|
|
418
|
+
LogStatus(`Memory Manager: LLM returned ${rawNotes.length} raw notes before filtering`);
|
|
419
|
+
for (const note of rawNotes) {
|
|
420
|
+
LogStatus(`Memory Manager: Raw note: [${note.type}] "${note.content}" (confidence: ${note.confidence})${note.mergeWithExistingIds?.length ? ` mergeWith: ${note.mergeWithExistingIds.join(', ')}` : ''}`);
|
|
335
421
|
}
|
|
336
422
|
}
|
|
337
|
-
// Step 3:
|
|
338
|
-
const
|
|
423
|
+
// Step 3: filter by confidence and content length
|
|
424
|
+
const candidateNotes = this.filterByConfidenceAndLength(rawNotes);
|
|
425
|
+
if (candidateNotes.length === 0) {
|
|
426
|
+
if (this._verbose)
|
|
427
|
+
LogStatus('Memory Manager: No candidates passed confidence/length thresholds');
|
|
428
|
+
return [];
|
|
429
|
+
}
|
|
430
|
+
// Step 4: apply per-message and per-conversation sparsity caps
|
|
431
|
+
const sparseCandidates = this.applySparsityLimits(candidateNotes);
|
|
432
|
+
// Step 5: collapse merge candidates with identical content
|
|
433
|
+
const collapsedCandidates = this.collapseMergeCandidates(sparseCandidates);
|
|
434
|
+
if (this._verbose && collapsedCandidates.length < sparseCandidates.length) {
|
|
435
|
+
LogStatus(`Memory Manager: Collapsed ${sparseCandidates.length} candidates to ${collapsedCandidates.length} after merging duplicate merge targets`);
|
|
436
|
+
}
|
|
437
|
+
// Step 6: deduplicate against existing notes (creates and finalizes Step 4 run step)
|
|
438
|
+
const approvedNotes = await this.deduplicateCandidates(collapsedCandidates, existingNotes, contextUser);
|
|
439
|
+
if (this._verbose)
|
|
440
|
+
LogStatus(`Memory Manager: Final approved notes: ${approvedNotes.length}`);
|
|
441
|
+
// Step 7: enrich with conversation context (userId, agentRunId)
|
|
442
|
+
this.enrichWithConversationContext(approvedNotes, promptData.conversationThreads);
|
|
443
|
+
return approvedNotes;
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Execute the extraction prompt, wrap it in an observability run step, and
|
|
447
|
+
* parse the raw result into a typed array. Returns null on any failure so the
|
|
448
|
+
* caller short-circuits cleanly.
|
|
449
|
+
*/
|
|
450
|
+
async runNoteExtractionPrompt(promptData, existingNoteCount, contextUser) {
|
|
451
|
+
const prompt = this.findNoteExtractionPrompt();
|
|
452
|
+
if (!prompt)
|
|
453
|
+
return null;
|
|
454
|
+
const conversationThreads = promptData.conversationThreads;
|
|
455
|
+
this.logExtractionPromptContext(prompt, conversationThreads);
|
|
456
|
+
const step = await this.CreateRunStep('Prompt', 'Extract Notes from Conversations', {
|
|
339
457
|
conversationCount: conversationThreads.length,
|
|
340
458
|
messageCount: conversationThreads.reduce((sum, t) => sum + t.messages.length, 0),
|
|
341
|
-
existingNoteCount
|
|
459
|
+
existingNoteCount
|
|
342
460
|
}, prompt.ID);
|
|
343
461
|
const runner = new AIPromptRunner();
|
|
344
462
|
const params = new AIPromptParams();
|
|
@@ -346,93 +464,111 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
346
464
|
params.data = promptData;
|
|
347
465
|
params.contextUser = contextUser;
|
|
348
466
|
params.attemptJSONRepair = true;
|
|
467
|
+
params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
|
|
349
468
|
const result = await runner.ExecutePrompt(params);
|
|
350
|
-
|
|
351
|
-
await this.FinalizeRunStep(step3, result.success, {
|
|
469
|
+
await this.FinalizeRunStep(step, result.success, {
|
|
352
470
|
success: result.success,
|
|
353
471
|
rawNoteCount: result.result && typeof result.result !== 'string' ? (result.result.notes?.length || 0) : 0
|
|
354
472
|
}, result.promptRun?.ID, result.errorMessage || undefined);
|
|
473
|
+
return this.parseNoteExtractionResult(result);
|
|
474
|
+
}
|
|
475
|
+
/** Look up the extraction prompt, logging available prompts on miss. */
|
|
476
|
+
findNoteExtractionPrompt() {
|
|
477
|
+
const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Extract Notes' && p.Category === 'MJ: System');
|
|
478
|
+
if (!prompt) {
|
|
479
|
+
LogError('Memory Manager note extraction prompt not found');
|
|
480
|
+
if (this._verbose) {
|
|
481
|
+
LogStatus(`Memory Manager: Available prompts in MJ: System category: ${AIEngine.Instance.Prompts.filter(p => p.Category === 'MJ: System').map(p => p.Name).join(', ')}`);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return prompt;
|
|
485
|
+
}
|
|
486
|
+
/** Verbose-only logging block for extraction prompt context. */
|
|
487
|
+
logExtractionPromptContext(prompt, conversationThreads) {
|
|
488
|
+
if (!this._verbose)
|
|
489
|
+
return;
|
|
490
|
+
LogStatus(`Memory Manager: Found extraction prompt "${prompt.Name}" (ID: ${prompt.ID})`);
|
|
491
|
+
LogStatus(`Memory Manager: Sending ${conversationThreads.length} conversation threads with ${conversationThreads.reduce((sum, t) => sum + t.messages.length, 0)} total messages for note extraction`);
|
|
492
|
+
if (conversationThreads.length > 0) {
|
|
493
|
+
const firstThread = conversationThreads[0];
|
|
494
|
+
LogStatus(`Memory Manager: Sample thread (conv ${firstThread.conversationId}): ${firstThread.messages.map(m => `[${m.role}] ${m.message?.substring(0, 80)}...`).join(' | ')}`);
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* Parse an extraction prompt result into an ExtractedNote array, tolerating JSON-fenced
|
|
499
|
+
* strings from models like Gemini. Returns null on failure.
|
|
500
|
+
*/
|
|
501
|
+
parseNoteExtractionResult(result) {
|
|
355
502
|
if (this._verbose) {
|
|
356
503
|
LogStatus(`Memory Manager: Extraction prompt result - success: ${result.success}, hasResult: ${!!result.result}`);
|
|
357
|
-
if (result.errorMessage)
|
|
504
|
+
if (result.errorMessage)
|
|
358
505
|
LogStatus(`Memory Manager: Extraction error: ${result.errorMessage}`);
|
|
359
|
-
|
|
360
|
-
if (result.result) {
|
|
506
|
+
if (result.result)
|
|
361
507
|
LogStatus(`Memory Manager: Raw extraction result: ${JSON.stringify(result.result)}`);
|
|
362
|
-
}
|
|
363
508
|
}
|
|
364
509
|
if (!result.success || !result.result) {
|
|
365
510
|
LogError('Failed to extract notes:', result.errorMessage);
|
|
366
|
-
return
|
|
511
|
+
return null;
|
|
367
512
|
}
|
|
368
|
-
// Parse result if it's a string (AI sometimes returns JSON as string)
|
|
369
|
-
// Some models (e.g., Gemini) wrap JSON in ```json fences
|
|
370
|
-
let parsedResult;
|
|
371
513
|
if (typeof result.result === 'string') {
|
|
372
|
-
const
|
|
373
|
-
if (!
|
|
514
|
+
const reparsed = CleanAndParseJSON(result.result, true);
|
|
515
|
+
if (!reparsed) {
|
|
374
516
|
LogError('Failed to parse extraction result as JSON');
|
|
375
|
-
return
|
|
517
|
+
return null;
|
|
376
518
|
}
|
|
377
|
-
parsedResult = parsed;
|
|
378
519
|
if (this._verbose) {
|
|
379
|
-
LogStatus(`Memory Manager: Parsed string result into object with ${
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
else {
|
|
383
|
-
parsedResult = result.result;
|
|
384
|
-
}
|
|
385
|
-
// Normalize singular mergeWithExistingId → plural mergeWithExistingIds
|
|
386
|
-
// The LLM still outputs the singular field; we convert it here.
|
|
387
|
-
// Notes are plain objects from JSON.parse, so the runtime cast is safe.
|
|
388
|
-
if (parsedResult.notes) {
|
|
389
|
-
for (const note of parsedResult.notes) {
|
|
390
|
-
if (note.mergeWithExistingId && !note.mergeWithExistingIds) {
|
|
391
|
-
note.mergeWithExistingIds = [note.mergeWithExistingId];
|
|
392
|
-
}
|
|
520
|
+
LogStatus(`Memory Manager: Parsed string result into object with ${reparsed.notes?.length || 0} notes`);
|
|
393
521
|
}
|
|
522
|
+
return reparsed.notes || [];
|
|
394
523
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
524
|
+
return result.result.notes || [];
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Normalize the legacy singular `mergeWithExistingId` field into the plural
|
|
528
|
+
* `mergeWithExistingIds` array. The extraction prompt still emits the singular
|
|
529
|
+
* field in some paths; downstream code only reads the plural form.
|
|
530
|
+
*/
|
|
531
|
+
normalizeMergeCandidates(notes) {
|
|
532
|
+
for (const note of notes) {
|
|
533
|
+
if (note.mergeWithExistingId && !note.mergeWithExistingIds) {
|
|
534
|
+
note.mergeWithExistingIds = [note.mergeWithExistingId];
|
|
400
535
|
}
|
|
401
536
|
}
|
|
402
|
-
|
|
403
|
-
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* Filter raw extracted notes against the confidence and content-length thresholds
|
|
540
|
+
* from EXTRACTION_CONFIG. Notes below either threshold are silently dropped.
|
|
541
|
+
*/
|
|
542
|
+
filterByConfidenceAndLength(notes) {
|
|
543
|
+
const filtered = notes.filter(n => n.confidence >= EXTRACTION_CONFIG.minConfidenceThreshold &&
|
|
404
544
|
n.content && n.content.length >= EXTRACTION_CONFIG.minContentLength);
|
|
405
|
-
if (
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
}
|
|
409
|
-
return [];
|
|
410
|
-
}
|
|
411
|
-
if (this._verbose) {
|
|
412
|
-
LogStatus(`Memory Manager: ${candidateNotes.length} candidates passed confidence threshold (>=${EXTRACTION_CONFIG.minConfidenceThreshold})`);
|
|
413
|
-
// Log each extracted note for debugging
|
|
414
|
-
for (const note of candidateNotes) {
|
|
545
|
+
if (this._verbose && filtered.length > 0) {
|
|
546
|
+
LogStatus(`Memory Manager: ${filtered.length} candidates passed confidence threshold (>=${EXTRACTION_CONFIG.minConfidenceThreshold})`);
|
|
547
|
+
for (const note of filtered) {
|
|
415
548
|
LogStatus(`Memory Manager: Extracted note: [${note.type}] "${note.content}" (confidence: ${note.confidence})`);
|
|
416
549
|
}
|
|
417
550
|
}
|
|
418
|
-
|
|
551
|
+
return filtered;
|
|
552
|
+
}
|
|
553
|
+
/**
|
|
554
|
+
* Enforce per-message and per-conversation note caps from EXTRACTION_CONFIG.
|
|
555
|
+
* Drops any candidates that would exceed either cap.
|
|
556
|
+
*/
|
|
557
|
+
applySparsityLimits(candidates) {
|
|
419
558
|
const notesByMessage = new Map();
|
|
420
559
|
const noteCountByConversation = new Map();
|
|
421
|
-
for (const note of
|
|
560
|
+
for (const note of candidates) {
|
|
422
561
|
const msgId = note.sourceConversationDetailId || 'unknown';
|
|
423
562
|
const convId = note.sourceConversationId || 'unknown';
|
|
424
563
|
const existingForMsg = notesByMessage.get(msgId) || [];
|
|
425
564
|
const convCount = noteCountByConversation.get(convId) || 0;
|
|
426
|
-
// Check both per-message and per-conversation limits
|
|
427
565
|
if (existingForMsg.length >= EXTRACTION_CONFIG.maxNotesPerMessage) {
|
|
428
|
-
if (this._verbose)
|
|
566
|
+
if (this._verbose)
|
|
429
567
|
LogStatus(`Memory Manager: Skipping note (max ${EXTRACTION_CONFIG.maxNotesPerMessage} per message reached)`);
|
|
430
|
-
}
|
|
431
568
|
}
|
|
432
569
|
else if (convCount >= EXTRACTION_CONFIG.maxNotesPerConversation) {
|
|
433
|
-
if (this._verbose)
|
|
570
|
+
if (this._verbose)
|
|
434
571
|
LogStatus(`Memory Manager: Skipping note (max ${EXTRACTION_CONFIG.maxNotesPerConversation} per conversation reached)`);
|
|
435
|
-
}
|
|
436
572
|
}
|
|
437
573
|
else {
|
|
438
574
|
existingForMsg.push(note);
|
|
@@ -440,110 +576,123 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
440
576
|
noteCountByConversation.set(convId, convCount + 1);
|
|
441
577
|
}
|
|
442
578
|
}
|
|
443
|
-
const
|
|
444
|
-
if (this._verbose)
|
|
445
|
-
LogStatus(`Memory Manager: ${
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
579
|
+
const sparse = Array.from(notesByMessage.values()).flat();
|
|
580
|
+
if (this._verbose)
|
|
581
|
+
LogStatus(`Memory Manager: ${sparse.length} candidates after sparsity filter`);
|
|
582
|
+
return sparse;
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Deduplicate candidate notes against existing notes in the database using:
|
|
586
|
+
* 1. Exact text match (fast reject),
|
|
587
|
+
* 2. Semantic similarity lookup (top-N with ≥85% threshold),
|
|
588
|
+
* 3. Dedup prompt to resolve genuinely-similar candidates.
|
|
589
|
+
*
|
|
590
|
+
* Candidates flagged as merge/replacement by the extraction LLM are auto-approved
|
|
591
|
+
* (dedup would reject them as "too similar" to the note they are intentionally
|
|
592
|
+
* replacing). Creates and finalizes its own observability run step.
|
|
593
|
+
*/
|
|
594
|
+
async deduplicateCandidates(candidates, existingNotes, contextUser) {
|
|
595
|
+
const step = await this.CreateRunStep('Decision', 'Deduplicate Note Candidates', {
|
|
596
|
+
candidateCount: candidates.length,
|
|
455
597
|
existingNoteCount: existingNotes.length
|
|
456
598
|
});
|
|
457
|
-
const
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
// Find deduplication prompt
|
|
599
|
+
const approved = [];
|
|
600
|
+
const stats = { rejectedCount: 0, llmCallCount: 0 };
|
|
601
|
+
const runner = new AIPromptRunner();
|
|
461
602
|
const dedupePrompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Deduplicate Note' && p.Category === 'MJ: System');
|
|
462
|
-
for (const candidate of
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
if (this._verbose) {
|
|
603
|
+
for (const candidate of candidates) {
|
|
604
|
+
if (approved.length >= EXTRACTION_CONFIG.maxNotesPerRun) {
|
|
605
|
+
if (this._verbose)
|
|
466
606
|
LogStatus(`Memory Manager: Stopping - max notes per run (${EXTRACTION_CONFIG.maxNotesPerRun}) reached`);
|
|
467
|
-
}
|
|
468
607
|
break;
|
|
469
608
|
}
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
if (candidate.mergeWithExistingIds?.length) {
|
|
474
|
-
approvedNotes.push(candidate);
|
|
475
|
-
if (this._verbose) {
|
|
476
|
-
LogStatus(`Memory Manager: Auto-approved merge note targeting ${candidate.mergeWithExistingIds.join(', ')}: "${candidate.content.substring(0, 50)}..."`);
|
|
477
|
-
}
|
|
478
|
-
continue;
|
|
479
|
-
}
|
|
480
|
-
// Step 1: Check for exact content match FIRST (case-insensitive, trimmed)
|
|
481
|
-
const normalizedContent = candidate.content.toLowerCase().trim();
|
|
482
|
-
const exactMatch = existingNotes.find(n => n.Note && n.Note.toLowerCase().trim() === normalizedContent);
|
|
483
|
-
if (exactMatch) {
|
|
484
|
-
dedupeRejectedCount++;
|
|
485
|
-
if (this._verbose) {
|
|
486
|
-
LogStatus(`Memory Manager: Skipping exact duplicate: "${candidate.content.substring(0, 50)}..."`);
|
|
487
|
-
}
|
|
488
|
-
continue;
|
|
489
|
-
}
|
|
490
|
-
// Step 2: Find similar existing notes using semantic search with tighter threshold.
|
|
491
|
-
// Cross-user dedup for org/global notes works naturally because those notes have
|
|
492
|
-
// UserID=null, so FindSimilarAgentNotes won't filter them out for any user.
|
|
493
|
-
const similarNotes = await AIEngine.Instance.FindSimilarAgentNotes(candidate.content, candidate.agentId, candidate.userId, candidate.companyId, 10, // Top 10 similar (increased from 5)
|
|
494
|
-
0.85 // 85% similarity threshold (increased from 70%)
|
|
495
|
-
);
|
|
496
|
-
// Step 3: If deduplication prompt exists and similar notes found, ask LLM
|
|
497
|
-
if (dedupePrompt && similarNotes.length > 0) {
|
|
498
|
-
const dedupeParams = new AIPromptParams();
|
|
499
|
-
dedupeParams.prompt = dedupePrompt;
|
|
500
|
-
dedupeParams.data = {
|
|
501
|
-
candidateNote: candidate,
|
|
502
|
-
similarNotes: similarNotes.map(s => ({
|
|
503
|
-
type: s.note.Type,
|
|
504
|
-
content: s.note.Note,
|
|
505
|
-
agentId: s.note.AgentID,
|
|
506
|
-
userId: s.note.UserID,
|
|
507
|
-
companyId: s.note.CompanyID,
|
|
508
|
-
similarity: s.similarity
|
|
509
|
-
}))
|
|
510
|
-
};
|
|
511
|
-
dedupeParams.contextUser = contextUser;
|
|
512
|
-
dedupeParams.attemptJSONRepair = true;
|
|
513
|
-
const dedupeResult = await runner.ExecutePrompt(dedupeParams);
|
|
514
|
-
dedupeLlmCallCount++;
|
|
515
|
-
if (dedupeResult.success && dedupeResult.result && dedupeResult.result.shouldAdd) {
|
|
516
|
-
approvedNotes.push(candidate);
|
|
517
|
-
if (this._verbose) {
|
|
518
|
-
LogStatus(`Memory Manager: Approved note - ${dedupeResult.result.reason}`);
|
|
519
|
-
}
|
|
520
|
-
}
|
|
521
|
-
else {
|
|
522
|
-
dedupeRejectedCount++;
|
|
523
|
-
if (this._verbose) {
|
|
524
|
-
LogStatus(`Memory Manager: Skipped duplicate note - ${dedupeResult.result?.reason || 'too similar to existing notes'}`);
|
|
525
|
-
}
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
else {
|
|
529
|
-
// No similar notes found or no deduplication prompt, add the note
|
|
530
|
-
approvedNotes.push(candidate);
|
|
531
|
-
if (this._verbose) {
|
|
532
|
-
LogStatus(`Memory Manager: Approved note (no similar notes found): "${candidate.content.substring(0, 50)}..."`);
|
|
533
|
-
}
|
|
609
|
+
const decision = await this.evaluateCandidateForApproval(candidate, existingNotes, dedupePrompt, runner, contextUser, stats);
|
|
610
|
+
if (decision) {
|
|
611
|
+
approved.push(candidate);
|
|
534
612
|
}
|
|
535
613
|
}
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
llmCallCount: dedupeLlmCallCount
|
|
614
|
+
await this.FinalizeRunStep(step, true, {
|
|
615
|
+
approvedCount: approved.length,
|
|
616
|
+
rejectedCount: stats.rejectedCount,
|
|
617
|
+
llmCallCount: stats.llmCallCount
|
|
541
618
|
});
|
|
542
|
-
|
|
543
|
-
|
|
619
|
+
return approved;
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* Evaluate a single candidate: merge markers auto-approve, exact-text duplicates are
|
|
623
|
+
* rejected, otherwise run semantic similarity + LLM dedup. Returns true if the candidate
|
|
624
|
+
* should be kept. Mutates `stats` for rejection + LLM call counts as a side effect.
|
|
625
|
+
*/
|
|
626
|
+
async evaluateCandidateForApproval(candidate, existingNotes, dedupePrompt, runner, contextUser, stats) {
|
|
627
|
+
// Merge/replacement candidates are auto-approved — the extraction LLM already
|
|
628
|
+
// determined they supersede an existing note. Running dedup on them would reject
|
|
629
|
+
// them as "too similar" to the very note they are replacing.
|
|
630
|
+
if (candidate.mergeWithExistingIds?.length) {
|
|
631
|
+
if (this._verbose) {
|
|
632
|
+
LogStatus(`Memory Manager: Auto-approved merge note targeting ${candidate.mergeWithExistingIds.join(', ')}: "${candidate.content.substring(0, 50)}..."`);
|
|
633
|
+
}
|
|
634
|
+
return true;
|
|
635
|
+
}
|
|
636
|
+
// Fast path: exact text match (case-insensitive, trimmed)
|
|
637
|
+
const normalizedContent = candidate.content.toLowerCase().trim();
|
|
638
|
+
if (existingNotes.some(n => n.Note && n.Note.toLowerCase().trim() === normalizedContent)) {
|
|
639
|
+
stats.rejectedCount++;
|
|
640
|
+
if (this._verbose)
|
|
641
|
+
LogStatus(`Memory Manager: Skipping exact duplicate: "${candidate.content.substring(0, 50)}..."`);
|
|
642
|
+
return false;
|
|
643
|
+
}
|
|
644
|
+
// Semantic similarity lookup. Cross-user dedup for org/global notes works naturally
|
|
645
|
+
// because those notes have UserID=null — FindSimilarAgentNotes won't filter them out.
|
|
646
|
+
const similarNotes = await AIEngine.Instance.FindSimilarAgentNotes(candidate.content, candidate.agentId, candidate.userId, candidate.companyId, 10, 0.85);
|
|
647
|
+
if (!dedupePrompt || similarNotes.length === 0) {
|
|
648
|
+
if (this._verbose)
|
|
649
|
+
LogStatus(`Memory Manager: Approved note (no similar notes found): "${candidate.content.substring(0, 50)}..."`);
|
|
650
|
+
return true;
|
|
651
|
+
}
|
|
652
|
+
const decision = await this.runDedupePromptForCandidate(runner, dedupePrompt, candidate, similarNotes, contextUser);
|
|
653
|
+
stats.llmCallCount++;
|
|
654
|
+
if (decision.shouldAdd) {
|
|
655
|
+
if (this._verbose)
|
|
656
|
+
LogStatus(`Memory Manager: Approved note - ${decision.reason}`);
|
|
657
|
+
return true;
|
|
658
|
+
}
|
|
659
|
+
stats.rejectedCount++;
|
|
660
|
+
if (this._verbose)
|
|
661
|
+
LogStatus(`Memory Manager: Skipped duplicate note - ${decision.reason}`);
|
|
662
|
+
return false;
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* Invoke the dedupe LLM prompt for a single candidate note against its similar
|
|
666
|
+
* neighbors. Returns the decision plus reason string — callers handle logging.
|
|
667
|
+
*/
|
|
668
|
+
async runDedupePromptForCandidate(runner, dedupePrompt, candidate, similarNotes, contextUser) {
|
|
669
|
+
const params = new AIPromptParams();
|
|
670
|
+
params.prompt = dedupePrompt;
|
|
671
|
+
params.data = {
|
|
672
|
+
candidateNote: candidate,
|
|
673
|
+
similarNotes: similarNotes.map(s => ({
|
|
674
|
+
type: s.note.Type,
|
|
675
|
+
content: s.note.Note,
|
|
676
|
+
agentId: s.note.AgentID,
|
|
677
|
+
userId: s.note.UserID,
|
|
678
|
+
companyId: s.note.CompanyID,
|
|
679
|
+
similarity: s.similarity
|
|
680
|
+
}))
|
|
681
|
+
};
|
|
682
|
+
params.contextUser = contextUser;
|
|
683
|
+
params.attemptJSONRepair = true;
|
|
684
|
+
params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
|
|
685
|
+
const result = await runner.ExecutePrompt(params);
|
|
686
|
+
if (result.success && result.result) {
|
|
687
|
+
return { shouldAdd: result.result.shouldAdd, reason: result.result.reason || '' };
|
|
544
688
|
}
|
|
545
|
-
|
|
546
|
-
|
|
689
|
+
return { shouldAdd: false, reason: result.result?.reason || 'too similar to existing notes' };
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Populate userId and sourceAgentRunId on approved notes from the source
|
|
693
|
+
* conversation's context. Only fills fields the LLM didn't already set.
|
|
694
|
+
*/
|
|
695
|
+
enrichWithConversationContext(notes, conversationThreads) {
|
|
547
696
|
const conversationContext = new Map();
|
|
548
697
|
for (const thread of conversationThreads) {
|
|
549
698
|
conversationContext.set(thread.conversationId, {
|
|
@@ -551,23 +700,17 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
551
700
|
agentRunId: thread.agentRunId || null
|
|
552
701
|
});
|
|
553
702
|
}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
if (!note.sourceAgentRunId && ctx.agentRunId) {
|
|
565
|
-
note.sourceAgentRunId = ctx.agentRunId;
|
|
566
|
-
}
|
|
567
|
-
}
|
|
568
|
-
}
|
|
703
|
+
for (const note of notes) {
|
|
704
|
+
if (!note.sourceConversationId)
|
|
705
|
+
continue;
|
|
706
|
+
const ctx = conversationContext.get(note.sourceConversationId);
|
|
707
|
+
if (!ctx)
|
|
708
|
+
continue;
|
|
709
|
+
if (!note.userId && ctx.userId)
|
|
710
|
+
note.userId = ctx.userId;
|
|
711
|
+
if (!note.sourceAgentRunId && ctx.agentRunId)
|
|
712
|
+
note.sourceAgentRunId = ctx.agentRunId;
|
|
569
713
|
}
|
|
570
|
-
return approvedNotes;
|
|
571
714
|
}
|
|
572
715
|
/**
|
|
573
716
|
* Extract examples from conversation details with high ratings.
|
|
@@ -601,6 +744,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
601
744
|
extractParams.data = { qaPairs };
|
|
602
745
|
extractParams.contextUser = contextUser;
|
|
603
746
|
extractParams.attemptJSONRepair = true;
|
|
747
|
+
extractParams.additionalParameters = { temperature: 0 };
|
|
604
748
|
const extractResult = await runner.ExecutePrompt(extractParams);
|
|
605
749
|
if (!extractResult.success || !extractResult.result) {
|
|
606
750
|
await this.FinalizeRunStep(step5, false, {
|
|
@@ -662,6 +806,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
662
806
|
};
|
|
663
807
|
dedupeParams.contextUser = contextUser;
|
|
664
808
|
dedupeParams.attemptJSONRepair = true;
|
|
809
|
+
dedupeParams.additionalParameters = { temperature: 0 };
|
|
665
810
|
const dedupeResult = await runner.ExecutePrompt(dedupeParams);
|
|
666
811
|
exampleDedupeLlmCallCount++;
|
|
667
812
|
if (dedupeResult.success && dedupeResult.result && dedupeResult.result.shouldAdd) {
|
|
@@ -690,13 +835,6 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
690
835
|
});
|
|
691
836
|
return approvedExamples;
|
|
692
837
|
}
|
|
693
|
-
/**
|
|
694
|
-
* Check if a string is a valid UUID format.
|
|
695
|
-
*/
|
|
696
|
-
isValidUUID(str) {
|
|
697
|
-
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
698
|
-
return uuidRegex.test(str);
|
|
699
|
-
}
|
|
700
838
|
/**
|
|
701
839
|
* Create note records from extracted data.
|
|
702
840
|
* Inherits scope from source agent run and applies scopeLevel to determine scope specificity.
|
|
@@ -746,6 +884,23 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
746
884
|
for (const mergeTargetId of extracted.mergeWithExistingIds) {
|
|
747
885
|
const existingNote = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
748
886
|
if (await existingNote.Load(mergeTargetId)) {
|
|
887
|
+
// Honor protection tiers — Immutable and Protected notes must NEVER be
|
|
888
|
+
// auto-revoked even when the extraction LLM flags them as merge targets.
|
|
889
|
+
// The contradiction-detection phase enforces the same rule via tier-based
|
|
890
|
+
// filtering, but this earlier merge path bypassed it. Skip the revocation
|
|
891
|
+
// and just annotate the existing note so the conflict is observable.
|
|
892
|
+
const tier = existingNote.ProtectionTier || 'Standard';
|
|
893
|
+
if (tier === 'Immutable' || tier === 'Protected') {
|
|
894
|
+
const flagComment = `[Merge-target conflict: extraction LLM flagged this ${tier} note for merge from conversation ${extracted.sourceConversationId || 'unknown'} but tier prevents auto-revocation]`;
|
|
895
|
+
existingNote.Comments = `${existingNote.Comments || ''} ${flagComment}`.trim();
|
|
896
|
+
if (!await existingNote.Save()) {
|
|
897
|
+
LogError(`Memory Manager: Failed to annotate ${tier} merge target ${existingNote.ID}`);
|
|
898
|
+
}
|
|
899
|
+
if (this._verbose) {
|
|
900
|
+
LogStatus(`Memory Manager: Skipped revocation of ${tier} note ${existingNote.ID} flagged as merge target`);
|
|
901
|
+
}
|
|
902
|
+
continue;
|
|
903
|
+
}
|
|
749
904
|
existingNote.Status = 'Revoked';
|
|
750
905
|
existingNote.Comments = `Superseded: contradiction detected, replaced by new note from conversation ${extracted.sourceConversationId || 'unknown'}`;
|
|
751
906
|
if (await existingNote.Save()) {
|
|
@@ -765,28 +920,25 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
765
920
|
if (createNewNote) {
|
|
766
921
|
// Create new note
|
|
767
922
|
const note = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
768
|
-
// Only use valid UUIDs, filter out placeholders like "user-uuid-here"
|
|
769
|
-
const isValidUUID = (id) => {
|
|
770
|
-
if (!id)
|
|
771
|
-
return false;
|
|
772
|
-
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id);
|
|
773
|
-
};
|
|
774
923
|
// Determine AgentID - prefer extracted, then inherit from source run
|
|
775
|
-
let agentId = isValidUUID(extracted.agentId) ? extracted.agentId : null;
|
|
924
|
+
let agentId = MemoryManagerAgent_1.isValidUUID(extracted.agentId) ? extracted.agentId : null;
|
|
776
925
|
if (!agentId && sourceRun?.AgentID) {
|
|
777
926
|
agentId = sourceRun.AgentID;
|
|
778
927
|
}
|
|
779
928
|
note.AgentID = agentId;
|
|
780
|
-
note.CompanyID =
|
|
929
|
+
note.CompanyID = sourceRun?.CompanyID || null;
|
|
781
930
|
note.AgentNoteTypeID = aiNoteTypeId; // "AI" type for AI-generated notes
|
|
782
931
|
note.Type = extracted.type; // Category: Preference, Constraint, Context, Issue, Example
|
|
783
932
|
note.Note = extracted.content;
|
|
784
933
|
note.IsAutoGenerated = true;
|
|
785
934
|
note.Status = 'Active'; // Auto-approve high-confidence notes
|
|
786
935
|
note.AccessCount = 1; // Required field
|
|
936
|
+
// Consolidation tracking fields
|
|
937
|
+
note.ConsolidationCount = 0; // Raw extraction, never consolidated
|
|
938
|
+
note.ProtectionTier = note.IsAutoGenerated ? 'Standard' : 'Protected';
|
|
787
939
|
note.SourceConversationID = extracted.sourceConversationId || null;
|
|
788
940
|
// Only use if it's a valid UUID (LLM now sees message IDs in the prompt)
|
|
789
|
-
note.SourceConversationDetailID = isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
|
|
941
|
+
note.SourceConversationDetailID = MemoryManagerAgent_1.isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
|
|
790
942
|
note.SourceAIAgentRunID = extracted.sourceAgentRunId || null;
|
|
791
943
|
// Apply scope: Lean towards user-specific memories by default
|
|
792
944
|
const scopeLevel = extracted.scopeLevel || 'user';
|
|
@@ -808,9 +960,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
808
960
|
// UserID: Only set for user-scoped notes. Company/global notes belong to the
|
|
809
961
|
// company, not a specific user. This also enables cross-user dedup — FindSimilarAgentNotes
|
|
810
962
|
// filters by userId, so null UserID makes company notes visible to all users.
|
|
811
|
-
note.UserID = scopeLevel === 'user'
|
|
812
|
-
? (isValidUUID(extracted.userId) ? extracted.userId : null)
|
|
813
|
-
: null;
|
|
963
|
+
note.UserID = scopeLevel === 'user' ? (sourceRun?.UserID || null) : null;
|
|
814
964
|
const saveResult = await note.Save();
|
|
815
965
|
if (saveResult) {
|
|
816
966
|
created++;
|
|
@@ -861,6 +1011,175 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
861
1011
|
}
|
|
862
1012
|
return result;
|
|
863
1013
|
}
|
|
1014
|
+
/**
|
|
1015
|
+
* Compute composite importance scores for all active notes.
|
|
1016
|
+
* Uses a 7-signal weighted formula based on Park et al.'s Generative Agents (2023).
|
|
1017
|
+
* Signals not available for a given note use neutral defaults (0.5) with proportional weight redistribution.
|
|
1018
|
+
*
|
|
1019
|
+
* Auto-promotes Standard-tier notes to Protected via two paths:
|
|
1020
|
+
* 1. ImportanceScore >= IMPORTANCE_CONFIG.autoPromoteThreshold (default 8.0)
|
|
1021
|
+
* 2. Uniqueness at or above the 95th percentile of the cohort (outlier protection,
|
|
1022
|
+
* FR-094). The cohort must be at least OUTLIER_PROMOTION_MIN_COHORT notes for
|
|
1023
|
+
* the percentile to be meaningful.
|
|
1024
|
+
*
|
|
1025
|
+
* Also reports the score distribution (min/max/mean/median) so the observability
|
|
1026
|
+
* step can surface it in the agent run step output.
|
|
1027
|
+
*/
|
|
1028
|
+
async computeImportanceScores(contextUser) {
|
|
1029
|
+
const allNotes = AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active');
|
|
1030
|
+
let notesScored = 0;
|
|
1031
|
+
let tierPromotions = 0;
|
|
1032
|
+
const md = new Metadata();
|
|
1033
|
+
const scoresForDistribution = [];
|
|
1034
|
+
// Pre-pass: compute uniqueness per note once, derive 95th-percentile threshold.
|
|
1035
|
+
// Passing uniqueness into gatherImportanceSignals also eliminates a redundant
|
|
1036
|
+
// FindSimilarAgentNotes call per note.
|
|
1037
|
+
const uniquenessByNoteId = await this.precomputeUniqueness(allNotes);
|
|
1038
|
+
const p95Threshold = this.computeUniquenessP95(uniquenessByNoteId);
|
|
1039
|
+
for (const note of allNotes) {
|
|
1040
|
+
const uniquenessInfo = uniquenessByNoteId.get(note.ID);
|
|
1041
|
+
const signals = this.gatherImportanceSignals(note, uniquenessInfo);
|
|
1042
|
+
const importanceScore = this.computeWeightedImportanceScore(signals);
|
|
1043
|
+
const noteToUpdate = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
1044
|
+
if (await noteToUpdate.Load(note.ID)) {
|
|
1045
|
+
noteToUpdate.ImportanceScore = importanceScore;
|
|
1046
|
+
// Promotion path 1: high ImportanceScore
|
|
1047
|
+
const currentTier = noteToUpdate.ProtectionTier || 'Standard';
|
|
1048
|
+
let promotedByScore = false;
|
|
1049
|
+
if (currentTier === 'Standard' && importanceScore >= IMPORTANCE_CONFIG.autoPromoteThreshold) {
|
|
1050
|
+
noteToUpdate.ProtectionTier = 'Protected';
|
|
1051
|
+
noteToUpdate.Comments = `${noteToUpdate.Comments || ''} [Auto-promoted to Protected: ImportanceScore ${importanceScore} >= ${IMPORTANCE_CONFIG.autoPromoteThreshold}]`.trim();
|
|
1052
|
+
tierPromotions++;
|
|
1053
|
+
promotedByScore = true;
|
|
1054
|
+
}
|
|
1055
|
+
// Promotion path 2: uniqueness outlier (p95+). Only apply when the cohort
|
|
1056
|
+
// is large enough for the percentile to be meaningful, the note has a
|
|
1057
|
+
// computed uniqueness, and we didn't already promote it via the score path.
|
|
1058
|
+
if (!promotedByScore &&
|
|
1059
|
+
currentTier === 'Standard' &&
|
|
1060
|
+
p95Threshold !== null &&
|
|
1061
|
+
uniquenessInfo?.available &&
|
|
1062
|
+
uniquenessInfo.value >= p95Threshold) {
|
|
1063
|
+
noteToUpdate.ProtectionTier = 'Protected';
|
|
1064
|
+
noteToUpdate.Comments = `${noteToUpdate.Comments || ''} [Auto-protected: uniqueness ${uniquenessInfo.value.toFixed(3)} at/above 95th percentile ${p95Threshold.toFixed(3)}]`.trim();
|
|
1065
|
+
tierPromotions++;
|
|
1066
|
+
}
|
|
1067
|
+
if (await noteToUpdate.Save()) {
|
|
1068
|
+
notesScored++;
|
|
1069
|
+
scoresForDistribution.push(importanceScore);
|
|
1070
|
+
}
|
|
1071
|
+
else {
|
|
1072
|
+
LogError(`Memory Manager: Failed to update ImportanceScore for note ${note.ID}`);
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
return {
|
|
1077
|
+
notesScored,
|
|
1078
|
+
tierPromotions,
|
|
1079
|
+
scoreDistribution: this.summarizeScores(scoresForDistribution)
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
/**
|
|
1083
|
+
* Pre-compute uniqueness (1 - maxSimilarity to nearest neighbor) for each note in
|
|
1084
|
+
* one pass. Returns a map keyed by note ID. Notes without content get an
|
|
1085
|
+
* unavailable signal so downstream scoring falls back to the neutral default.
|
|
1086
|
+
*/
|
|
1087
|
+
async precomputeUniqueness(notes) {
|
|
1088
|
+
const map = new Map();
|
|
1089
|
+
for (const note of notes) {
|
|
1090
|
+
if (!note.Note) {
|
|
1091
|
+
map.set(note.ID, { value: 0.5, available: false });
|
|
1092
|
+
continue;
|
|
1093
|
+
}
|
|
1094
|
+
const similar = await AIEngine.Instance.FindSimilarAgentNotes(note.Note, note.AgentID, undefined, undefined, 1, 0.0);
|
|
1095
|
+
const uniqueness = similar.length > 0 ? 1.0 - similar[0].similarity : 1.0;
|
|
1096
|
+
map.set(note.ID, { value: uniqueness, available: true });
|
|
1097
|
+
}
|
|
1098
|
+
return map;
|
|
1099
|
+
}
|
|
1100
|
+
/**
|
|
1101
|
+
* Derive the 95th-percentile uniqueness threshold across the cohort. Returns null
|
|
1102
|
+
* when the cohort is smaller than OUTLIER_PROMOTION_MIN_COHORT (percentile isn't
|
|
1103
|
+
* meaningful) or when no uniqueness was actually computable.
|
|
1104
|
+
*/
|
|
1105
|
+
computeUniquenessP95(uniquenessMap) {
|
|
1106
|
+
const available = Array.from(uniquenessMap.values())
|
|
1107
|
+
.filter(u => u.available)
|
|
1108
|
+
.map(u => u.value);
|
|
1109
|
+
if (available.length < OUTLIER_PROMOTION_MIN_COHORT) {
|
|
1110
|
+
return null;
|
|
1111
|
+
}
|
|
1112
|
+
const sorted = available.slice().sort((a, b) => a - b);
|
|
1113
|
+
const index = Math.floor(sorted.length * 0.95);
|
|
1114
|
+
const clampedIndex = Math.min(sorted.length - 1, index);
|
|
1115
|
+
return sorted[clampedIndex];
|
|
1116
|
+
}
|
|
1117
|
+
/**
|
|
1118
|
+
* Summarize importance scores into min/max/mean/median. Returns null for an empty
|
|
1119
|
+
* input so the caller can serialize it as an explicit "no data" signal in the
|
|
1120
|
+
* observability step.
|
|
1121
|
+
*/
|
|
1122
|
+
summarizeScores(scores) {
|
|
1123
|
+
if (scores.length === 0)
|
|
1124
|
+
return null;
|
|
1125
|
+
const sorted = scores.slice().sort((a, b) => a - b);
|
|
1126
|
+
const min = sorted[0];
|
|
1127
|
+
const max = sorted[sorted.length - 1];
|
|
1128
|
+
const mean = sorted.reduce((sum, v) => sum + v, 0) / sorted.length;
|
|
1129
|
+
const mid = sorted.length / 2;
|
|
1130
|
+
const median = sorted.length % 2 === 0
|
|
1131
|
+
? (sorted[mid - 1] + sorted[mid]) / 2
|
|
1132
|
+
: sorted[Math.floor(mid)];
|
|
1133
|
+
return {
|
|
1134
|
+
min: Math.round(min * 100) / 100,
|
|
1135
|
+
max: Math.round(max * 100) / 100,
|
|
1136
|
+
mean: Math.round(mean * 100) / 100,
|
|
1137
|
+
median: Math.round(median * 100) / 100
|
|
1138
|
+
};
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* Gather the 7 importance signals for a single note. Uniqueness is precomputed by
|
|
1142
|
+
* the caller so the full cohort can be examined in one pass for percentile
|
|
1143
|
+
* analysis. Other signals are derived from the note itself.
|
|
1144
|
+
*/
|
|
1145
|
+
gatherImportanceSignals(note, precomputedUniqueness) {
|
|
1146
|
+
const signals = [];
|
|
1147
|
+
const hoursSinceCreation = (Date.now() - new Date(note.__mj_CreatedAt).getTime()) / (1000 * 60 * 60);
|
|
1148
|
+
const hasConfidence = note.Comments?.includes('confidence') || false;
|
|
1149
|
+
const isCorrection = note.Comments?.includes('Superseded') || note.Comments?.includes('contradiction');
|
|
1150
|
+
signals.push({ signal: 'recencyDecay', value: Math.pow(0.995, hoursSinceCreation), available: true });
|
|
1151
|
+
signals.push({ signal: 'llmImportance', value: hasConfidence ? 0.7 : 0.5, available: hasConfidence });
|
|
1152
|
+
signals.push({ signal: 'relevance', value: 0.5, available: false }); // Phase 2 enrichment
|
|
1153
|
+
signals.push({
|
|
1154
|
+
signal: 'uniqueness',
|
|
1155
|
+
value: precomputedUniqueness?.value ?? 0.5,
|
|
1156
|
+
available: precomputedUniqueness?.available ?? false
|
|
1157
|
+
});
|
|
1158
|
+
signals.push({ signal: 'correctionBoost', value: isCorrection ? 1.0 : 0.0, available: true });
|
|
1159
|
+
signals.push({ signal: 'goalAlignment', value: 0.5, available: false }); // Phase 2 enrichment
|
|
1160
|
+
signals.push({ signal: 'userMark', value: note.IsAutoGenerated ? 0.0 : 1.0, available: true });
|
|
1161
|
+
return signals;
|
|
1162
|
+
}
|
|
1163
|
+
/** Compute a weighted importance score (0-10) from signals, redistributing weight from unavailable signals. */
|
|
1164
|
+
computeWeightedImportanceScore(signals) {
|
|
1165
|
+
const weights = IMPORTANCE_CONFIG.weights;
|
|
1166
|
+
const availableSignals = signals.filter(s => s.available);
|
|
1167
|
+
const totalAvailableWeight = availableSignals.reduce((sum, s) => sum + (weights[s.signal] || 0), 0);
|
|
1168
|
+
const unavailableWeight = signals.filter(s => !s.available).reduce((sum, s) => sum + (weights[s.signal] || 0), 0);
|
|
1169
|
+
let rawScore = 0;
|
|
1170
|
+
for (const s of availableSignals) {
|
|
1171
|
+
if (totalAvailableWeight > 0) {
|
|
1172
|
+
const adjustedWeight = (weights[s.signal] || 0) + ((weights[s.signal] || 0) / totalAvailableWeight) * unavailableWeight;
|
|
1173
|
+
rawScore += s.value * adjustedWeight;
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
let score = rawScore * 10.0;
|
|
1177
|
+
const isCorrection = signals.some(s => s.signal === 'correctionBoost' && s.value > 0);
|
|
1178
|
+
if (isCorrection) {
|
|
1179
|
+
score = Math.min(10.0, score + IMPORTANCE_CONFIG.correctionBonusPoints);
|
|
1180
|
+
}
|
|
1181
|
+
return Math.round(score * 100) / 100;
|
|
1182
|
+
}
|
|
864
1183
|
/**
|
|
865
1184
|
* Determine whether consolidation should run based on CONSOLIDATION_CONFIG.frequency.
|
|
866
1185
|
* Uses AIAgentRun history instead of static in-memory state so that timing
|
|
@@ -871,45 +1190,90 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
871
1190
|
* - 'hourly': Run if 1+ hour since last completed MM run
|
|
872
1191
|
* - 'daily': Run if 24+ hours since last completed MM run
|
|
873
1192
|
* - number: Run every N completed MM runs
|
|
1193
|
+
*
|
|
1194
|
+
* @param forceMaintenance When true, bypasses all gating and runs maintenance
|
|
1195
|
+
* regardless of frequency/history. Set by callers via `params.data.forceMaintenance`
|
|
1196
|
+
* (see `executeAgentInternal`). Intended for tests, manual admin triggers, and
|
|
1197
|
+
* the E2E verification script — NOT for production config. Previously this was
|
|
1198
|
+
* an env var (`FORCE_MAINTENANCE=1`), which was removed because an env-var
|
|
1199
|
+
* sidechannel could accidentally leak into production and cause massive DB/LLM
|
|
1200
|
+
* churn on every run.
|
|
1201
|
+
*
|
|
1202
|
+
* Returns both a boolean and the reason it fired so the observability layer can
|
|
1203
|
+
* record the trigger type on the parent consolidation run step.
|
|
874
1204
|
*/
|
|
875
|
-
async shouldRunConsolidation(agentId, contextUser) {
|
|
1205
|
+
async shouldRunConsolidation(agentId, contextUser, forceMaintenance) {
|
|
1206
|
+
if (forceMaintenance)
|
|
1207
|
+
return { shouldRun: true, triggerType: 'forced' };
|
|
876
1208
|
const freq = CONSOLIDATION_CONFIG.frequency;
|
|
877
|
-
if (freq === 'disabled')
|
|
878
|
-
return false;
|
|
1209
|
+
if (freq === 'disabled')
|
|
1210
|
+
return { shouldRun: false, triggerType: 'disabled' };
|
|
1211
|
+
if (freq === 'every-run')
|
|
1212
|
+
return { shouldRun: true, triggerType: 'every-run' };
|
|
1213
|
+
if (freq === 'hourly' || freq === 'daily') {
|
|
1214
|
+
return this.shouldRunConsolidationByTimeWindow(agentId, contextUser, freq === 'hourly' ? 1 : 24);
|
|
879
1215
|
}
|
|
880
|
-
if (freq === '
|
|
881
|
-
return
|
|
1216
|
+
if (typeof freq === 'number') {
|
|
1217
|
+
return this.shouldRunConsolidationByRunCount(agentId, contextUser, freq);
|
|
882
1218
|
}
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
1219
|
+
return { shouldRun: true, triggerType: 'every-run' };
|
|
1220
|
+
}
|
|
1221
|
+
/**
|
|
1222
|
+
* Time-window frequency gate: returns shouldRun=true with triggerType='time' when
|
|
1223
|
+
* enough time has passed since the last memory-manager run, or triggerType='event'
|
|
1224
|
+
* when the event-driven new-note threshold has been crossed. Falls back to
|
|
1225
|
+
* triggerType='not-triggered' when neither condition fires.
|
|
1226
|
+
*/
|
|
1227
|
+
async shouldRunConsolidationByTimeWindow(agentId, contextUser, thresholdHours) {
|
|
1228
|
+
const lastRun = await this.GetLastRunTime(agentId, contextUser);
|
|
1229
|
+
if (!lastRun)
|
|
1230
|
+
return { shouldRun: true, triggerType: 'time' };
|
|
1231
|
+
const hoursSinceLast = (Date.now() - lastRun.getTime()) / (1000 * 60 * 60);
|
|
1232
|
+
if (hoursSinceLast >= thresholdHours)
|
|
1233
|
+
return { shouldRun: true, triggerType: 'time' };
|
|
1234
|
+
// Event-driven trigger: consolidate when at least `noteCountTrigger` new notes
|
|
1235
|
+
// have been added system-wide since the last Memory Manager run.
|
|
1236
|
+
//
|
|
1237
|
+
// NOTE: this query intentionally does NOT filter by AgentID. The `agentId` parameter
|
|
1238
|
+
// here is the Memory Manager's own ID (passed from `params.agent.ID` at the call
|
|
1239
|
+
// site), not the agent that owns the notes. Filtering the count by MM's own agent
|
|
1240
|
+
// ID would never match anything because MM doesn't own notes — it processes notes
|
|
1241
|
+
// belonging to other (memory-enabled) agents.
|
|
1242
|
+
const rv = new RunView();
|
|
1243
|
+
const newNoteResult = await rv.RunView({
|
|
1244
|
+
EntityName: 'MJ: AI Agent Notes',
|
|
1245
|
+
ExtraFilter: `IsAutoGenerated=1 AND Status='Active' AND __mj_CreatedAt > '${lastRun.toISOString()}'`,
|
|
1246
|
+
ResultType: 'count_only'
|
|
1247
|
+
}, contextUser);
|
|
1248
|
+
if (newNoteResult.Success && (newNoteResult.TotalRowCount || 0) >= CONSOLIDATION_CONFIG.noteCountTrigger) {
|
|
1249
|
+
if (this._verbose) {
|
|
1250
|
+
LogStatus(`Memory Manager: Event-driven consolidation trigger fired (${newNoteResult.TotalRowCount} new notes >= ${CONSOLIDATION_CONFIG.noteCountTrigger} threshold since last MM run)`);
|
|
888
1251
|
}
|
|
889
|
-
|
|
890
|
-
return hoursSinceLast >= thresholdHours;
|
|
1252
|
+
return { shouldRun: true, triggerType: 'event' };
|
|
891
1253
|
}
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
}
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
return
|
|
1254
|
+
return { shouldRun: false, triggerType: 'not-triggered' };
|
|
1255
|
+
}
|
|
1256
|
+
/** Run-count frequency gate: consolidate every N completed runs. */
|
|
1257
|
+
async shouldRunConsolidationByRunCount(agentId, contextUser, everyN) {
|
|
1258
|
+
const rv = new RunView();
|
|
1259
|
+
const countResult = await rv.RunView({
|
|
1260
|
+
EntityName: 'MJ: AI Agent Runs',
|
|
1261
|
+
ExtraFilter: `AgentID='${agentId}' AND Status='Completed'`,
|
|
1262
|
+
ResultType: 'count_only'
|
|
1263
|
+
}, contextUser);
|
|
1264
|
+
const completedCount = countResult.Success ? (countResult.TotalRowCount || 0) : 0;
|
|
1265
|
+
const shouldRun = completedCount > 0 && completedCount % everyN === 0;
|
|
1266
|
+
return { shouldRun, triggerType: shouldRun ? 'count' : 'not-triggered' };
|
|
905
1267
|
}
|
|
906
1268
|
/**
|
|
907
1269
|
* Consolidate related notes into single, comprehensive notes.
|
|
908
1270
|
* This method finds clusters of similar notes and synthesizes them.
|
|
909
1271
|
*
|
|
1272
|
+
* Returns per-cluster verification results alongside the consolidation counts so
|
|
1273
|
+
* the phase-level "Verify Consolidation Output" run step can aggregate them.
|
|
1274
|
+
*
|
|
910
1275
|
* @param agentId Optional - consolidate notes for a specific agent only
|
|
911
1276
|
* @param contextUser The context user for database operations
|
|
912
|
-
* @returns Number of notes consolidated (archived)
|
|
913
1277
|
*/
|
|
914
1278
|
async consolidateRelatedNotes(agentId, contextUser) {
|
|
915
1279
|
const allNotes = AIEngine.Instance.AgentNotes;
|
|
@@ -921,7 +1285,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
921
1285
|
if (this._verbose) {
|
|
922
1286
|
LogStatus(`Memory Manager: Only ${activeNotes.length} active notes - need at least ${CONSOLIDATION_CONFIG.minClusterSize} to consolidate`);
|
|
923
1287
|
}
|
|
924
|
-
return { consolidated: 0, archived: 0 };
|
|
1288
|
+
return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
|
|
925
1289
|
}
|
|
926
1290
|
if (this._verbose) {
|
|
927
1291
|
LogStatus(`Memory Manager: Analyzing ${activeNotes.length} active notes for consolidation`);
|
|
@@ -931,7 +1295,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
931
1295
|
if (this._verbose) {
|
|
932
1296
|
LogStatus(`Memory Manager: No clusters found with ${CONSOLIDATION_CONFIG.minClusterSize}+ similar notes`);
|
|
933
1297
|
}
|
|
934
|
-
return { consolidated: 0, archived: 0 };
|
|
1298
|
+
return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
|
|
935
1299
|
}
|
|
936
1300
|
if (this._verbose) {
|
|
937
1301
|
LogStatus(`Memory Manager: Found ${clusters.length} clusters to consolidate`);
|
|
@@ -940,15 +1304,17 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
940
1304
|
const consolidatePrompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Consolidate Notes' && p.Category === 'MJ: System');
|
|
941
1305
|
if (!consolidatePrompt) {
|
|
942
1306
|
LogError('Memory Manager: Consolidation prompt not found');
|
|
943
|
-
return { consolidated: 0, archived: 0 };
|
|
1307
|
+
return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
|
|
944
1308
|
}
|
|
945
1309
|
const aiNoteTypeId = AIEngine.Instance.AgenteNoteTypeIDByName('AI');
|
|
946
1310
|
if (!aiNoteTypeId) {
|
|
947
1311
|
LogError('Memory Manager: Could not find "AI" note type');
|
|
948
|
-
return { consolidated: 0, archived: 0 };
|
|
1312
|
+
return { consolidated: 0, archived: 0, newNoteIds: [], verifications: [] };
|
|
949
1313
|
}
|
|
950
1314
|
let consolidated = 0;
|
|
951
1315
|
let archived = 0;
|
|
1316
|
+
const newNoteIds = [];
|
|
1317
|
+
const verifications = [];
|
|
952
1318
|
const runner = new AIPromptRunner();
|
|
953
1319
|
const md = new Metadata();
|
|
954
1320
|
for (const cluster of clusters) {
|
|
@@ -956,12 +1322,16 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
956
1322
|
const result = await this.processConsolidationCluster(cluster, consolidatePrompt, aiNoteTypeId, runner, md, contextUser);
|
|
957
1323
|
consolidated += result.consolidated;
|
|
958
1324
|
archived += result.archived;
|
|
1325
|
+
if (result.newNoteId)
|
|
1326
|
+
newNoteIds.push(result.newNoteId);
|
|
1327
|
+
if (result.verification)
|
|
1328
|
+
verifications.push(result.verification);
|
|
959
1329
|
}
|
|
960
1330
|
catch (error) {
|
|
961
1331
|
LogError('Memory Manager: Exception during consolidation:', error);
|
|
962
1332
|
}
|
|
963
1333
|
}
|
|
964
|
-
return { consolidated, archived };
|
|
1334
|
+
return { consolidated, archived, newNoteIds, verifications };
|
|
965
1335
|
}
|
|
966
1336
|
/**
|
|
967
1337
|
* Find clusters of semantically similar notes suitable for consolidation.
|
|
@@ -970,7 +1340,12 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
970
1340
|
async findConsolidationClusters(activeNotes) {
|
|
971
1341
|
const clusters = [];
|
|
972
1342
|
const processedIds = new Set();
|
|
973
|
-
|
|
1343
|
+
// Protection tier filtering: exclude Immutable and Protected notes from consolidation clustering
|
|
1344
|
+
const eligibleNotes = activeNotes.filter(n => {
|
|
1345
|
+
const tier = n.ProtectionTier || 'Standard';
|
|
1346
|
+
return tier !== 'Immutable' && tier !== 'Protected';
|
|
1347
|
+
});
|
|
1348
|
+
for (const note of eligibleNotes) {
|
|
974
1349
|
if (processedIds.has(note.ID)) {
|
|
975
1350
|
continue;
|
|
976
1351
|
}
|
|
@@ -979,101 +1354,527 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
979
1354
|
processedIds.add(note.ID);
|
|
980
1355
|
for (const similar of similarNotes) {
|
|
981
1356
|
if (!processedIds.has(similar.note.ID)) {
|
|
982
|
-
|
|
983
|
-
|
|
1357
|
+
// Also check that similar notes pass protection tier filter
|
|
1358
|
+
const similarTier = similar.note.ProtectionTier || 'Standard';
|
|
1359
|
+
if (similarTier !== 'Immutable' && similarTier !== 'Protected') {
|
|
1360
|
+
cluster.push(similar.note);
|
|
1361
|
+
processedIds.add(similar.note.ID);
|
|
1362
|
+
}
|
|
984
1363
|
}
|
|
985
1364
|
}
|
|
986
|
-
|
|
987
|
-
|
|
1365
|
+
// Determine minClusterSize based on tier: Ephemeral uses relaxed threshold of 2
|
|
1366
|
+
const noteTier = note.ProtectionTier || 'Standard';
|
|
1367
|
+
const minSize = noteTier === 'Ephemeral' ? 2 : CONSOLIDATION_CONFIG.minClusterSize;
|
|
1368
|
+
if (cluster.length >= minSize) {
|
|
1369
|
+
// Enforce maxClusterSize by splitting oversized clusters
|
|
1370
|
+
if (cluster.length > CONSOLIDATION_CONFIG.maxClusterSize) {
|
|
1371
|
+
const splitClusters = this.splitOversizedCluster(cluster);
|
|
1372
|
+
clusters.push(...splitClusters);
|
|
1373
|
+
}
|
|
1374
|
+
else {
|
|
1375
|
+
clusters.push(cluster);
|
|
1376
|
+
}
|
|
988
1377
|
}
|
|
989
1378
|
}
|
|
990
1379
|
return clusters;
|
|
991
1380
|
}
|
|
992
1381
|
/**
|
|
993
|
-
*
|
|
994
|
-
*
|
|
1382
|
+
* Split a cluster that exceeds maxClusterSize into sub-clusters.
|
|
1383
|
+
* Takes the first maxClusterSize notes (sorted by position in original cluster, which preserves
|
|
1384
|
+
* similarity ordering from FindSimilarAgentNotes), and forms a second cluster from the remainder
|
|
1385
|
+
* if it meets minClusterSize.
|
|
995
1386
|
*/
|
|
996
|
-
|
|
997
|
-
const
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1387
|
+
splitOversizedCluster(cluster) {
|
|
1388
|
+
const result = [];
|
|
1389
|
+
const maxSize = CONSOLIDATION_CONFIG.maxClusterSize;
|
|
1390
|
+
const minSize = CONSOLIDATION_CONFIG.minClusterSize;
|
|
1391
|
+
// Split into chunks of maxClusterSize
|
|
1392
|
+
for (let i = 0; i < cluster.length; i += maxSize) {
|
|
1393
|
+
const chunk = cluster.slice(i, i + maxSize);
|
|
1394
|
+
if (chunk.length >= minSize) {
|
|
1395
|
+
result.push(chunk);
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
return result;
|
|
1399
|
+
}
|
|
1400
|
+
/**
|
|
1401
|
+
* Walk DerivedFromNoteIDs chains breadth-first to find all generation-0 (original) source notes.
|
|
1402
|
+
* Batches each depth level into a single `ID IN (...)` query instead of one query per ID.
|
|
1403
|
+
*/
|
|
1404
|
+
async resolveOriginalSources(noteIds, contextUser) {
|
|
1405
|
+
const originals = [];
|
|
1406
|
+
const visited = new Set();
|
|
1407
|
+
let currentIds = noteIds.filter(id => !visited.has(id));
|
|
1408
|
+
currentIds.forEach(id => visited.add(id));
|
|
1409
|
+
let depth = 0;
|
|
1410
|
+
while (currentIds.length > 0 && depth <= MAX_SOURCE_RESOLUTION_DEPTH) {
|
|
1411
|
+
if (depth > MAX_SOURCE_RESOLUTION_DEPTH) {
|
|
1412
|
+
LogError(`Memory Manager: resolveOriginalSources() exceeded max depth ${MAX_SOURCE_RESOLUTION_DEPTH} — treating remaining ${currentIds.length} IDs as terminal to prevent runaway recursion`);
|
|
1413
|
+
break;
|
|
1414
|
+
}
|
|
1415
|
+
const results = await this.loadResolvedNotes(currentIds, contextUser);
|
|
1416
|
+
if (results.length === 0)
|
|
1417
|
+
break;
|
|
1418
|
+
currentIds = this.collectNextLevelSourceIds(results, originals, visited);
|
|
1419
|
+
depth++;
|
|
1420
|
+
}
|
|
1421
|
+
return originals;
|
|
1422
|
+
}
|
|
1423
|
+
/** Load a batch of notes by ID with the fields needed for source resolution. */
|
|
1424
|
+
async loadResolvedNotes(noteIds, contextUser) {
|
|
1425
|
+
const resolvedFields = [
|
|
1426
|
+
'ID', 'ConsolidationCount', 'DerivedFromNoteIDs', 'Note', 'Type',
|
|
1427
|
+
'AccessCount', 'ImportanceScore', 'AgentID', 'UserID', 'CompanyID', '__mj_CreatedAt'
|
|
1428
|
+
];
|
|
1429
|
+
const inClause = noteIds.map(id => `'${id}'`).join(',');
|
|
1430
|
+
const rv = new RunView();
|
|
1431
|
+
const result = await rv.RunView({
|
|
1432
|
+
EntityName: 'MJ: AI Agent Notes',
|
|
1433
|
+
ExtraFilter: `ID IN (${inClause})`,
|
|
1434
|
+
Fields: resolvedFields,
|
|
1435
|
+
}, contextUser);
|
|
1436
|
+
return result.Success && result.Results ? result.Results : [];
|
|
1437
|
+
}
|
|
1438
|
+
/**
|
|
1439
|
+
* Partition a batch of resolved notes into terminal originals (pushed into `originals`)
|
|
1440
|
+
* and parent IDs that need further resolution. Mutates `originals` and `visited`.
|
|
1441
|
+
*/
|
|
1442
|
+
collectNextLevelSourceIds(notes, originals, visited) {
|
|
1443
|
+
const nextLevelIds = [];
|
|
1444
|
+
for (const note of notes) {
|
|
1445
|
+
const consolidationCount = note.ConsolidationCount || 0;
|
|
1446
|
+
if (consolidationCount === 0 || !note.DerivedFromNoteIDs) {
|
|
1447
|
+
originals.push(note);
|
|
1448
|
+
continue;
|
|
1449
|
+
}
|
|
1450
|
+
try {
|
|
1451
|
+
const parentIds = JSON.parse(note.DerivedFromNoteIDs);
|
|
1452
|
+
for (const parentId of parentIds) {
|
|
1453
|
+
if (!visited.has(parentId)) {
|
|
1454
|
+
visited.add(parentId);
|
|
1455
|
+
nextLevelIds.push(parentId);
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
catch {
|
|
1460
|
+
LogError(`Memory Manager: Malformed DerivedFromNoteIDs on note ${note.ID} — treating as terminal source`);
|
|
1461
|
+
originals.push(note);
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
return nextLevelIds;
|
|
1465
|
+
}
|
|
1466
|
+
/**
|
|
1467
|
+
* Verify that a consolidated note preserves key entities from its source notes.
|
|
1468
|
+
* Extracts numbers, dates, and proper nouns from source notes and checks they appear in the output.
|
|
1469
|
+
* Non-blocking: returns verification results but does not prevent consolidation.
|
|
1470
|
+
*/
|
|
1471
|
+
verifyConsolidationOutput(sourceNotes, consolidatedText) {
|
|
1472
|
+
const entities = new Set();
|
|
1473
|
+
const consolidatedLower = consolidatedText.toLowerCase();
|
|
1474
|
+
for (const note of sourceNotes) {
|
|
1475
|
+
this.extractVerifiableEntities(note.Note || '', entities);
|
|
1476
|
+
}
|
|
1477
|
+
const missing = [];
|
|
1478
|
+
for (const entity of entities) {
|
|
1479
|
+
if (!consolidatedLower.includes(entity.toLowerCase())) {
|
|
1480
|
+
missing.push(entity);
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
return {
|
|
1484
|
+
entitiesChecked: entities.size,
|
|
1485
|
+
entitiesMissing: missing,
|
|
1486
|
+
passed: missing.length === 0
|
|
1487
|
+
};
|
|
1488
|
+
}
|
|
1489
|
+
/**
|
|
1490
|
+
* Harvest numbers, dates, proper nouns, and camelCase identifiers from a note's text
|
|
1491
|
+
* into the supplied set. Used only by verifyConsolidationOutput to assemble a
|
|
1492
|
+
* best-effort entity inventory for post-consolidation checks.
|
|
1493
|
+
*/
|
|
1494
|
+
extractVerifiableEntities(text, entities) {
|
|
1495
|
+
const numbers = text.match(/\b\d+\.?\d*%?\b/g);
|
|
1496
|
+
if (numbers)
|
|
1497
|
+
numbers.forEach(n => entities.add(n));
|
|
1498
|
+
const dates = text.match(/\b\d{4}-\d{2}-\d{2}\b|\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\s+\d{1,2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b/gi);
|
|
1499
|
+
if (dates)
|
|
1500
|
+
dates.forEach(d => entities.add(d));
|
|
1501
|
+
const properNouns = text.match(/(?<=[.!?]\s+|^)(?:[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)/g);
|
|
1502
|
+
if (properNouns) {
|
|
1503
|
+
const skipWords = new Set(['the', 'a', 'an', 'this', 'that', 'it', 'we', 'they', 'user', 'note']);
|
|
1504
|
+
properNouns
|
|
1505
|
+
.filter(pn => !skipWords.has(pn.toLowerCase()) && pn.length > 2)
|
|
1506
|
+
.forEach(pn => entities.add(pn));
|
|
1507
|
+
}
|
|
1508
|
+
const techTerms = text.match(/\b[a-z]+(?:[A-Z][a-z]+)+\b|\b(?:[A-Z][a-z]+){2,}\b/g);
|
|
1509
|
+
if (techTerms)
|
|
1510
|
+
techTerms.forEach(t => entities.add(t));
|
|
1511
|
+
}
|
|
1512
|
+
/**
|
|
1513
|
+
* Detect and resolve contradictions between semantically similar notes.
|
|
1514
|
+
* Uses a three-stage pipeline: embedding pre-filter, entity-attribute extraction, LLM judgment.
|
|
1515
|
+
* High-importance pairs (both >= highImportanceThreshold) are flagged but preserved.
|
|
1516
|
+
* Protected-tier notes are never auto-revoked.
|
|
1517
|
+
*
|
|
1518
|
+
* Aggregates the per-pair entity-attribute-value triple count so the observability
|
|
1519
|
+
* step can report how much structural reasoning the LLM produced.
|
|
1520
|
+
*/
|
|
1521
|
+
async detectAndResolveContradictions(contextUser, consolidatedNoteIds) {
|
|
1522
|
+
const result = { contradictionsFound: 0, resolved: 0, flagged: 0, pairsAnalyzed: 0, entityTriplesExtracted: 0 };
|
|
1523
|
+
const prompt = AIEngine.Instance.Prompts.find(p => p.Name === 'Memory Manager - Detect Contradictions' && p.Category === 'MJ: System');
|
|
1524
|
+
if (!prompt) {
|
|
1525
|
+
LogError('Memory Manager: Contradiction detection prompt not found, skipping phase');
|
|
1526
|
+
return result;
|
|
1527
|
+
}
|
|
1528
|
+
const candidatePairs = await this.buildContradictionCandidatePairs(consolidatedNoteIds);
|
|
1529
|
+
if (candidatePairs.length === 0)
|
|
1530
|
+
return result;
|
|
1531
|
+
const runner = new AIPromptRunner();
|
|
1532
|
+
const md = new Metadata();
|
|
1533
|
+
for (const [noteA, noteB] of candidatePairs) {
|
|
1534
|
+
result.pairsAnalyzed++;
|
|
1535
|
+
const llmResult = await this.evaluateContradictionPair(noteA, noteB, prompt, runner, contextUser);
|
|
1536
|
+
if (!llmResult)
|
|
1537
|
+
continue;
|
|
1538
|
+
result.entityTriplesExtracted += llmResult.triplesExtracted;
|
|
1539
|
+
if (!llmResult.isContradiction)
|
|
1540
|
+
continue;
|
|
1541
|
+
result.contradictionsFound++;
|
|
1542
|
+
const resolved = await this.resolveContradiction(noteA, noteB, llmResult, md, contextUser);
|
|
1543
|
+
if (resolved === 'flagged')
|
|
1544
|
+
result.flagged++;
|
|
1545
|
+
else if (resolved === 'resolved')
|
|
1546
|
+
result.resolved++;
|
|
1547
|
+
}
|
|
1548
|
+
return result;
|
|
1549
|
+
}
|
|
1550
|
+
/** Build unique candidate pairs of semantically similar notes for contradiction analysis. */
|
|
1551
|
+
async buildContradictionCandidatePairs(consolidatedNoteIds) {
|
|
1552
|
+
const allNotes = AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active' && n.IsAutoGenerated && (n.ProtectionTier || 'Standard') !== 'Immutable');
|
|
1553
|
+
const processedPairs = new Set();
|
|
1554
|
+
const pairs = [];
|
|
1555
|
+
for (const note of allNotes) {
|
|
1556
|
+
if (!note.Note || pairs.length >= CONTRADICTION_CONFIG.maxPairsPerRun)
|
|
1557
|
+
break;
|
|
1558
|
+
const similarNotes = await AIEngine.Instance.FindSimilarAgentNotes(note.Note, note.AgentID, undefined, undefined, 5, CONTRADICTION_CONFIG.similarityThreshold);
|
|
1559
|
+
for (const similar of similarNotes) {
|
|
1560
|
+
const pairKey = [note.ID, similar.note.ID].sort().join('|');
|
|
1561
|
+
if (processedPairs.has(pairKey))
|
|
1562
|
+
continue;
|
|
1563
|
+
processedPairs.add(pairKey);
|
|
1564
|
+
if (consolidatedNoteIds.has(note.ID) && consolidatedNoteIds.has(similar.note.ID))
|
|
1565
|
+
continue;
|
|
1566
|
+
pairs.push([note, similar.note]);
|
|
1567
|
+
if (pairs.length >= CONTRADICTION_CONFIG.maxPairsPerRun)
|
|
1568
|
+
break;
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
return pairs;
|
|
1572
|
+
}
|
|
1573
|
+
/** Evaluate a single note pair for contradiction using the LLM prompt. */
|
|
1574
|
+
async evaluateContradictionPair(noteA, noteB, prompt, runner, contextUser) {
|
|
1575
|
+
const params = new AIPromptParams();
|
|
1576
|
+
params.prompt = prompt;
|
|
1577
|
+
params.data = {
|
|
1578
|
+
noteA: { id: noteA.ID, type: noteA.Type, content: noteA.Note, createdAt: noteA.__mj_CreatedAt, accessCount: noteA.AccessCount, importanceScore: noteA.ImportanceScore || null, agentId: noteA.AgentID, userId: noteA.UserID, companyId: noteA.CompanyID },
|
|
1579
|
+
noteB: { id: noteB.ID, type: noteB.Type, content: noteB.Note, createdAt: noteB.__mj_CreatedAt, accessCount: noteB.AccessCount, importanceScore: noteB.ImportanceScore || null, agentId: noteB.AgentID, userId: noteB.UserID, companyId: noteB.CompanyID }
|
|
1580
|
+
};
|
|
1581
|
+
params.contextUser = contextUser;
|
|
1582
|
+
params.attemptJSONRepair = true;
|
|
1583
|
+
params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
|
|
1584
|
+
const llmResult = await runner.ExecutePrompt(params);
|
|
1585
|
+
if (!llmResult.success || !llmResult.result)
|
|
1586
|
+
return null;
|
|
1587
|
+
let parsed = llmResult.result;
|
|
1588
|
+
if (typeof parsed === 'string') {
|
|
1589
|
+
const cleaned = CleanAndParseJSON(parsed, true);
|
|
1590
|
+
if (!cleaned)
|
|
1591
|
+
return null;
|
|
1592
|
+
parsed = cleaned;
|
|
1593
|
+
}
|
|
1594
|
+
return {
|
|
1595
|
+
isContradiction: parsed.isContradiction,
|
|
1596
|
+
keepNoteId: parsed.keepNoteId,
|
|
1597
|
+
revokeNoteId: parsed.revokeNoteId,
|
|
1598
|
+
reason: parsed.reason,
|
|
1599
|
+
triplesExtracted: this.countEntityTriples(parsed.entityTriples)
|
|
1600
|
+
};
|
|
1601
|
+
}
|
|
1602
|
+
/**
|
|
1603
|
+
* Count entity-attribute-value triples returned by the contradiction detection prompt.
|
|
1604
|
+
* Accepts any shape the LLM may produce: object of arrays keyed by noteId, flat array,
|
|
1605
|
+
* or null/undefined. Unrecognized shapes return 0 so a malformed response can't poison
|
|
1606
|
+
* the aggregate counter.
|
|
1607
|
+
*/
|
|
1608
|
+
countEntityTriples(entityTriples) {
|
|
1609
|
+
if (!entityTriples)
|
|
1610
|
+
return 0;
|
|
1611
|
+
if (Array.isArray(entityTriples))
|
|
1612
|
+
return entityTriples.length;
|
|
1613
|
+
if (typeof entityTriples === 'object') {
|
|
1614
|
+
let total = 0;
|
|
1615
|
+
for (const value of Object.values(entityTriples)) {
|
|
1616
|
+
if (Array.isArray(value))
|
|
1617
|
+
total += value.length;
|
|
1618
|
+
}
|
|
1619
|
+
return total;
|
|
1620
|
+
}
|
|
1621
|
+
return 0;
|
|
1622
|
+
}
|
|
1623
|
+
/** Resolve a detected contradiction: flag both notes if high-importance/Protected, otherwise revoke the loser. */
|
|
1624
|
+
async resolveContradiction(noteA, noteB, llmResult, md, contextUser) {
|
|
1625
|
+
if (this.shouldFlagContradiction(noteA, noteB)) {
|
|
1626
|
+
await this.flagContradictionPair(noteA, noteB, llmResult, md, contextUser);
|
|
1627
|
+
return 'flagged';
|
|
1628
|
+
}
|
|
1629
|
+
const { revokeNoteId: revokeId, keepNoteId: keepId } = llmResult;
|
|
1630
|
+
if (!revokeId || !keepId)
|
|
1631
|
+
return 'skipped';
|
|
1632
|
+
const noteToRevoke = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
1633
|
+
if (!(await noteToRevoke.Load(revokeId)))
|
|
1634
|
+
return 'skipped';
|
|
1635
|
+
noteToRevoke.Status = 'Revoked';
|
|
1636
|
+
noteToRevoke.ConsolidatedIntoNoteID = keepId;
|
|
1637
|
+
noteToRevoke.Comments = `[Contradiction resolved: superseded by note ${keepId}. ${llmResult.reason}]`;
|
|
1638
|
+
if (!(await noteToRevoke.Save()))
|
|
1639
|
+
return 'skipped';
|
|
1640
|
+
if (this._verbose) {
|
|
1641
|
+
LogStatus(`Memory Manager: Contradiction resolved — revoked note ${revokeId} in favor of ${keepId}`);
|
|
1642
|
+
}
|
|
1643
|
+
return 'resolved';
|
|
1644
|
+
}
|
|
1645
|
+
/** Decide whether a contradiction pair must be flagged rather than auto-resolved. */
|
|
1646
|
+
shouldFlagContradiction(noteA, noteB) {
|
|
1647
|
+
const bothHighImportance = (noteA.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold &&
|
|
1648
|
+
(noteB.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold;
|
|
1649
|
+
const hasProtected = (noteA.ProtectionTier || 'Standard') === 'Protected' || (noteB.ProtectionTier || 'Standard') === 'Protected';
|
|
1650
|
+
return bothHighImportance || hasProtected;
|
|
1651
|
+
}
|
|
1652
|
+
/** Flag both notes in a contradiction pair with a cross-referencing audit comment. */
|
|
1653
|
+
async flagContradictionPair(noteA, noteB, llmResult, md, contextUser) {
|
|
1654
|
+
const flagComment = `[Contradiction flagged: conflicts with note ${llmResult.revokeNoteId && UUIDsEqual(noteA.ID, llmResult.revokeNoteId) ? llmResult.keepNoteId : llmResult.revokeNoteId}. ${llmResult.reason}]`;
|
|
1655
|
+
for (const flagNote of [noteA, noteB]) {
|
|
1656
|
+
const noteToFlag = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
1657
|
+
if (await noteToFlag.Load(flagNote.ID)) {
|
|
1658
|
+
noteToFlag.Comments = `${noteToFlag.Comments || ''} ${flagComment}`.trim();
|
|
1659
|
+
await noteToFlag.Save();
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
if (this._verbose) {
|
|
1663
|
+
const bothHighImportance = (noteA.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold &&
|
|
1664
|
+
(noteB.ImportanceScore || 0) >= CONTRADICTION_CONFIG.highImportanceThreshold;
|
|
1665
|
+
const reason = bothHighImportance ? 'both high-importance' : 'Protected tier present';
|
|
1666
|
+
LogStatus(`Memory Manager: Contradiction flagged (${reason}): "${noteA.Note?.substring(0, 40)}..." vs "${noteB.Note?.substring(0, 40)}..."`);
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
/**
|
|
1670
|
+
* Process a single consolidation cluster: run the LLM prompt, create the consolidated note,
|
|
1671
|
+
* and revoke source notes.
|
|
1672
|
+
*
|
|
1673
|
+
* Emits a per-cluster "Process Consolidation Cluster" observability step that runs
|
|
1674
|
+
* between the parent consolidation step's create/finalize calls, producing an
|
|
1675
|
+
* implicit parent-child relationship via step-number ordering.
|
|
1676
|
+
*/
|
|
1677
|
+
async processConsolidationCluster(cluster, consolidatePrompt, aiNoteTypeId, runner, md, contextUser) {
|
|
1678
|
+
const maxGeneration = Math.max(...cluster.map(n => n.ConsolidationCount || 0));
|
|
1679
|
+
const clusterStep = await this.CreateRunStep('Prompt', 'Process Consolidation Cluster', {
|
|
1680
|
+
clusterSize: cluster.length,
|
|
1681
|
+
noteIds: cluster.map(n => n.ID),
|
|
1682
|
+
maxGeneration
|
|
1683
|
+
});
|
|
1684
|
+
const outcome = {
|
|
1685
|
+
consolidated: 0,
|
|
1686
|
+
archived: 0,
|
|
1687
|
+
newNoteId: null,
|
|
1688
|
+
verification: null
|
|
1689
|
+
};
|
|
1690
|
+
let stepOutput = { clusterSize: cluster.length };
|
|
1691
|
+
try {
|
|
1692
|
+
const { notesForPrompt, promptData } = await this.buildConsolidationPromptData(cluster, contextUser);
|
|
1693
|
+
const parsedResult = await this.runConsolidationPrompt(consolidatePrompt, promptData, runner, contextUser);
|
|
1694
|
+
if (!parsedResult) {
|
|
1695
|
+
stepOutput = { ...stepOutput, shouldConsolidate: false, skipReason: 'prompt-failed' };
|
|
1696
|
+
return outcome;
|
|
1697
|
+
}
|
|
1698
|
+
if (!parsedResult.shouldConsolidate) {
|
|
1699
|
+
if (this._verbose) {
|
|
1700
|
+
LogStatus(`Memory Manager: Skipping cluster consolidation - ${parsedResult.reason}`);
|
|
1701
|
+
}
|
|
1702
|
+
stepOutput = { ...stepOutput, shouldConsolidate: false, skipReason: parsedResult.reason };
|
|
1703
|
+
return outcome;
|
|
1704
|
+
}
|
|
1705
|
+
const consolidatedNoteData = parsedResult.consolidatedNote;
|
|
1706
|
+
const newNote = await this.createConsolidatedNote(cluster, consolidatedNoteData, parsedResult.reason, aiNoteTypeId, md, contextUser);
|
|
1707
|
+
if (!newNote) {
|
|
1708
|
+
stepOutput = { ...stepOutput, shouldConsolidate: true, skipReason: 'save-failed' };
|
|
1709
|
+
return outcome;
|
|
1710
|
+
}
|
|
1711
|
+
outcome.verification = await this.annotateConsolidationVerification(newNote, notesForPrompt, consolidatedNoteData.content);
|
|
1712
|
+
outcome.archived = await this.revokeSourceNotes(cluster, newNote.ID, parsedResult.reason, md, contextUser);
|
|
1713
|
+
outcome.consolidated = 1;
|
|
1714
|
+
outcome.newNoteId = newNote.ID;
|
|
1715
|
+
if (this._verbose) {
|
|
1716
|
+
LogStatus(`Memory Manager: Consolidated ${cluster.length} notes into: "${consolidatedNoteData.content.substring(0, 50)}..."`);
|
|
1717
|
+
}
|
|
1718
|
+
stepOutput = {
|
|
1719
|
+
...stepOutput,
|
|
1720
|
+
shouldConsolidate: true,
|
|
1721
|
+
consolidatedNoteId: newNote.ID,
|
|
1722
|
+
sourceNotesArchived: outcome.archived,
|
|
1723
|
+
verificationPassed: outcome.verification?.passed ?? null,
|
|
1724
|
+
entitiesChecked: outcome.verification?.entitiesChecked ?? 0,
|
|
1725
|
+
entitiesMissing: outcome.verification?.entitiesMissing ?? []
|
|
1726
|
+
};
|
|
1727
|
+
return outcome;
|
|
1728
|
+
}
|
|
1729
|
+
finally {
|
|
1730
|
+
// Treat skip (shouldConsolidate: false) as a successful completion rather than a
|
|
1731
|
+
// failure — the phase intentionally chose not to consolidate this cluster.
|
|
1732
|
+
await this.FinalizeRunStep(clusterStep, outcome.consolidated === 1 || stepOutput.shouldConsolidate === false, stepOutput);
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
/**
|
|
1736
|
+
* Prepare the prompt payload for consolidation. Handles drift prevention: if any source
|
|
1737
|
+
* note is at the consolidation count cap, resolve back to original generation-0 sources
|
|
1738
|
+
* to avoid re-summarizing summaries. Returns both the notes used (for verification) and
|
|
1739
|
+
* the prompt data.
|
|
1740
|
+
*/
|
|
1741
|
+
async buildConsolidationPromptData(cluster, contextUser) {
|
|
1742
|
+
const maxConsolidationCount = Math.max(...cluster.map(n => n.ConsolidationCount || 0));
|
|
1743
|
+
let notesForPrompt = cluster;
|
|
1744
|
+
let anchoredMode = false;
|
|
1745
|
+
if (maxConsolidationCount >= CONSOLIDATION_CONFIG.maxConsolidationCount) {
|
|
1746
|
+
const originals = await this.resolveOriginalSources(cluster.map(n => n.ID), contextUser);
|
|
1747
|
+
if (originals.length > 0) {
|
|
1748
|
+
notesForPrompt = originals;
|
|
1749
|
+
if (this._verbose) {
|
|
1750
|
+
LogStatus(`Memory Manager: ConsolidationCount cap reached (${maxConsolidationCount}). Resolved ${cluster.length} notes to ${originals.length} original sources.`);
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
else if (maxConsolidationCount > 0) {
|
|
1755
|
+
anchoredMode = true;
|
|
1756
|
+
}
|
|
1757
|
+
const promptData = {
|
|
1758
|
+
anchoredMode,
|
|
1759
|
+
notesToConsolidate: notesForPrompt.map(n => ({
|
|
1760
|
+
id: n.ID,
|
|
1761
|
+
type: n.Type,
|
|
1762
|
+
content: n.Note,
|
|
1763
|
+
createdAt: n.__mj_CreatedAt,
|
|
1764
|
+
accessCount: n.AccessCount,
|
|
1765
|
+
importanceScore: n.ImportanceScore || null,
|
|
1766
|
+
consolidationCount: n.ConsolidationCount || 0,
|
|
1767
|
+
agentId: n.AgentID,
|
|
1005
1768
|
userId: n.UserID,
|
|
1006
1769
|
companyId: n.CompanyID
|
|
1007
1770
|
}))
|
|
1008
1771
|
};
|
|
1772
|
+
return { notesForPrompt, promptData };
|
|
1773
|
+
}
|
|
1774
|
+
/**
|
|
1775
|
+
* Execute the consolidation prompt and parse the result. Returns null on any failure
|
|
1776
|
+
* so the caller can short-circuit.
|
|
1777
|
+
*/
|
|
1778
|
+
async runConsolidationPrompt(consolidatePrompt, promptData, runner, contextUser) {
|
|
1009
1779
|
const params = new AIPromptParams();
|
|
1010
1780
|
params.prompt = consolidatePrompt;
|
|
1011
1781
|
params.data = promptData;
|
|
1012
1782
|
params.contextUser = contextUser;
|
|
1013
1783
|
params.attemptJSONRepair = true;
|
|
1784
|
+
params.additionalParameters = DETERMINISTIC_PROMPT_PARAMS;
|
|
1014
1785
|
const result = await runner.ExecutePrompt(params);
|
|
1015
1786
|
if (!result.success || !result.result) {
|
|
1016
1787
|
LogError(`Memory Manager: Consolidation prompt failed for cluster: ${result.errorMessage}`);
|
|
1017
|
-
return
|
|
1788
|
+
return null;
|
|
1018
1789
|
}
|
|
1019
|
-
// Parse result if string (some models wrap JSON in ```json fences)
|
|
1020
|
-
let parsedResult = result.result;
|
|
1021
1790
|
if (typeof result.result === 'string') {
|
|
1022
1791
|
const parsed = CleanAndParseJSON(result.result, true);
|
|
1023
1792
|
if (!parsed) {
|
|
1024
1793
|
LogError('Memory Manager: Failed to parse consolidation result');
|
|
1025
|
-
return
|
|
1026
|
-
}
|
|
1027
|
-
parsedResult = parsed;
|
|
1028
|
-
}
|
|
1029
|
-
if (!parsedResult.shouldConsolidate) {
|
|
1030
|
-
if (this._verbose) {
|
|
1031
|
-
LogStatus(`Memory Manager: Skipping cluster consolidation - ${parsedResult.reason}`);
|
|
1794
|
+
return null;
|
|
1032
1795
|
}
|
|
1033
|
-
return
|
|
1796
|
+
return parsed;
|
|
1034
1797
|
}
|
|
1035
|
-
|
|
1036
|
-
|
|
1798
|
+
return result.result;
|
|
1799
|
+
}
|
|
1800
|
+
/**
|
|
1801
|
+
* Build, validate, and save the consolidated note. Returns the saved entity on success,
|
|
1802
|
+
* null on save failure. The LLM-returned Type is validated against the allowed union and
|
|
1803
|
+
* falls back to the template note's Type on hallucination — previously a blind `as` cast
|
|
1804
|
+
* silently wrote invalid values.
|
|
1805
|
+
*/
|
|
1806
|
+
async createConsolidatedNote(cluster, consolidatedNoteData, reason, aiNoteTypeId, md, contextUser) {
|
|
1037
1807
|
const newNote = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
1038
1808
|
const templateNote = cluster[0];
|
|
1809
|
+
let validatedType;
|
|
1810
|
+
if (isValidNoteType(consolidatedNoteData.type)) {
|
|
1811
|
+
validatedType = consolidatedNoteData.type;
|
|
1812
|
+
}
|
|
1813
|
+
else {
|
|
1814
|
+
LogError(`Memory Manager: Consolidation LLM returned invalid note Type "${consolidatedNoteData.type}" — falling back to template note's Type "${templateNote.Type}"`);
|
|
1815
|
+
validatedType = templateNote.Type;
|
|
1816
|
+
}
|
|
1039
1817
|
newNote.AgentID = templateNote.AgentID;
|
|
1040
1818
|
newNote.UserID = templateNote.UserID;
|
|
1041
1819
|
newNote.CompanyID = templateNote.CompanyID;
|
|
1042
1820
|
newNote.AgentNoteTypeID = aiNoteTypeId;
|
|
1043
|
-
newNote.Type =
|
|
1821
|
+
newNote.Type = validatedType;
|
|
1044
1822
|
newNote.Note = consolidatedNoteData.content;
|
|
1045
1823
|
newNote.IsAutoGenerated = true;
|
|
1046
1824
|
newNote.Status = 'Active';
|
|
1047
1825
|
newNote.AccessCount = cluster.reduce((sum, n) => sum + (n.AccessCount || 0), 0);
|
|
1048
|
-
newNote.Comments = `Consolidated from ${cluster.length} notes: ${
|
|
1826
|
+
newNote.Comments = `Consolidated from ${cluster.length} notes: ${reason}`;
|
|
1827
|
+
const maxSourceConsolidationCount = Math.max(...cluster.map(n => n.ConsolidationCount || 0));
|
|
1828
|
+
newNote.ConsolidationCount = maxSourceConsolidationCount + 1;
|
|
1829
|
+
newNote.DerivedFromNoteIDs = JSON.stringify(cluster.map(n => n.ID));
|
|
1830
|
+
newNote.ProtectionTier = 'Standard';
|
|
1049
1831
|
this.applyScopeToConsolidatedNote(newNote, templateNote, consolidatedNoteData.scopeLevel);
|
|
1050
|
-
|
|
1051
|
-
if (!saveResult) {
|
|
1832
|
+
if (!(await newNote.Save())) {
|
|
1052
1833
|
LogError(`Memory Manager: Failed to save consolidated note: ${JSON.stringify(newNote.LatestResult)}`);
|
|
1053
|
-
return
|
|
1834
|
+
return null;
|
|
1054
1835
|
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1836
|
+
return newNote;
|
|
1837
|
+
}
|
|
1838
|
+
/**
|
|
1839
|
+
* Run post-consolidation entity verification and, if entities are missing, append a
|
|
1840
|
+
* warning to the saved note's Comments (best-effort, non-blocking). Returns the
|
|
1841
|
+
* verification result so the caller can aggregate it into the phase-level
|
|
1842
|
+
* observability step.
|
|
1843
|
+
*/
|
|
1844
|
+
async annotateConsolidationVerification(newNote, notesForPrompt, consolidatedContent) {
|
|
1845
|
+
const verification = this.verifyConsolidationOutput(notesForPrompt, consolidatedContent);
|
|
1846
|
+
if (verification.passed || verification.entitiesMissing.length === 0) {
|
|
1847
|
+
return verification;
|
|
1848
|
+
}
|
|
1849
|
+
const missingList = verification.entitiesMissing.slice(0, 10).join(', ');
|
|
1850
|
+
const warning = `[Verification: potentially missing entities: ${missingList}]`;
|
|
1851
|
+
if (this._verbose) {
|
|
1852
|
+
LogStatus(`Memory Manager: Post-consolidation verification warning — ${verification.entitiesMissing.length} entities not found in consolidated output (${verification.entitiesChecked} checked)`);
|
|
1853
|
+
}
|
|
1854
|
+
newNote.Comments = `${newNote.Comments || ''} ${warning}`.trim();
|
|
1855
|
+
await newNote.Save();
|
|
1856
|
+
return verification;
|
|
1857
|
+
}
|
|
1858
|
+
/**
|
|
1859
|
+
* Revoke source notes in parallel, linking each to the consolidated replacement.
|
|
1860
|
+
* Returns the number of notes successfully revoked.
|
|
1861
|
+
*/
|
|
1862
|
+
async revokeSourceNotes(cluster, newNoteId, reason, md, contextUser) {
|
|
1863
|
+
const results = await Promise.all(cluster.map(async (sourceNote) => {
|
|
1059
1864
|
const noteToRevoke = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
1060
|
-
if (await noteToRevoke.Load(sourceNote.ID)) {
|
|
1061
|
-
|
|
1062
|
-
noteToRevoke.Comments = 'Revoked during note consolidation';
|
|
1063
|
-
if (await noteToRevoke.Save()) {
|
|
1064
|
-
return true;
|
|
1065
|
-
}
|
|
1066
|
-
else {
|
|
1067
|
-
LogError(`Memory Manager: Failed to revoke source note ${sourceNote.ID}`);
|
|
1068
|
-
}
|
|
1865
|
+
if (!(await noteToRevoke.Load(sourceNote.ID))) {
|
|
1866
|
+
return false;
|
|
1069
1867
|
}
|
|
1868
|
+
noteToRevoke.Status = 'Revoked';
|
|
1869
|
+
noteToRevoke.ConsolidatedIntoNoteID = newNoteId;
|
|
1870
|
+
noteToRevoke.Comments = `Consolidated into note ${newNoteId}: ${reason}`;
|
|
1871
|
+
if (await noteToRevoke.Save()) {
|
|
1872
|
+
return true;
|
|
1873
|
+
}
|
|
1874
|
+
LogError(`Memory Manager: Failed to revoke source note ${sourceNote.ID}`);
|
|
1070
1875
|
return false;
|
|
1071
1876
|
}));
|
|
1072
|
-
|
|
1073
|
-
if (this._verbose) {
|
|
1074
|
-
LogStatus(`Memory Manager: Consolidated ${cluster.length} notes into: "${consolidatedNoteData.content.substring(0, 50)}..."`);
|
|
1075
|
-
}
|
|
1076
|
-
return { consolidated: 1, archived };
|
|
1877
|
+
return results.filter(Boolean).length;
|
|
1077
1878
|
}
|
|
1078
1879
|
/**
|
|
1079
1880
|
* Apply scope fields to a consolidated note based on the LLM's scope recommendation.
|
|
@@ -1098,6 +1899,577 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
1098
1899
|
newNote.SecondaryScopes = templateNote.SecondaryScopes;
|
|
1099
1900
|
}
|
|
1100
1901
|
}
|
|
1902
|
+
/**
|
|
1903
|
+
* Prune notes that reference entities which no longer exist or are inactive.
|
|
1904
|
+
* Checks AgentID, UserID, CompanyID, and SourceConversationID against current database state.
|
|
1905
|
+
* Immutable-tier notes are never pruned. Caps at 200 notes per cycle.
|
|
1906
|
+
*/
|
|
1907
|
+
async pruneStaleReferences(contextUser) {
|
|
1908
|
+
const maxPerCycle = STALE_PRUNING_CONFIG.maxNotesPerRun;
|
|
1909
|
+
const result = { notesArchived: 0, orphanedAgents: 0, orphanedUsers: 0, orphanedCompanies: 0, orphanedConversations: 0 };
|
|
1910
|
+
const notes = await this.loadPruneCandidateNotes(contextUser, maxPerCycle * 2);
|
|
1911
|
+
if (notes.length === 0) {
|
|
1912
|
+
return result;
|
|
1913
|
+
}
|
|
1914
|
+
const caches = await this.buildReferenceExistenceCache(notes, contextUser);
|
|
1915
|
+
const md = new Metadata();
|
|
1916
|
+
for (const note of notes) {
|
|
1917
|
+
if (result.notesArchived >= maxPerCycle)
|
|
1918
|
+
break;
|
|
1919
|
+
const orphanReasons = this.detectOrphanReasons(note, caches, result);
|
|
1920
|
+
if (orphanReasons.length > 0 && await this.archiveOrphanedNote(note.ID, orphanReasons, md, contextUser)) {
|
|
1921
|
+
result.notesArchived++;
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
return result;
|
|
1925
|
+
}
|
|
1926
|
+
/**
|
|
1927
|
+
* Load auto-generated, non-immutable Active notes that are candidates for orphan pruning.
|
|
1928
|
+
*/
|
|
1929
|
+
async loadPruneCandidateNotes(contextUser, maxRows) {
|
|
1930
|
+
const rv = new RunView();
|
|
1931
|
+
const notesResult = await rv.RunView({
|
|
1932
|
+
EntityName: 'MJ: AI Agent Notes',
|
|
1933
|
+
ExtraFilter: `IsAutoGenerated = 1 AND Status = 'Active' AND ProtectionTier != 'Immutable'`,
|
|
1934
|
+
Fields: ['ID', 'AgentID', 'UserID', 'CompanyID', 'SourceConversationID'],
|
|
1935
|
+
OrderBy: '__mj_CreatedAt ASC',
|
|
1936
|
+
MaxRows: maxRows,
|
|
1937
|
+
ResultType: 'simple'
|
|
1938
|
+
}, contextUser);
|
|
1939
|
+
if (!notesResult.Success || !notesResult.Results?.length) {
|
|
1940
|
+
return [];
|
|
1941
|
+
}
|
|
1942
|
+
return notesResult.Results;
|
|
1943
|
+
}
|
|
1944
|
+
/**
|
|
1945
|
+
* Build in-memory lookup caches of which referenced users/companies/conversations still
|
|
1946
|
+
* exist and which agents are still Active. Runs all three existence lookups in a single
|
|
1947
|
+
* RunViews batch to minimize round trips.
|
|
1948
|
+
*/
|
|
1949
|
+
async buildReferenceExistenceCache(notes, contextUser) {
|
|
1950
|
+
const caches = {
|
|
1951
|
+
activeAgentIds: new Set(AIEngine.Instance.Agents.filter(a => a.Status === 'Active').map(a => a.ID?.toLowerCase())),
|
|
1952
|
+
existingUserIds: new Set(),
|
|
1953
|
+
existingCompanyIds: new Set(),
|
|
1954
|
+
existingConversationIds: new Set()
|
|
1955
|
+
};
|
|
1956
|
+
const { lookupQueries, queryMapping } = this.buildExistenceLookupQueries(notes);
|
|
1957
|
+
if (lookupQueries.length === 0)
|
|
1958
|
+
return caches;
|
|
1959
|
+
const rv = new RunView();
|
|
1960
|
+
const lookupResults = await rv.RunViews(lookupQueries, contextUser);
|
|
1961
|
+
for (let i = 0; i < lookupResults.length; i++) {
|
|
1962
|
+
if (!lookupResults[i].Success)
|
|
1963
|
+
continue;
|
|
1964
|
+
const targetSet = queryMapping[i] === 'users' ? caches.existingUserIds
|
|
1965
|
+
: queryMapping[i] === 'companies' ? caches.existingCompanyIds
|
|
1966
|
+
: caches.existingConversationIds;
|
|
1967
|
+
lookupResults[i].Results.forEach(r => targetSet.add(r.ID?.toLowerCase()));
|
|
1968
|
+
}
|
|
1969
|
+
return caches;
|
|
1970
|
+
}
|
|
1971
|
+
/**
|
|
1972
|
+
* Collect unique referenced IDs from the note batch and assemble a RunViews query list
|
|
1973
|
+
* for existence checks. Returns the query list alongside a parallel mapping array that
|
|
1974
|
+
* tells the caller which target cache each query result belongs in.
|
|
1975
|
+
*/
|
|
1976
|
+
buildExistenceLookupQueries(notes) {
|
|
1977
|
+
const referencedUserIds = new Set();
|
|
1978
|
+
const referencedCompanyIds = new Set();
|
|
1979
|
+
const referencedConversationIds = new Set();
|
|
1980
|
+
for (const note of notes) {
|
|
1981
|
+
if (note.UserID)
|
|
1982
|
+
referencedUserIds.add(note.UserID);
|
|
1983
|
+
if (note.CompanyID)
|
|
1984
|
+
referencedCompanyIds.add(note.CompanyID);
|
|
1985
|
+
if (note.SourceConversationID)
|
|
1986
|
+
referencedConversationIds.add(note.SourceConversationID);
|
|
1987
|
+
}
|
|
1988
|
+
const lookupQueries = [];
|
|
1989
|
+
const queryMapping = [];
|
|
1990
|
+
const push = (entityName, ids, kind) => {
|
|
1991
|
+
if (ids.size === 0)
|
|
1992
|
+
return;
|
|
1993
|
+
lookupQueries.push({ EntityName: entityName, ExtraFilter: `ID IN (${Array.from(ids).map(id => `'${id}'`).join(',')})`, Fields: ['ID'] });
|
|
1994
|
+
queryMapping.push(kind);
|
|
1995
|
+
};
|
|
1996
|
+
push('MJ: Users', referencedUserIds, 'users');
|
|
1997
|
+
push('MJ: Companies', referencedCompanyIds, 'companies');
|
|
1998
|
+
push('MJ: Conversations', referencedConversationIds, 'conversations');
|
|
1999
|
+
return { lookupQueries, queryMapping };
|
|
2000
|
+
}
|
|
2001
|
+
/**
|
|
2002
|
+
* Evaluate a single note's foreign-key references against the existence caches, returning
|
|
2003
|
+
* human-readable reasons for any orphaned fields. Increments the supplied counter as a
|
|
2004
|
+
* side effect so aggregate totals are tracked without a second pass.
|
|
2005
|
+
*/
|
|
2006
|
+
detectOrphanReasons(note, caches, counter) {
|
|
2007
|
+
const reasons = [];
|
|
2008
|
+
if (note.AgentID && !caches.activeAgentIds.has(note.AgentID.toLowerCase())) {
|
|
2009
|
+
reasons.push('referenced agent is inactive or deleted');
|
|
2010
|
+
counter.orphanedAgents++;
|
|
2011
|
+
}
|
|
2012
|
+
if (note.UserID && !caches.existingUserIds.has(note.UserID.toLowerCase())) {
|
|
2013
|
+
reasons.push('referenced user no longer exists');
|
|
2014
|
+
counter.orphanedUsers++;
|
|
2015
|
+
}
|
|
2016
|
+
if (note.CompanyID && !caches.existingCompanyIds.has(note.CompanyID.toLowerCase())) {
|
|
2017
|
+
reasons.push('referenced company no longer exists');
|
|
2018
|
+
counter.orphanedCompanies++;
|
|
2019
|
+
}
|
|
2020
|
+
if (note.SourceConversationID && !caches.existingConversationIds.has(note.SourceConversationID.toLowerCase())) {
|
|
2021
|
+
reasons.push('source conversation no longer exists');
|
|
2022
|
+
counter.orphanedConversations++;
|
|
2023
|
+
}
|
|
2024
|
+
return reasons;
|
|
2025
|
+
}
|
|
2026
|
+
/**
|
|
2027
|
+
* Archive a single orphaned note with an audit-trail Comments entry. Returns true on success.
|
|
2028
|
+
*/
|
|
2029
|
+
async archiveOrphanedNote(noteId, reasons, md, contextUser) {
|
|
2030
|
+
const noteToArchive = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
2031
|
+
if (!(await noteToArchive.Load(noteId))) {
|
|
2032
|
+
return false;
|
|
2033
|
+
}
|
|
2034
|
+
noteToArchive.Status = 'Archived';
|
|
2035
|
+
noteToArchive.Comments = `[Archived by Memory Manager: stale reference pruning - ${reasons.join('; ')} on ${new Date().toISOString()}]`;
|
|
2036
|
+
if (!(await noteToArchive.Save())) {
|
|
2037
|
+
LogError(`Memory Manager: Failed to archive orphaned note ${noteId}`);
|
|
2038
|
+
return false;
|
|
2039
|
+
}
|
|
2040
|
+
return true;
|
|
2041
|
+
}
|
|
2042
|
+
/**
|
|
2043
|
+
* Compute Ebbinghaus-inspired proportional decay factor for a note.
|
|
2044
|
+
* Returns a multiplier (0-1) to apply to the note's ImportanceScore.
|
|
2045
|
+
* Per research: "decay should be proportional (multiply all strengths by a factor <1),
|
|
2046
|
+
* not threshold-based deletion — this preserves relative importance ordering."
|
|
2047
|
+
*/
|
|
2048
|
+
computeDecayFactor(note) {
|
|
2049
|
+
const importanceScore = note.ImportanceScore || 5.0;
|
|
2050
|
+
const normalizedImportance = importanceScore / 10.0;
|
|
2051
|
+
let lambdaEff = DECAY_CONFIG.baseLambda * (1 - normalizedImportance * DECAY_CONFIG.importanceDampening);
|
|
2052
|
+
if (note.ProtectionTier === 'Ephemeral') {
|
|
2053
|
+
lambdaEff *= DECAY_CONFIG.ephemeralDecayMultiplier;
|
|
2054
|
+
}
|
|
2055
|
+
const lastAccess = note.LastAccessedAt || note.__mj_CreatedAt;
|
|
2056
|
+
const daysSinceAccess = (Date.now() - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
|
|
2057
|
+
// Decay factor: how much of the current score to retain.
|
|
2058
|
+
// Higher access count slows decay (well-used notes persist longer), but the
|
|
2059
|
+
// multiplier is CAPPED — without a cap, a note with AccessCount=1000 would get
|
|
2060
|
+
// accessRetention=201 and the `Math.min(1.0, ...)` clamp below would force the
|
|
2061
|
+
// decay factor to 1.0 forever, making frequently-accessed notes effectively
|
|
2062
|
+
// immortal. Capping at ~1.5 means access count can meaningfully slow decay but
|
|
2063
|
+
// cannot indefinitely prevent it.
|
|
2064
|
+
const rawAccessRetention = 1 + (note.AccessCount || 0) * DECAY_CONFIG.accessBoostFactor;
|
|
2065
|
+
const accessRetention = Math.min(DECAY_CONFIG.accessRetentionCap, rawAccessRetention);
|
|
2066
|
+
return Math.min(1.0, Math.exp(-lambdaEff * daysSinceAccess) * accessRetention);
|
|
2067
|
+
}
|
|
2068
|
+
/**
|
|
2069
|
+
* Decay-based archival replacing fixed 90/180-day retention windows.
|
|
2070
|
+
* Uses Ebbinghaus-inspired decay function weighted by ImportanceScore.
|
|
2071
|
+
* Respects protection tiers: Immutable=never, Protected=365d extended, Ephemeral=2x decay.
|
|
2072
|
+
* Handles both notes and examples. Respects per-agent AutoArchiveEnabled.
|
|
2073
|
+
*/
|
|
2074
|
+
async decayBasedArchival(contextUser) {
|
|
2075
|
+
const archiveEnabledAgentIds = new Set(AIEngine.Instance.Agents
|
|
2076
|
+
.filter(a => a.AutoArchiveEnabled !== false)
|
|
2077
|
+
.map(a => a.ID?.toLowerCase()));
|
|
2078
|
+
const [expiredResult, decayResult] = await Promise.all([
|
|
2079
|
+
this.archiveExpiredItems(contextUser),
|
|
2080
|
+
this.applyProportionalDecay(contextUser, archiveEnabledAgentIds)
|
|
2081
|
+
]);
|
|
2082
|
+
return {
|
|
2083
|
+
notesExpired: expiredResult.notesExpired,
|
|
2084
|
+
examplesExpired: expiredResult.examplesExpired,
|
|
2085
|
+
notesArchived: decayResult.notesArchived,
|
|
2086
|
+
examplesArchived: decayResult.examplesArchived,
|
|
2087
|
+
decayScoreDistribution: this.summarizeDecayFactors(decayResult.stats.decayFactors),
|
|
2088
|
+
protectedPreserved: decayResult.stats.protectedPreserved,
|
|
2089
|
+
ephemeralAccelerated: decayResult.stats.ephemeralAccelerated
|
|
2090
|
+
};
|
|
2091
|
+
}
|
|
2092
|
+
/**
|
|
2093
|
+
* Summarize raw decay factors (values in [0,1]) into a min/max/mean triple for
|
|
2094
|
+
* the observability step. Returns null when no factors were computed (empty
|
|
2095
|
+
* cohort or all notes skipped via tier/config).
|
|
2096
|
+
*/
|
|
2097
|
+
summarizeDecayFactors(factors) {
|
|
2098
|
+
if (factors.length === 0)
|
|
2099
|
+
return null;
|
|
2100
|
+
let min = factors[0];
|
|
2101
|
+
let max = factors[0];
|
|
2102
|
+
let sum = 0;
|
|
2103
|
+
for (const f of factors) {
|
|
2104
|
+
if (f < min)
|
|
2105
|
+
min = f;
|
|
2106
|
+
if (f > max)
|
|
2107
|
+
max = f;
|
|
2108
|
+
sum += f;
|
|
2109
|
+
}
|
|
2110
|
+
return {
|
|
2111
|
+
min: Math.round(min * 1000) / 1000,
|
|
2112
|
+
max: Math.round(max * 1000) / 1000,
|
|
2113
|
+
mean: Math.round((sum / factors.length) * 1000) / 1000
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
/** Archive notes and examples past their explicit ExpiresAt timestamp, regardless of decay score. */
|
|
2117
|
+
async archiveExpiredItems(contextUser) {
|
|
2118
|
+
const result = { notesExpired: 0, examplesExpired: 0 };
|
|
2119
|
+
const md = new Metadata();
|
|
2120
|
+
const nowISO = new Date().toISOString();
|
|
2121
|
+
const [expiredNotes, expiredExamples] = await this.loadExpiredItems(contextUser, nowISO);
|
|
2122
|
+
for (const note of expiredNotes) {
|
|
2123
|
+
if ((note.ProtectionTier || 'Standard') === 'Immutable')
|
|
2124
|
+
continue;
|
|
2125
|
+
if (await this.archiveExpiredNote(note, md, contextUser))
|
|
2126
|
+
result.notesExpired++;
|
|
2127
|
+
}
|
|
2128
|
+
for (const example of expiredExamples) {
|
|
2129
|
+
if (await this.archiveExpiredExample(example, md, contextUser))
|
|
2130
|
+
result.examplesExpired++;
|
|
2131
|
+
}
|
|
2132
|
+
return result;
|
|
2133
|
+
}
|
|
2134
|
+
async loadExpiredItems(contextUser, nowISO) {
|
|
2135
|
+
const rv = new RunView();
|
|
2136
|
+
const [notesResult, examplesResult] = await Promise.all([
|
|
2137
|
+
rv.RunView({
|
|
2138
|
+
EntityName: 'MJ: AI Agent Notes',
|
|
2139
|
+
ExtraFilter: `Status = 'Active' AND ExpiresAt IS NOT NULL AND ExpiresAt < '${nowISO}'`,
|
|
2140
|
+
Fields: ['ID', 'ExpiresAt', 'ProtectionTier'],
|
|
2141
|
+
MaxRows: 200,
|
|
2142
|
+
}, contextUser),
|
|
2143
|
+
rv.RunView({
|
|
2144
|
+
EntityName: 'MJ: AI Agent Examples',
|
|
2145
|
+
ExtraFilter: `Status = 'Active' AND ExpiresAt IS NOT NULL AND ExpiresAt < '${nowISO}'`,
|
|
2146
|
+
Fields: ['ID', 'ExpiresAt'],
|
|
2147
|
+
MaxRows: 200,
|
|
2148
|
+
}, contextUser)
|
|
2149
|
+
]);
|
|
2150
|
+
return [
|
|
2151
|
+
notesResult.Success && notesResult.Results ? notesResult.Results : [],
|
|
2152
|
+
examplesResult.Success && examplesResult.Results ? examplesResult.Results : []
|
|
2153
|
+
];
|
|
2154
|
+
}
|
|
2155
|
+
async archiveExpiredNote(note, md, contextUser) {
|
|
2156
|
+
const noteToArchive = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
2157
|
+
if (!(await noteToArchive.Load(note.ID)))
|
|
2158
|
+
return false;
|
|
2159
|
+
noteToArchive.Status = 'Archived';
|
|
2160
|
+
noteToArchive.Comments = `${noteToArchive.Comments || ''} [Archived by Memory Manager: expired on ${note.ExpiresAt}]`.trim();
|
|
2161
|
+
return await noteToArchive.Save();
|
|
2162
|
+
}
|
|
2163
|
+
async archiveExpiredExample(example, md, contextUser) {
|
|
2164
|
+
const exampleToArchive = await md.GetEntityObject('MJ: AI Agent Examples', contextUser);
|
|
2165
|
+
if (!(await exampleToArchive.Load(example.ID)))
|
|
2166
|
+
return false;
|
|
2167
|
+
exampleToArchive.Status = 'Archived';
|
|
2168
|
+
exampleToArchive.Comments = `${exampleToArchive.Comments || ''} [Archived by Memory Manager: expired on ${example.ExpiresAt}]`.trim();
|
|
2169
|
+
return await exampleToArchive.Save();
|
|
2170
|
+
}
|
|
2171
|
+
/**
|
|
2172
|
+
* Apply proportional decay to ImportanceScores, archiving only when score reaches the floor.
|
|
2173
|
+
* Per research: "decay should be proportional (multiply all strengths by a factor <1),
|
|
2174
|
+
* not threshold-based deletion — this preserves relative importance ordering while reducing overall load."
|
|
2175
|
+
* Notes without ImportanceScore (not yet scored) are skipped entirely.
|
|
2176
|
+
*
|
|
2177
|
+
* Threads a stats accumulator through decayOneNote/decayOneExample to collect
|
|
2178
|
+
* decay-factor distribution and tier-specific counts for observability.
|
|
2179
|
+
*/
|
|
2180
|
+
async applyProportionalDecay(contextUser, archiveEnabledAgentIds) {
|
|
2181
|
+
const result = { notesDecayed: 0, notesArchived: 0, examplesDecayed: 0, examplesArchived: 0 };
|
|
2182
|
+
const stats = { decayFactors: [], protectedPreserved: 0, ephemeralAccelerated: 0 };
|
|
2183
|
+
const md = new Metadata();
|
|
2184
|
+
const [activeNotes, activeExamples] = await this.loadDecayCandidates(contextUser);
|
|
2185
|
+
for (const note of activeNotes) {
|
|
2186
|
+
await this.decayOneNote(note, archiveEnabledAgentIds, md, contextUser, result, stats);
|
|
2187
|
+
}
|
|
2188
|
+
for (const example of activeExamples) {
|
|
2189
|
+
await this.decayOneExample(example, archiveEnabledAgentIds, md, contextUser, result, stats);
|
|
2190
|
+
}
|
|
2191
|
+
return { ...result, stats };
|
|
2192
|
+
}
|
|
2193
|
+
/**
|
|
2194
|
+
* Fetch notes and examples in parallel — they're independent.
|
|
2195
|
+
*/
|
|
2196
|
+
async loadDecayCandidates(contextUser) {
|
|
2197
|
+
const rv = new RunView();
|
|
2198
|
+
const [activeNotesResult, activeExamplesResult] = await Promise.all([
|
|
2199
|
+
rv.RunView({
|
|
2200
|
+
EntityName: 'MJ: AI Agent Notes',
|
|
2201
|
+
ExtraFilter: `IsAutoGenerated = 1 AND Status = 'Active'`,
|
|
2202
|
+
Fields: ['ID', 'ImportanceScore', 'ProtectionTier', 'LastAccessedAt', '__mj_CreatedAt', 'AccessCount', 'AgentID'],
|
|
2203
|
+
MaxRows: 1000,
|
|
2204
|
+
}, contextUser),
|
|
2205
|
+
rv.RunView({
|
|
2206
|
+
EntityName: 'MJ: AI Agent Examples',
|
|
2207
|
+
ExtraFilter: `IsAutoGenerated = 1 AND Status = 'Active' AND SuccessScore < 50`,
|
|
2208
|
+
Fields: ['ID', 'SuccessScore', 'LastAccessedAt', '__mj_CreatedAt', 'AccessCount', 'AgentID'],
|
|
2209
|
+
MaxRows: 500,
|
|
2210
|
+
}, contextUser)
|
|
2211
|
+
]);
|
|
2212
|
+
return [
|
|
2213
|
+
activeNotesResult.Success && activeNotesResult.Results ? activeNotesResult.Results : [],
|
|
2214
|
+
activeExamplesResult.Success && activeExamplesResult.Results ? activeExamplesResult.Results : []
|
|
2215
|
+
];
|
|
2216
|
+
}
|
|
2217
|
+
/**
|
|
2218
|
+
* Apply proportional decay to a single note. Archives if the decayed score drops below
|
|
2219
|
+
* the archival floor, otherwise writes the reduced score (skipping when the factor is
|
|
2220
|
+
* effectively 1.0 to avoid unnecessary saves).
|
|
2221
|
+
*/
|
|
2222
|
+
async decayOneNote(note, archiveEnabledAgentIds, md, contextUser, result, stats) {
|
|
2223
|
+
if (note.AgentID && !archiveEnabledAgentIds.has(note.AgentID.toLowerCase()))
|
|
2224
|
+
return;
|
|
2225
|
+
if (note.ImportanceScore === null)
|
|
2226
|
+
return;
|
|
2227
|
+
const tier = note.ProtectionTier || 'Standard';
|
|
2228
|
+
if (tier === 'Immutable')
|
|
2229
|
+
return;
|
|
2230
|
+
if (tier === 'Protected') {
|
|
2231
|
+
const lastAccess = note.LastAccessedAt || note.__mj_CreatedAt;
|
|
2232
|
+
const daysSinceAccess = (Date.now() - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
|
|
2233
|
+
if (daysSinceAccess < DECAY_CONFIG.protectedInactivityDays) {
|
|
2234
|
+
// Protected note within the extended-retention window — evaluated but preserved.
|
|
2235
|
+
stats.protectedPreserved++;
|
|
2236
|
+
return;
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
if (tier === 'Ephemeral') {
|
|
2240
|
+
stats.ephemeralAccelerated++;
|
|
2241
|
+
}
|
|
2242
|
+
const decayFactor = this.computeDecayFactor(note);
|
|
2243
|
+
stats.decayFactors.push(decayFactor);
|
|
2244
|
+
const newScore = Math.round(note.ImportanceScore * decayFactor * 100) / 100;
|
|
2245
|
+
const noteToUpdate = await md.GetEntityObject('MJ: AI Agent Notes', contextUser);
|
|
2246
|
+
if (!(await noteToUpdate.Load(note.ID)))
|
|
2247
|
+
return;
|
|
2248
|
+
if (newScore < DECAY_CONFIG.archivalFloor) {
|
|
2249
|
+
noteToUpdate.Status = 'Archived';
|
|
2250
|
+
noteToUpdate.ImportanceScore = newScore;
|
|
2251
|
+
noteToUpdate.Comments = `${noteToUpdate.Comments || ''} [Archived by Memory Manager: ImportanceScore decayed to ${newScore.toFixed(2)}, below floor ${DECAY_CONFIG.archivalFloor}]`.trim();
|
|
2252
|
+
if (await noteToUpdate.Save())
|
|
2253
|
+
result.notesArchived++;
|
|
2254
|
+
}
|
|
2255
|
+
else if (decayFactor < 0.99) {
|
|
2256
|
+
noteToUpdate.ImportanceScore = newScore;
|
|
2257
|
+
if (await noteToUpdate.Save())
|
|
2258
|
+
result.notesDecayed++;
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
2261
|
+
/**
|
|
2262
|
+
* Apply proportional decay to a single example, using SuccessScore as a proxy importance.
|
|
2263
|
+
* Archives if the effective score drops below the archival floor.
|
|
2264
|
+
*/
|
|
2265
|
+
async decayOneExample(example, archiveEnabledAgentIds, md, contextUser, result, stats) {
|
|
2266
|
+
if (example.AgentID && !archiveEnabledAgentIds.has(example.AgentID.toLowerCase()))
|
|
2267
|
+
return;
|
|
2268
|
+
const normalizedImportance = (example.SuccessScore || 25) / 100.0;
|
|
2269
|
+
const lambdaEff = DECAY_CONFIG.baseLambda * (1 - normalizedImportance * DECAY_CONFIG.importanceDampening);
|
|
2270
|
+
const lastAccess = example.LastAccessedAt || example.__mj_CreatedAt;
|
|
2271
|
+
const daysSinceAccess = (Date.now() - new Date(lastAccess).getTime()) / (1000 * 60 * 60 * 24);
|
|
2272
|
+
const decayFactor = Math.min(1.0, Math.exp(-lambdaEff * daysSinceAccess) * (1 + (example.AccessCount || 0) * DECAY_CONFIG.accessBoostFactor));
|
|
2273
|
+
stats.decayFactors.push(decayFactor);
|
|
2274
|
+
const effectiveScore = normalizedImportance * 10.0 * decayFactor;
|
|
2275
|
+
if (effectiveScore >= DECAY_CONFIG.archivalFloor)
|
|
2276
|
+
return;
|
|
2277
|
+
const exampleToArchive = await md.GetEntityObject('MJ: AI Agent Examples', contextUser);
|
|
2278
|
+
if (!(await exampleToArchive.Load(example.ID)))
|
|
2279
|
+
return;
|
|
2280
|
+
exampleToArchive.Status = 'Archived';
|
|
2281
|
+
exampleToArchive.Comments = `${exampleToArchive.Comments || ''} [Archived by Memory Manager: effective score ${effectiveScore.toFixed(2)} below floor]`.trim();
|
|
2282
|
+
if (await exampleToArchive.Save())
|
|
2283
|
+
result.examplesArchived++;
|
|
2284
|
+
}
|
|
2285
|
+
/**
|
|
2286
|
+
* Run all 5 maintenance phases in order: importance scoring, consolidation,
|
|
2287
|
+
* contradiction detection, stale-reference pruning, and decay-based archival.
|
|
2288
|
+
* Each phase is wrapped in try/catch so a failure in one does not abort the rest.
|
|
2289
|
+
*
|
|
2290
|
+
* `triggerType` flows through to the parent consolidation run step so observability
|
|
2291
|
+
* records *why* the maintenance cycle fired this run.
|
|
2292
|
+
*/
|
|
2293
|
+
async runMaintenancePhases(contextUser, forceMaintenance, triggerType) {
|
|
2294
|
+
const r = {
|
|
2295
|
+
consolidatedCount: 0, consolidationArchived: 0,
|
|
2296
|
+
contradictionsFound: 0, contradictionsResolved: 0, contradictionsFlagged: 0,
|
|
2297
|
+
staleNotesArchived: 0, decayNotesArchived: 0, decayExamplesArchived: 0,
|
|
2298
|
+
importanceScored: 0, tierPromotions: 0
|
|
2299
|
+
};
|
|
2300
|
+
LogStatus(`Memory Manager: Running maintenance phases${forceMaintenance ? ' (forced via params.data.forceMaintenance)' : ''}...`);
|
|
2301
|
+
await this.runImportancePhase(r, contextUser);
|
|
2302
|
+
const consolidatedNoteIds = await this.runConsolidationPhase(r, contextUser, triggerType);
|
|
2303
|
+
await this.runContradictionPhase(r, contextUser, consolidatedNoteIds);
|
|
2304
|
+
await this.runPruneAndDecayPhases(r, contextUser);
|
|
2305
|
+
return r;
|
|
2306
|
+
}
|
|
2307
|
+
async runImportancePhase(r, contextUser) {
|
|
2308
|
+
const importanceStep = await this.CreateRunStep('Decision', 'Compute Importance Scores', {
|
|
2309
|
+
activeNoteCount: AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active').length
|
|
2310
|
+
});
|
|
2311
|
+
try {
|
|
2312
|
+
const scoringResult = await this.computeImportanceScores(contextUser);
|
|
2313
|
+
r.importanceScored = scoringResult.notesScored;
|
|
2314
|
+
r.tierPromotions = scoringResult.tierPromotions;
|
|
2315
|
+
if (this._verbose)
|
|
2316
|
+
LogStatus(`Memory Manager: Scored ${r.importanceScored} notes, ${r.tierPromotions} tier promotions`);
|
|
2317
|
+
await this.FinalizeRunStep(importanceStep, true, {
|
|
2318
|
+
notesScored: r.importanceScored,
|
|
2319
|
+
tierPromotions: r.tierPromotions,
|
|
2320
|
+
scoreDistribution: scoringResult.scoreDistribution
|
|
2321
|
+
});
|
|
2322
|
+
}
|
|
2323
|
+
catch (error) {
|
|
2324
|
+
LogError('Memory Manager: Importance scoring failed, continuing with other phases:', error);
|
|
2325
|
+
await this.FinalizeRunStep(importanceStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
async runConsolidationPhase(r, contextUser, triggerType) {
|
|
2329
|
+
const consolidatedNoteIds = new Set();
|
|
2330
|
+
const consolidationStep = await this.CreateRunStep('Decision', 'Consolidate Related Notes', {
|
|
2331
|
+
frequency: CONSOLIDATION_CONFIG.frequency,
|
|
2332
|
+
triggerType,
|
|
2333
|
+
activeNoteCount: AIEngine.Instance.AgentNotes.filter(n => n.Status === 'Active' && n.IsAutoGenerated).length
|
|
2334
|
+
});
|
|
2335
|
+
let consolidationResult = null;
|
|
2336
|
+
try {
|
|
2337
|
+
consolidationResult = await this.consolidateRelatedNotes(null, contextUser);
|
|
2338
|
+
r.consolidatedCount = consolidationResult.consolidated;
|
|
2339
|
+
r.consolidationArchived = consolidationResult.archived;
|
|
2340
|
+
for (const id of consolidationResult.newNoteIds)
|
|
2341
|
+
consolidatedNoteIds.add(id);
|
|
2342
|
+
if (r.consolidatedCount > 0 && this._verbose) {
|
|
2343
|
+
LogStatus(`Memory Manager: Consolidated ${r.consolidatedCount} clusters, archived ${r.consolidationArchived} source notes`);
|
|
2344
|
+
}
|
|
2345
|
+
await this.FinalizeRunStep(consolidationStep, true, {
|
|
2346
|
+
consolidatedClusterCount: r.consolidatedCount,
|
|
2347
|
+
sourceNotesArchived: r.consolidationArchived,
|
|
2348
|
+
newConsolidatedNoteIds: Array.from(consolidatedNoteIds)
|
|
2349
|
+
});
|
|
2350
|
+
}
|
|
2351
|
+
catch (error) {
|
|
2352
|
+
LogError('Memory Manager: Consolidation failed, continuing with other phases:', error);
|
|
2353
|
+
await this.FinalizeRunStep(consolidationStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
|
|
2354
|
+
}
|
|
2355
|
+
// Emit the phase-level verification run step (spec Task 8c). Runs whether or not
|
|
2356
|
+
// any clusters were consolidated so "no items" runs still produce a visible step.
|
|
2357
|
+
await this.emitVerificationRunStep(consolidationResult?.verifications ?? []);
|
|
2358
|
+
return consolidatedNoteIds;
|
|
2359
|
+
}
|
|
2360
|
+
/** Aggregate per-cluster verification results into a single phase-level run step. */
|
|
2361
|
+
async emitVerificationRunStep(verifications) {
|
|
2362
|
+
const verificationStep = await this.CreateRunStep('Validation', 'Verify Consolidation Output', {
|
|
2363
|
+
clustersVerified: verifications.length
|
|
2364
|
+
});
|
|
2365
|
+
const verificationsFlagged = verifications.filter(v => !v.passed).length;
|
|
2366
|
+
const verificationsPassed = verifications.filter(v => v.passed).length;
|
|
2367
|
+
const entitiesChecked = verifications.reduce((sum, v) => sum + v.entitiesChecked, 0);
|
|
2368
|
+
const entitiesMissing = verifications.flatMap(v => v.entitiesMissing);
|
|
2369
|
+
await this.FinalizeRunStep(verificationStep, true, {
|
|
2370
|
+
clustersVerified: verifications.length,
|
|
2371
|
+
entitiesChecked,
|
|
2372
|
+
entitiesMissing,
|
|
2373
|
+
verificationsPassed,
|
|
2374
|
+
verificationsFlagged
|
|
2375
|
+
});
|
|
2376
|
+
}
|
|
2377
|
+
async runContradictionPhase(r, contextUser, consolidatedNoteIds) {
|
|
2378
|
+
const contradictionStep = await this.CreateRunStep('Decision', 'Detect Note Contradictions', {
|
|
2379
|
+
consolidatedNoteIdsCount: consolidatedNoteIds.size
|
|
2380
|
+
});
|
|
2381
|
+
try {
|
|
2382
|
+
const contradictionResult = await this.detectAndResolveContradictions(contextUser, consolidatedNoteIds);
|
|
2383
|
+
r.contradictionsFound = contradictionResult.contradictionsFound;
|
|
2384
|
+
r.contradictionsResolved = contradictionResult.resolved;
|
|
2385
|
+
r.contradictionsFlagged = contradictionResult.flagged;
|
|
2386
|
+
if (r.contradictionsFound > 0 && this._verbose) {
|
|
2387
|
+
LogStatus(`Memory Manager: Found ${r.contradictionsFound} contradictions — resolved ${r.contradictionsResolved}, flagged ${r.contradictionsFlagged}`);
|
|
2388
|
+
}
|
|
2389
|
+
await this.FinalizeRunStep(contradictionStep, true, {
|
|
2390
|
+
pairsAnalyzed: contradictionResult.pairsAnalyzed,
|
|
2391
|
+
contradictionsFound: r.contradictionsFound,
|
|
2392
|
+
contradictionsResolved: r.contradictionsResolved,
|
|
2393
|
+
contradictionsFlagged: r.contradictionsFlagged,
|
|
2394
|
+
entityTriplesExtracted: contradictionResult.entityTriplesExtracted
|
|
2395
|
+
});
|
|
2396
|
+
}
|
|
2397
|
+
catch (error) {
|
|
2398
|
+
LogError('Memory Manager: Contradiction detection failed, continuing with other phases:', error);
|
|
2399
|
+
await this.FinalizeRunStep(contradictionStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
|
|
2400
|
+
}
|
|
2401
|
+
}
|
|
2402
|
+
/**
|
|
2403
|
+
* Prune and decay are independent — run in parallel via Promise.allSettled so one failure
|
|
2404
|
+
* doesn't block the other.
|
|
2405
|
+
*/
|
|
2406
|
+
async runPruneAndDecayPhases(r, contextUser) {
|
|
2407
|
+
const pruneStep = await this.CreateRunStep('Decision', 'Prune Stale References', {});
|
|
2408
|
+
const decayStep = await this.CreateRunStep('Decision', 'Decay-Based Archival', {});
|
|
2409
|
+
await Promise.allSettled([
|
|
2410
|
+
this.runPrunePhase(r, contextUser, pruneStep),
|
|
2411
|
+
this.runDecayPhase(r, contextUser, decayStep),
|
|
2412
|
+
]);
|
|
2413
|
+
}
|
|
2414
|
+
async runPrunePhase(r, contextUser, pruneStep) {
|
|
2415
|
+
try {
|
|
2416
|
+
const pruneResult = await this.pruneStaleReferences(contextUser);
|
|
2417
|
+
r.staleNotesArchived = pruneResult.notesArchived;
|
|
2418
|
+
if (r.staleNotesArchived > 0 && this._verbose) {
|
|
2419
|
+
LogStatus(`Memory Manager: Pruned ${r.staleNotesArchived} orphaned notes`);
|
|
2420
|
+
}
|
|
2421
|
+
await this.FinalizeRunStep(pruneStep, true, {
|
|
2422
|
+
notesArchived: r.staleNotesArchived,
|
|
2423
|
+
orphanedAgents: pruneResult.orphanedAgents,
|
|
2424
|
+
orphanedUsers: pruneResult.orphanedUsers,
|
|
2425
|
+
orphanedCompanies: pruneResult.orphanedCompanies,
|
|
2426
|
+
orphanedConversations: pruneResult.orphanedConversations
|
|
2427
|
+
});
|
|
2428
|
+
}
|
|
2429
|
+
catch (error) {
|
|
2430
|
+
LogError('Memory Manager: Stale reference pruning failed:', error);
|
|
2431
|
+
await this.FinalizeRunStep(pruneStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
|
|
2432
|
+
}
|
|
2433
|
+
}
|
|
2434
|
+
async runDecayPhase(r, contextUser, decayStep) {
|
|
2435
|
+
try {
|
|
2436
|
+
const decayResult = await this.decayBasedArchival(contextUser);
|
|
2437
|
+
r.decayNotesArchived = decayResult.notesArchived;
|
|
2438
|
+
r.decayExamplesArchived = decayResult.examplesArchived;
|
|
2439
|
+
const totalDecayArchived = decayResult.notesArchived + decayResult.examplesArchived + decayResult.notesExpired + decayResult.examplesExpired;
|
|
2440
|
+
if (totalDecayArchived > 0 && this._verbose) {
|
|
2441
|
+
LogStatus(`Memory Manager: Decay archival — ${decayResult.notesArchived} notes, ${decayResult.examplesArchived} examples, ${decayResult.notesExpired} expired notes, ${decayResult.examplesExpired} expired examples`);
|
|
2442
|
+
}
|
|
2443
|
+
await this.FinalizeRunStep(decayStep, true, {
|
|
2444
|
+
notesArchived: decayResult.notesArchived,
|
|
2445
|
+
examplesArchived: decayResult.examplesArchived,
|
|
2446
|
+
notesExpired: decayResult.notesExpired,
|
|
2447
|
+
examplesExpired: decayResult.examplesExpired,
|
|
2448
|
+
decayScoreDistribution: decayResult.decayScoreDistribution,
|
|
2449
|
+
protectedPreserved: decayResult.protectedPreserved,
|
|
2450
|
+
ephemeralAccelerated: decayResult.ephemeralAccelerated
|
|
2451
|
+
});
|
|
2452
|
+
}
|
|
2453
|
+
catch (error) {
|
|
2454
|
+
LogError('Memory Manager: Decay-based archival failed:', error);
|
|
2455
|
+
await this.FinalizeRunStep(decayStep, false, undefined, undefined, error instanceof Error ? error.message : String(error));
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
/** Build a human-readable summary string from maintenance phase results. */
|
|
2459
|
+
buildMaintenanceSummary(m) {
|
|
2460
|
+
const parts = [];
|
|
2461
|
+
if (m.consolidatedCount > 0)
|
|
2462
|
+
parts.push(`consolidated ${m.consolidatedCount} clusters`);
|
|
2463
|
+
if (m.contradictionsResolved > 0)
|
|
2464
|
+
parts.push(`resolved ${m.contradictionsResolved} contradictions`);
|
|
2465
|
+
if (m.contradictionsFlagged > 0)
|
|
2466
|
+
parts.push(`flagged ${m.contradictionsFlagged} high-importance contradictions`);
|
|
2467
|
+
if (m.staleNotesArchived > 0)
|
|
2468
|
+
parts.push(`pruned ${m.staleNotesArchived} stale notes`);
|
|
2469
|
+
if (m.decayNotesArchived + m.decayExamplesArchived > 0)
|
|
2470
|
+
parts.push(`decay-archived ${m.decayNotesArchived} notes + ${m.decayExamplesArchived} examples`);
|
|
2471
|
+
return parts.length > 0 ? ` Maintenance: ${parts.join(', ')}.` : '';
|
|
2472
|
+
}
|
|
1101
2473
|
/**
|
|
1102
2474
|
* Create example records from extracted data.
|
|
1103
2475
|
* Inherits scope from source agent run and applies scopeLevel to determine scope specificity.
|
|
@@ -1150,7 +2522,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
1150
2522
|
example.Status = 'Active'; // Auto-approve high-confidence examples
|
|
1151
2523
|
example.SourceConversationID = extracted.sourceConversationId || null;
|
|
1152
2524
|
// Only use if it's a valid UUID
|
|
1153
|
-
example.SourceConversationDetailID =
|
|
2525
|
+
example.SourceConversationDetailID = MemoryManagerAgent_1.isValidUUID(extracted.sourceConversationDetailId) ? extracted.sourceConversationDetailId : null;
|
|
1154
2526
|
example.SourceAIAgentRunID = extracted.sourceAgentRunId || null;
|
|
1155
2527
|
// Apply scope from source agent run based on scopeLevel hint
|
|
1156
2528
|
if (sourceRun && sourceRun.PrimaryScopeEntityID) {
|
|
@@ -1201,7 +2573,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
1201
2573
|
async executeAgentInternal(params, config) {
|
|
1202
2574
|
try {
|
|
1203
2575
|
// Use verbose flag from agent execution params or data payload (UI passes it via data)
|
|
1204
|
-
this._verbose = params.verbose
|
|
2576
|
+
this._verbose = params.verbose || params.data?.verbose || false;
|
|
1205
2577
|
// Initialize observability state for this run
|
|
1206
2578
|
this._agentRunID = this.AgentRun?.ID || null;
|
|
1207
2579
|
this._stepCounter = 0;
|
|
@@ -1252,112 +2624,90 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
1252
2624
|
if (this._verbose) {
|
|
1253
2625
|
LogStatus(`Memory Manager: Found ${agentRuns.length} high-value agent runs`);
|
|
1254
2626
|
}
|
|
1255
|
-
|
|
1256
|
-
if (this._verbose) {
|
|
1257
|
-
LogStatus('Memory Manager: No data to process');
|
|
1258
|
-
}
|
|
1259
|
-
const finalStep = {
|
|
1260
|
-
terminate: true,
|
|
1261
|
-
step: 'Success',
|
|
1262
|
-
message: 'No conversations with new activity or agent runs to process'
|
|
1263
|
-
};
|
|
1264
|
-
return { finalStep, stepCount: 1 };
|
|
1265
|
-
}
|
|
2627
|
+
const hasNewData = conversations.length > 0 || agentRuns.length > 0;
|
|
1266
2628
|
// Initialization guard: ensure vector services exist for semantic dedup.
|
|
1267
2629
|
// If no notes existed at server startup, _noteVectorService is null and
|
|
1268
2630
|
// AddOrUpdateSingleNoteEmbedding will throw when saving new notes.
|
|
1269
2631
|
// The two post-creation/post-consolidation Config(true) calls were removed
|
|
1270
2632
|
// because entity save hooks now update vectors incrementally.
|
|
1271
2633
|
await AIEngine.Instance.Config(true, params.contextUser);
|
|
1272
|
-
// Extract notes
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
conversationContextForExamples
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
if (
|
|
1304
|
-
|
|
1305
|
-
if (
|
|
1306
|
-
example.userId
|
|
1307
|
-
|
|
1308
|
-
|
|
2634
|
+
// Extract notes and examples only when there's new data to process
|
|
2635
|
+
let notesCreated = 0;
|
|
2636
|
+
let examplesCreated = 0;
|
|
2637
|
+
if (hasNewData) {
|
|
2638
|
+
const extractedNotes = await this.ExtractNotesFromConversations(conversations, params.contextUser);
|
|
2639
|
+
if (this._verbose) {
|
|
2640
|
+
LogStatus(`Memory Manager: Extracted ${extractedNotes.length} potential notes`);
|
|
2641
|
+
}
|
|
2642
|
+
// Convert conversations back to flat details for example extraction.
|
|
2643
|
+
// ExtractExamples accepts a ConversationDetailProjection, so we only
|
|
2644
|
+
// need the fields it reads — no entity cast.
|
|
2645
|
+
const conversationDetails = conversations.flatMap(conv => conv.messages.map(msg => ({
|
|
2646
|
+
ID: msg.id,
|
|
2647
|
+
ConversationID: conv.conversationId,
|
|
2648
|
+
Role: msg.role,
|
|
2649
|
+
Message: msg.message,
|
|
2650
|
+
__mj_CreatedAt: msg.createdAt
|
|
2651
|
+
})));
|
|
2652
|
+
const extractedExamples = await this.ExtractExamples(conversationDetails, params.contextUser);
|
|
2653
|
+
if (this._verbose) {
|
|
2654
|
+
LogStatus(`Memory Manager: Extracted ${extractedExamples.length} potential examples`);
|
|
2655
|
+
}
|
|
2656
|
+
// Enrich examples with userId and agentRunId from conversation context
|
|
2657
|
+
const conversationContextForExamples = new Map();
|
|
2658
|
+
for (const conv of conversations) {
|
|
2659
|
+
conversationContextForExamples.set(conv.conversationId, {
|
|
2660
|
+
userId: conv.userId || null,
|
|
2661
|
+
agentRunId: conv.agentRunId || null
|
|
2662
|
+
});
|
|
2663
|
+
}
|
|
2664
|
+
for (const example of extractedExamples) {
|
|
2665
|
+
if (example.sourceConversationId) {
|
|
2666
|
+
const ctx = conversationContextForExamples.get(example.sourceConversationId);
|
|
2667
|
+
if (ctx) {
|
|
2668
|
+
if (!example.userId && ctx.userId) {
|
|
2669
|
+
example.userId = ctx.userId;
|
|
2670
|
+
if (this._verbose) {
|
|
2671
|
+
LogStatus(`Memory Manager: Enriched example with userId from conversation: ${ctx.userId}`);
|
|
2672
|
+
}
|
|
1309
2673
|
}
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
LogStatus(`Memory Manager: Enriched example with agentRunId from conversation: ${ctx.agentRunId}`);
|
|
2674
|
+
if (!example.sourceAgentRunId && ctx.agentRunId) {
|
|
2675
|
+
example.sourceAgentRunId = ctx.agentRunId;
|
|
2676
|
+
if (this._verbose) {
|
|
2677
|
+
LogStatus(`Memory Manager: Enriched example with agentRunId from conversation: ${ctx.agentRunId}`);
|
|
2678
|
+
}
|
|
1316
2679
|
}
|
|
1317
2680
|
}
|
|
1318
2681
|
}
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
2682
|
+
if (example.agentId && !MemoryManagerAgent_1.isValidUUID(example.agentId)) {
|
|
2683
|
+
if (this._verbose) {
|
|
2684
|
+
LogStatus(`Memory Manager: Clearing invalid agentId "${example.agentId}" from example`);
|
|
2685
|
+
}
|
|
2686
|
+
example.agentId = '';
|
|
1324
2687
|
}
|
|
1325
|
-
example.agentId = '';
|
|
1326
|
-
}
|
|
1327
|
-
}
|
|
1328
|
-
// Create records
|
|
1329
|
-
const notesCreated = await this.CreateNoteRecords(extractedNotes, params.contextUser);
|
|
1330
|
-
const examplesCreated = await this.CreateExampleRecords(extractedExamples, params.contextUser);
|
|
1331
|
-
LogStatus(`Memory Manager: Created ${notesCreated} notes and ${examplesCreated} examples`);
|
|
1332
|
-
// Step 9: Consolidate related notes
|
|
1333
|
-
// This finds clusters of similar notes and synthesizes them into single comprehensive notes
|
|
1334
|
-
let consolidatedCount = 0;
|
|
1335
|
-
let archivedCount = 0;
|
|
1336
|
-
if (await this.shouldRunConsolidation(params.agent.ID, params.contextUser)) {
|
|
1337
|
-
LogStatus(`Memory Manager: Running note consolidation...`);
|
|
1338
|
-
const consolidationResult = await this.consolidateRelatedNotes(null, params.contextUser);
|
|
1339
|
-
consolidatedCount = consolidationResult.consolidated;
|
|
1340
|
-
archivedCount = consolidationResult.archived;
|
|
1341
|
-
if (consolidatedCount > 0 || archivedCount > 0) {
|
|
1342
|
-
LogStatus(`Memory Manager: Consolidated ${consolidatedCount} note clusters, archived ${archivedCount} source notes`);
|
|
1343
|
-
}
|
|
1344
|
-
else {
|
|
1345
|
-
LogStatus(`Memory Manager: No notes to consolidate`);
|
|
1346
2688
|
}
|
|
2689
|
+
notesCreated = await this.CreateNoteRecords(extractedNotes, params.contextUser);
|
|
2690
|
+
examplesCreated = await this.CreateExampleRecords(extractedExamples, params.contextUser);
|
|
2691
|
+
if (this._verbose)
|
|
2692
|
+
LogStatus(`Memory Manager: Created ${notesCreated} notes and ${examplesCreated} examples`);
|
|
1347
2693
|
}
|
|
1348
|
-
const
|
|
2694
|
+
const forceMaintenance = params.data?.forceMaintenance === true;
|
|
2695
|
+
const triggerDecision = await this.shouldRunConsolidation(params.agent.ID, params.contextUser, forceMaintenance);
|
|
2696
|
+
const maintenance = triggerDecision.shouldRun
|
|
2697
|
+
? await this.runMaintenancePhases(params.contextUser, forceMaintenance, triggerDecision.triggerType)
|
|
2698
|
+
: { consolidatedCount: 0, consolidationArchived: 0, contradictionsFound: 0, contradictionsResolved: 0, contradictionsFlagged: 0, staleNotesArchived: 0, decayNotesArchived: 0, decayExamplesArchived: 0, importanceScored: 0, tierPromotions: 0 };
|
|
2699
|
+
const maintenanceSummary = this.buildMaintenanceSummary(maintenance);
|
|
1349
2700
|
const finalStep = {
|
|
1350
2701
|
terminate: true,
|
|
1351
2702
|
step: 'Success',
|
|
1352
|
-
message: `Processed ${conversations.length} conversations (${totalMessages} messages) and ${agentRuns.length} agent runs. Created ${notesCreated} notes and ${examplesCreated} examples.${
|
|
2703
|
+
message: `Processed ${conversations.length} conversations (${totalMessages} messages) and ${agentRuns.length} agent runs. Created ${notesCreated} notes and ${examplesCreated} examples.${maintenanceSummary}`,
|
|
1353
2704
|
newPayload: {
|
|
1354
2705
|
notesCreated,
|
|
1355
2706
|
examplesCreated,
|
|
1356
2707
|
conversationsProcessed: conversations.length,
|
|
1357
2708
|
messagesProcessed: totalMessages,
|
|
1358
2709
|
agentRunsProcessed: agentRuns.length,
|
|
1359
|
-
|
|
1360
|
-
notesArchived: archivedCount
|
|
2710
|
+
...maintenance
|
|
1361
2711
|
}
|
|
1362
2712
|
};
|
|
1363
2713
|
return { finalStep, stepCount: 1 };
|
|
@@ -1373,7 +2723,7 @@ let MemoryManagerAgent = class MemoryManagerAgent extends BaseAgent {
|
|
|
1373
2723
|
}
|
|
1374
2724
|
}
|
|
1375
2725
|
};
|
|
1376
|
-
MemoryManagerAgent = __decorate([
|
|
2726
|
+
MemoryManagerAgent = MemoryManagerAgent_1 = __decorate([
|
|
1377
2727
|
RegisterClass(BaseAgent, 'MemoryManagerAgent')
|
|
1378
2728
|
], MemoryManagerAgent);
|
|
1379
2729
|
export { MemoryManagerAgent };
|