@timmeck/brain-core 2.36.85 → 2.36.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dream/consolidator.d.ts +7 -1
- package/dist/dream/consolidator.js +81 -0
- package/dist/dream/consolidator.js.map +1 -1
- package/dist/dream/dream-engine.js +24 -3
- package/dist/dream/dream-engine.js.map +1 -1
- package/dist/dream/index.d.ts +1 -1
- package/dist/dream/types.d.ts +6 -0
- package/dist/experiments/experiment-ledger.d.ts +85 -0
- package/dist/experiments/experiment-ledger.js +195 -0
- package/dist/experiments/experiment-ledger.js.map +1 -0
- package/dist/experiments/index.d.ts +2 -0
- package/dist/experiments/index.js +2 -0
- package/dist/experiments/index.js.map +1 -0
- package/dist/hypothesis/engine.d.ts +8 -2
- package/dist/hypothesis/engine.js +21 -1
- package/dist/hypothesis/engine.js.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.js +5 -0
- package/dist/index.js.map +1 -1
- package/dist/memory/conversation-memory.d.ts +27 -0
- package/dist/memory/conversation-memory.js +235 -7
- package/dist/memory/conversation-memory.js.map +1 -1
- package/dist/memory/retrieval-maintenance.d.ts +57 -0
- package/dist/memory/retrieval-maintenance.js +201 -0
- package/dist/memory/retrieval-maintenance.js.map +1 -0
- package/dist/memory/types.d.ts +1 -1
- package/dist/retention/index.d.ts +2 -0
- package/dist/retention/index.js +2 -0
- package/dist/retention/index.js.map +1 -0
- package/dist/retention/retention-engine.d.ts +77 -0
- package/dist/retention/retention-engine.js +325 -0
- package/dist/retention/retention-engine.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// ── Retrieval Maintenance Engine ──────────────────────────
|
|
2
|
+
//
|
|
3
|
+
// Offline maintenance for memory retrieval quality.
|
|
4
|
+
// Runs periodically (every 2h) to:
|
|
5
|
+
// 1. Mark cold memories as archive candidates
|
|
6
|
+
// 2. Build pre-computed candidate sets for fast typed retrieval
|
|
7
|
+
//
|
|
8
|
+
// Design principle (ChatGPT): "Retrieval online einfach halten, Struktur offline pflegen."
|
|
9
|
+
import { getLogger } from '../utils/logger.js';
|
|
10
|
+
// ── Intent Definitions ────────────────────────────────────
|
|
11
|
+
const INTENT_DEFINITIONS = {
|
|
12
|
+
decision_lookup: { categories: ['decision'], minImportance: 3 },
|
|
13
|
+
project_context: { categories: ['context', 'fact'], minImportance: 3 },
|
|
14
|
+
user_preference_lookup: { categories: ['preference', 'constraint'], minImportance: 2 },
|
|
15
|
+
open_problem_lookup: { categories: ['open_question', 'goal'], minImportance: 2 },
|
|
16
|
+
};
|
|
17
|
+
// ── Engine ────────────────────────────────────────────────
|
|
18
|
+
export class RetrievalMaintenanceEngine {
|
|
19
|
+
db;
|
|
20
|
+
config;
|
|
21
|
+
log = getLogger();
|
|
22
|
+
timer = null;
|
|
23
|
+
totalRuns = 0;
|
|
24
|
+
lastRunAt = null;
|
|
25
|
+
constructor(db, config = {}) {
|
|
26
|
+
this.db = db;
|
|
27
|
+
this.config = {
|
|
28
|
+
coldThresholdDays: config.coldThresholdDays ?? 30,
|
|
29
|
+
minImportanceForProtection: config.minImportanceForProtection ?? 4,
|
|
30
|
+
candidateSetSize: config.candidateSetSize ?? 50,
|
|
31
|
+
};
|
|
32
|
+
this.ensureTable();
|
|
33
|
+
}
|
|
34
|
+
/** Start periodic maintenance (default: every 2h). */
|
|
35
|
+
start(intervalMs = 2 * 60 * 60 * 1000) {
|
|
36
|
+
if (this.timer)
|
|
37
|
+
return;
|
|
38
|
+
this.timer = setInterval(() => {
|
|
39
|
+
try {
|
|
40
|
+
this.runMaintenance();
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
this.log.debug(`[retrieval-maintenance] Cycle error: ${err.message}`);
|
|
44
|
+
}
|
|
45
|
+
}, intervalMs);
|
|
46
|
+
this.log.info(`[retrieval-maintenance] Started (interval: ${Math.round(intervalMs / 3600000)}h)`);
|
|
47
|
+
}
|
|
48
|
+
/** Stop periodic maintenance. */
|
|
49
|
+
stop() {
|
|
50
|
+
if (this.timer) {
|
|
51
|
+
clearInterval(this.timer);
|
|
52
|
+
this.timer = null;
|
|
53
|
+
this.log.info('[retrieval-maintenance] Stopped');
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/** Main cycle — call periodically (every 2h). */
|
|
57
|
+
runMaintenance() {
|
|
58
|
+
const start = Date.now();
|
|
59
|
+
const archiveCandidatesMarked = this.markArchiveCandidates();
|
|
60
|
+
const candidateReport = this.refreshCandidateSets();
|
|
61
|
+
this.totalRuns++;
|
|
62
|
+
this.lastRunAt = new Date().toISOString();
|
|
63
|
+
const durationMs = Date.now() - start;
|
|
64
|
+
this.log.info(`[retrieval-maintenance] Run #${this.totalRuns}: ${archiveCandidatesMarked} archive candidates, ${candidateReport.setsCreated} sets refreshed (${durationMs}ms)`);
|
|
65
|
+
return {
|
|
66
|
+
archiveCandidatesMarked,
|
|
67
|
+
candidateSetsRefreshed: candidateReport.setsCreated,
|
|
68
|
+
durationMs,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Cold Memory Detection.
|
|
73
|
+
* Mark as archive_candidate = 1 when:
|
|
74
|
+
* - created_at > coldThresholdDays AND
|
|
75
|
+
* - access_count = 0 AND use_count = 0 AND
|
|
76
|
+
* - importance < minImportanceForProtection
|
|
77
|
+
*
|
|
78
|
+
* Safety: No automatic deletion, only marking.
|
|
79
|
+
*/
|
|
80
|
+
markArchiveCandidates() {
|
|
81
|
+
try {
|
|
82
|
+
// Reset existing candidates that no longer qualify (accessed since last run)
|
|
83
|
+
this.db.prepare(`
|
|
84
|
+
UPDATE conversation_memories SET archive_candidate = 0
|
|
85
|
+
WHERE archive_candidate = 1 AND (access_count > 0 OR use_count > 0 OR importance >= ?)
|
|
86
|
+
`).run(this.config.minImportanceForProtection);
|
|
87
|
+
// Mark new cold memories
|
|
88
|
+
const result = this.db.prepare(`
|
|
89
|
+
UPDATE conversation_memories SET archive_candidate = 1
|
|
90
|
+
WHERE active = 1
|
|
91
|
+
AND archive_candidate = 0
|
|
92
|
+
AND created_at < datetime('now', '-' || ? || ' days')
|
|
93
|
+
AND access_count = 0
|
|
94
|
+
AND use_count = 0
|
|
95
|
+
AND importance < ?
|
|
96
|
+
`).run(this.config.coldThresholdDays, this.config.minImportanceForProtection);
|
|
97
|
+
return result.changes;
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
this.log.debug(`[retrieval-maintenance] markArchiveCandidates error: ${err.message}`);
|
|
101
|
+
return 0;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/** Refresh pre-computed candidate sets for typed retrieval. */
|
|
105
|
+
refreshCandidateSets() {
|
|
106
|
+
let setsCreated = 0;
|
|
107
|
+
let totalMemories = 0;
|
|
108
|
+
// Per-category sets
|
|
109
|
+
const categories = ['preference', 'decision', 'context', 'fact', 'goal', 'lesson', 'constraint', 'open_question'];
|
|
110
|
+
for (const cat of categories) {
|
|
111
|
+
const count = this.buildCandidateSet('category', cat, [cat]);
|
|
112
|
+
if (count > 0) {
|
|
113
|
+
setsCreated++;
|
|
114
|
+
totalMemories += count;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// Per-intent sets
|
|
118
|
+
for (const [intent, def] of Object.entries(INTENT_DEFINITIONS)) {
|
|
119
|
+
const count = this.buildCandidateSet('intent', intent, def.categories, def.minImportance);
|
|
120
|
+
if (count > 0) {
|
|
121
|
+
setsCreated++;
|
|
122
|
+
totalMemories += count;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return { setsCreated, totalMemories };
|
|
126
|
+
}
|
|
127
|
+
/** Get current status. */
|
|
128
|
+
getStatus() {
|
|
129
|
+
let totalArchiveCandidates = 0;
|
|
130
|
+
let candidateSets = 0;
|
|
131
|
+
try {
|
|
132
|
+
const archiveRow = this.db.prepare('SELECT COUNT(*) as cnt FROM conversation_memories WHERE archive_candidate = 1').get();
|
|
133
|
+
totalArchiveCandidates = archiveRow?.cnt ?? 0;
|
|
134
|
+
}
|
|
135
|
+
catch { /* table may not have column yet */ }
|
|
136
|
+
try {
|
|
137
|
+
const setsRow = this.db.prepare('SELECT COUNT(*) as cnt FROM retrieval_candidate_sets').get();
|
|
138
|
+
candidateSets = setsRow?.cnt ?? 0;
|
|
139
|
+
}
|
|
140
|
+
catch { /* table may not exist */ }
|
|
141
|
+
return {
|
|
142
|
+
lastRunAt: this.lastRunAt,
|
|
143
|
+
totalRuns: this.totalRuns,
|
|
144
|
+
totalArchiveCandidates,
|
|
145
|
+
candidateSets,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
/** Get a candidate set by type and key. Returns memory IDs. */
|
|
149
|
+
getCandidateSet(setType, setKey) {
|
|
150
|
+
try {
|
|
151
|
+
const row = this.db.prepare('SELECT memory_ids FROM retrieval_candidate_sets WHERE set_type = ? AND set_key = ?').get(setType, setKey);
|
|
152
|
+
if (!row)
|
|
153
|
+
return [];
|
|
154
|
+
return JSON.parse(row.memory_ids);
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
return [];
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
// ── Private ─────────────────────────────────────────────
|
|
161
|
+
ensureTable() {
|
|
162
|
+
this.db.exec(`
|
|
163
|
+
CREATE TABLE IF NOT EXISTS retrieval_candidate_sets (
|
|
164
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
165
|
+
set_type TEXT NOT NULL,
|
|
166
|
+
set_key TEXT NOT NULL,
|
|
167
|
+
memory_ids TEXT NOT NULL DEFAULT '[]',
|
|
168
|
+
set_size INTEGER NOT NULL DEFAULT 0,
|
|
169
|
+
refreshed_at TEXT DEFAULT (datetime('now')),
|
|
170
|
+
UNIQUE(set_type, set_key)
|
|
171
|
+
);
|
|
172
|
+
`);
|
|
173
|
+
}
|
|
174
|
+
buildCandidateSet(setType, setKey, categories, minImportance = 1) {
|
|
175
|
+
try {
|
|
176
|
+
const placeholders = categories.map(() => '?').join(',');
|
|
177
|
+
const rows = this.db.prepare(`
|
|
178
|
+
SELECT id FROM conversation_memories
|
|
179
|
+
WHERE active = 1
|
|
180
|
+
AND archive_candidate = 0
|
|
181
|
+
AND category IN (${placeholders})
|
|
182
|
+
AND importance >= ?
|
|
183
|
+
ORDER BY importance DESC, use_count DESC, access_count DESC
|
|
184
|
+
LIMIT ?
|
|
185
|
+
`).all(...categories, minImportance, this.config.candidateSetSize);
|
|
186
|
+
const ids = rows.map(r => r.id);
|
|
187
|
+
this.db.prepare(`
|
|
188
|
+
INSERT INTO retrieval_candidate_sets (set_type, set_key, memory_ids, set_size, refreshed_at)
|
|
189
|
+
VALUES (?, ?, ?, ?, datetime('now'))
|
|
190
|
+
ON CONFLICT(set_type, set_key)
|
|
191
|
+
DO UPDATE SET memory_ids = excluded.memory_ids, set_size = excluded.set_size, refreshed_at = datetime('now')
|
|
192
|
+
`).run(setType, setKey, JSON.stringify(ids), ids.length);
|
|
193
|
+
return ids.length;
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
this.log.debug(`[retrieval-maintenance] buildCandidateSet error for ${setType}/${setKey}: ${err.message}`);
|
|
197
|
+
return 0;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=retrieval-maintenance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retrieval-maintenance.js","sourceRoot":"","sources":["../../src/memory/retrieval-maintenance.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAC7D,EAAE;AACF,oDAAoD;AACpD,mCAAmC;AACnC,gDAAgD;AAChD,kEAAkE;AAClE,EAAE;AACF,2FAA2F;AAG3F,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AA+B/C,6DAA6D;AAE7D,MAAM,kBAAkB,GAAoE;IAC1F,eAAe,EAAE,EAAE,UAAU,EAAE,CAAC,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;IAC/D,eAAe,EAAE,EAAE,UAAU,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;IACtE,sBAAsB,EAAE,EAAE,UAAU,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;IACtF,mBAAmB,EAAE,EAAE,UAAU,EAAE,CAAC,eAAe,EAAE,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE;CACjF,CAAC;AAEF,6DAA6D;AAE7D,MAAM,OAAO,0BAA0B;IACpB,EAAE,CAAoB;IACtB,MAAM,CAAuC;IAC7C,GAAG,GAAG,SAAS,EAAE,CAAC;IAC3B,KAAK,GAA0C,IAAI,CAAC;IACpD,SAAS,GAAG,CAAC,CAAC;IACd,SAAS,GAAkB,IAAI,CAAC;IAExC,YAAY,EAAqB,EAAE,SAAqC,EAAE;QACxE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG;YACZ,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,EAAE;YACjD,0BAA0B,EAAE,MAAM,CAAC,0BAA0B,IAAI,CAAC;YAClE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,EAAE;SAChD,CAAC;QACF,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,sDAAsD;IACtD,KAAK,CAAC,UAAU,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI;QACnC,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC5B,IAAI,CAAC;gBACH,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wCAAyC,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACnF,CAAC;QACH,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,8CAA8C,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpG,CAAC;IAED,iCAAiC;IACjC,IAAI;QACF,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,cAAc;QACZ,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,MAAM,uBAAuB,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAC7D,MAAM,eAAe,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAEpD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gCAAgC,IAAI,CAAC,SAAS,KAAK,uBAAuB,wBAAwB,eAAe,CAAC,WAAW,oBAAoB,UAAU,KAAK,CAAC,CAAC;QAEhL,OAAO;YACL,uBAAuB;YACvB,sBAAsB,EAAE,eAAe,CAAC,WAAW;YACnD,UAAU;SACX,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,qBAAqB;QACnB,IAAI,CAAC;YACH,6EAA6E;YAC7E,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;OAGf,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;YAE/C,yBAAyB;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;OAQ9B,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;YAE9E,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wDAAyD,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACjG,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,oBAAoB;QAClB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,oBAAoB;QACpB,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;QAClH,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7D,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBAAC,WAAW,EAAE,CAAC;gBAAC,aAAa,IAAI,KAAK,CAAC;YAAC,CAAC;QAC3D,CAAC;QAED,kBAAkB;QAClB,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,aAAa,CAAC,CAAC;YAC1F,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBAAC,WAAW,EAAE,CAAC;gBAAC,aAAa,IAAI,KAAK,CAAC;YAAC,CAAC;QAC3D,CAAC;QAED,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC;IACxC,CAAC;IAED,0BAA0B;IAC1B,SAAS;QACP,IAAI,sBAAsB,GAAG,CAAC,CAAC;QAC/B,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAChC,+EAA+E,CAChF,CAAC,GAAG,EAAiC,CAAC;YACvC,sBAAsB,GAAG,UAAU,EAAE,GAAG,IAAI,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC,CAAC,mCAAmC,CAAC,CAAC;QAE/C,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAC7B,sDAAsD,CACvD,CAAC,GAAG,EAAiC,CAAC;YACvC,aAAa,GAAG,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;QAErC,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,sBAAsB;YACtB,aAAa;SACd,CAAC;IACJ,CAAC;IAED,+DAA+D;IAC/D,eAAe,CAAC,OAAe,EAAE,MAAc;QAC7C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB,oFAAoF,CACrF,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAuC,CAAC;YAC7D,IAAI,CAAC,GAAG;gBAAE,OAAO,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,2DAA2D;IAEnD,WAAW;QACjB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;KAUZ,CAAC,CAAC;IACL,CAAC;IAEO,iBAAiB,CAAC,OAAe,EAAE,MAAc,EAAE,UAAoB,EAAE,aAAa,GAAG,CAAC;QAChG,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;6BAIN,YAAY;;;;OAIlC,CAAC,CAAC,GAAG,CAAC,GAAG,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAA0B,CAAC;YAE5F,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAEhC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;OAKf,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;YAEzD,OAAO,GAAG,CAAC,MAAM,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,uDAAuD,OAAO,IAAI,MAAM,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;YACtH,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;CACF"}
|
package/dist/memory/types.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type MemoryCategory = 'preference' | 'decision' | 'context' | 'fact' | 'goal' | 'lesson';
|
|
1
|
+
export type MemoryCategory = 'preference' | 'decision' | 'context' | 'fact' | 'goal' | 'lesson' | 'constraint' | 'open_question';
|
|
2
2
|
export type MemorySource = 'explicit' | 'inferred' | 'hook';
|
|
3
3
|
export type SessionOutcome = 'completed' | 'paused' | 'abandoned';
|
|
4
4
|
export interface MemoryRecord {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retention/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RetentionPolicyEngine — intelligent DB cleanup with protection rules.
|
|
3
|
+
* Targets: rag_vectors (cache TTL), conversation_memories (value-based),
|
|
4
|
+
* compressed_clusters (age TTL), insights (lifecycle supplement).
|
|
5
|
+
*
|
|
6
|
+
* Every method supports dry-run (count only) or live (actual DELETE).
|
|
7
|
+
*/
|
|
8
|
+
import type Database from 'better-sqlite3';
|
|
9
|
+
export interface RetentionConfig {
|
|
10
|
+
/** rag_vectors TTL in days. Default: 30 */
|
|
11
|
+
ragVectorTTLDays: number;
|
|
12
|
+
/** Minimum importance for vector protection. Default: 6 */
|
|
13
|
+
ragVectorProtectionImportance: number;
|
|
14
|
+
/** conversation_memories: days without access + low importance. Default: 90 */
|
|
15
|
+
memoryTTLDays: number;
|
|
16
|
+
/** Minimum importance for memory protection. Default: 4 */
|
|
17
|
+
memoryProtectionImportance: number;
|
|
18
|
+
/** compressed_clusters TTL. Default: 60 */
|
|
19
|
+
clusterTTLDays: number;
|
|
20
|
+
/** insights: archived insights TTL. Default: 120 */
|
|
21
|
+
insightTTLDays: number;
|
|
22
|
+
/** Max rows per batch (safety). Default: 10000 */
|
|
23
|
+
batchLimit: number;
|
|
24
|
+
}
|
|
25
|
+
export interface TableReport {
|
|
26
|
+
before: number;
|
|
27
|
+
affected: number;
|
|
28
|
+
protected: number;
|
|
29
|
+
estimatedMB: number;
|
|
30
|
+
}
|
|
31
|
+
export interface ProtectionSummary {
|
|
32
|
+
byImportance: number;
|
|
33
|
+
byUseCount: number;
|
|
34
|
+
byReferences: number;
|
|
35
|
+
byConsolidation: number;
|
|
36
|
+
}
|
|
37
|
+
export interface RetentionReport {
|
|
38
|
+
dryRun: boolean;
|
|
39
|
+
timestamp: string;
|
|
40
|
+
tables: {
|
|
41
|
+
rag_vectors: TableReport;
|
|
42
|
+
conversation_memories: TableReport;
|
|
43
|
+
compressed_clusters: TableReport;
|
|
44
|
+
insights: TableReport;
|
|
45
|
+
};
|
|
46
|
+
totalRowsAffected: number;
|
|
47
|
+
estimatedSpaceMB: number;
|
|
48
|
+
protectedRows: ProtectionSummary;
|
|
49
|
+
durationMs: number;
|
|
50
|
+
}
|
|
51
|
+
export interface TableSizeInfo {
|
|
52
|
+
table: string;
|
|
53
|
+
rowCount: number;
|
|
54
|
+
estimatedMB: number;
|
|
55
|
+
}
|
|
56
|
+
export interface RetentionStatus {
|
|
57
|
+
totalRuns: number;
|
|
58
|
+
lastReport: RetentionReport | null;
|
|
59
|
+
config: RetentionConfig;
|
|
60
|
+
}
|
|
61
|
+
export declare class RetentionPolicyEngine {
|
|
62
|
+
private readonly db;
|
|
63
|
+
private readonly config;
|
|
64
|
+
private readonly log;
|
|
65
|
+
private lastReport;
|
|
66
|
+
private totalRuns;
|
|
67
|
+
constructor(db: Database.Database, config?: Partial<RetentionConfig>);
|
|
68
|
+
run(dryRun?: boolean): RetentionReport;
|
|
69
|
+
cleanRagVectors(dryRun: boolean, protectedRows?: ProtectionSummary): TableReport;
|
|
70
|
+
cleanMemories(dryRun: boolean, protectedRows?: ProtectionSummary): TableReport;
|
|
71
|
+
cleanClusters(dryRun: boolean): TableReport;
|
|
72
|
+
cleanInsights(dryRun: boolean): TableReport;
|
|
73
|
+
getStatus(): RetentionStatus;
|
|
74
|
+
getLastReport(): RetentionReport | null;
|
|
75
|
+
getTableSizes(): TableSizeInfo[];
|
|
76
|
+
private countRows;
|
|
77
|
+
}
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
import { getLogger } from '../utils/logger.js';
|
|
2
|
+
const DEFAULT_CONFIG = {
|
|
3
|
+
ragVectorTTLDays: 30,
|
|
4
|
+
ragVectorProtectionImportance: 6,
|
|
5
|
+
memoryTTLDays: 90,
|
|
6
|
+
memoryProtectionImportance: 4,
|
|
7
|
+
clusterTTLDays: 60,
|
|
8
|
+
insightTTLDays: 120,
|
|
9
|
+
batchLimit: 10_000,
|
|
10
|
+
};
|
|
11
|
+
// ── Engine ──────────────────────────────────────────────────
|
|
12
|
+
export class RetentionPolicyEngine {
|
|
13
|
+
db;
|
|
14
|
+
config;
|
|
15
|
+
log = getLogger();
|
|
16
|
+
lastReport = null;
|
|
17
|
+
totalRuns = 0;
|
|
18
|
+
constructor(db, config) {
|
|
19
|
+
this.db = db;
|
|
20
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
21
|
+
}
|
|
22
|
+
// ── Main Entry ────────────────────────────────────────────
|
|
23
|
+
run(dryRun = true) {
|
|
24
|
+
const start = Date.now();
|
|
25
|
+
const protectedRows = { byImportance: 0, byUseCount: 0, byReferences: 0, byConsolidation: 0 };
|
|
26
|
+
const ragReport = this.cleanRagVectors(dryRun, protectedRows);
|
|
27
|
+
const memReport = this.cleanMemories(dryRun, protectedRows);
|
|
28
|
+
const clusterReport = this.cleanClusters(dryRun);
|
|
29
|
+
const insightReport = this.cleanInsights(dryRun);
|
|
30
|
+
const report = {
|
|
31
|
+
dryRun,
|
|
32
|
+
timestamp: new Date().toISOString(),
|
|
33
|
+
tables: {
|
|
34
|
+
rag_vectors: ragReport,
|
|
35
|
+
conversation_memories: memReport,
|
|
36
|
+
compressed_clusters: clusterReport,
|
|
37
|
+
insights: insightReport,
|
|
38
|
+
},
|
|
39
|
+
totalRowsAffected: ragReport.affected + memReport.affected + clusterReport.affected + insightReport.affected,
|
|
40
|
+
estimatedSpaceMB: +(ragReport.estimatedMB + memReport.estimatedMB + clusterReport.estimatedMB + insightReport.estimatedMB).toFixed(2),
|
|
41
|
+
protectedRows,
|
|
42
|
+
durationMs: Date.now() - start,
|
|
43
|
+
};
|
|
44
|
+
this.lastReport = report;
|
|
45
|
+
this.totalRuns++;
|
|
46
|
+
const mode = dryRun ? 'DRY-RUN' : 'LIVE';
|
|
47
|
+
this.log.info(`[retention] ${mode}: ${report.totalRowsAffected} rows affected (~${report.estimatedSpaceMB} MB), ${report.durationMs}ms`);
|
|
48
|
+
return report;
|
|
49
|
+
}
|
|
50
|
+
// ── rag_vectors ───────────────────────────────────────────
|
|
51
|
+
cleanRagVectors(dryRun, protectedRows) {
|
|
52
|
+
const cfg = this.config;
|
|
53
|
+
const before = this.countRows('rag_vectors');
|
|
54
|
+
// Protected: vectors linked to important/active/used memories
|
|
55
|
+
// We need to check if conversation_memories table exists and has the right columns
|
|
56
|
+
let protectedCount = 0;
|
|
57
|
+
try {
|
|
58
|
+
// Count vectors whose linked memory is protected (importance >= threshold OR use_count > 0 OR active+not-archive)
|
|
59
|
+
const protectedRow = this.db.prepare(`
|
|
60
|
+
SELECT COUNT(DISTINCT rv.id) as cnt FROM rag_vectors rv
|
|
61
|
+
INNER JOIN conversation_memories cm ON rv.source_id = cm.id AND rv.collection = 'conversation_memories'
|
|
62
|
+
WHERE rv.created_at < datetime('now', ?)
|
|
63
|
+
AND (cm.importance >= ? OR cm.use_count > 0 OR (cm.active = 1 AND cm.archive_candidate = 0))
|
|
64
|
+
`).get(`-${cfg.ragVectorTTLDays} days`, cfg.ragVectorProtectionImportance);
|
|
65
|
+
protectedCount = protectedRow?.cnt ?? 0;
|
|
66
|
+
if (protectedRows) {
|
|
67
|
+
// Break down protection reasons
|
|
68
|
+
const byImp = this.db.prepare(`
|
|
69
|
+
SELECT COUNT(DISTINCT rv.id) as cnt FROM rag_vectors rv
|
|
70
|
+
INNER JOIN conversation_memories cm ON rv.source_id = cm.id AND rv.collection = 'conversation_memories'
|
|
71
|
+
WHERE rv.created_at < datetime('now', ?) AND cm.importance >= ?
|
|
72
|
+
`).get(`-${cfg.ragVectorTTLDays} days`, cfg.ragVectorProtectionImportance);
|
|
73
|
+
protectedRows.byImportance += byImp?.cnt ?? 0;
|
|
74
|
+
const byUse = this.db.prepare(`
|
|
75
|
+
SELECT COUNT(DISTINCT rv.id) as cnt FROM rag_vectors rv
|
|
76
|
+
INNER JOIN conversation_memories cm ON rv.source_id = cm.id AND rv.collection = 'conversation_memories'
|
|
77
|
+
WHERE rv.created_at < datetime('now', ?) AND cm.use_count > 0
|
|
78
|
+
`).get(`-${cfg.ragVectorTTLDays} days`);
|
|
79
|
+
protectedRows.byUseCount += byUse?.cnt ?? 0;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// conversation_memories table may not exist — skip protection checks
|
|
84
|
+
}
|
|
85
|
+
// Deletable: old vectors NOT linked to important memories
|
|
86
|
+
// Strategy: delete old vectors where linked memory is deleted/inactive/low-importance+unused
|
|
87
|
+
let affected = 0;
|
|
88
|
+
try {
|
|
89
|
+
if (dryRun) {
|
|
90
|
+
const row = this.db.prepare(`
|
|
91
|
+
SELECT COUNT(*) as cnt FROM rag_vectors rv
|
|
92
|
+
WHERE rv.created_at < datetime('now', ?)
|
|
93
|
+
AND NOT EXISTS (
|
|
94
|
+
SELECT 1 FROM conversation_memories cm
|
|
95
|
+
WHERE cm.id = rv.source_id AND rv.collection = 'conversation_memories'
|
|
96
|
+
AND (cm.importance >= ? OR cm.use_count > 0 OR (cm.active = 1 AND cm.archive_candidate = 0))
|
|
97
|
+
)
|
|
98
|
+
LIMIT ?
|
|
99
|
+
`).get(`-${cfg.ragVectorTTLDays} days`, cfg.ragVectorProtectionImportance, cfg.batchLimit);
|
|
100
|
+
// The LIMIT in subquery doesn't work for COUNT, use a different approach
|
|
101
|
+
const countRow = this.db.prepare(`
|
|
102
|
+
SELECT COUNT(*) as cnt FROM (
|
|
103
|
+
SELECT rv.id FROM rag_vectors rv
|
|
104
|
+
WHERE rv.created_at < datetime('now', ?)
|
|
105
|
+
AND NOT EXISTS (
|
|
106
|
+
SELECT 1 FROM conversation_memories cm
|
|
107
|
+
WHERE cm.id = rv.source_id AND rv.collection = 'conversation_memories'
|
|
108
|
+
AND (cm.importance >= ? OR cm.use_count > 0 OR (cm.active = 1 AND cm.archive_candidate = 0))
|
|
109
|
+
)
|
|
110
|
+
LIMIT ?
|
|
111
|
+
)
|
|
112
|
+
`).get(`-${cfg.ragVectorTTLDays} days`, cfg.ragVectorProtectionImportance, cfg.batchLimit);
|
|
113
|
+
affected = countRow.cnt;
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
const result = this.db.prepare(`
|
|
117
|
+
DELETE FROM rag_vectors WHERE id IN (
|
|
118
|
+
SELECT rv.id FROM rag_vectors rv
|
|
119
|
+
WHERE rv.created_at < datetime('now', ?)
|
|
120
|
+
AND NOT EXISTS (
|
|
121
|
+
SELECT 1 FROM conversation_memories cm
|
|
122
|
+
WHERE cm.id = rv.source_id AND rv.collection = 'conversation_memories'
|
|
123
|
+
AND (cm.importance >= ? OR cm.use_count > 0 OR (cm.active = 1 AND cm.archive_candidate = 0))
|
|
124
|
+
)
|
|
125
|
+
LIMIT ?
|
|
126
|
+
)
|
|
127
|
+
`).run(`-${cfg.ragVectorTTLDays} days`, cfg.ragVectorProtectionImportance, cfg.batchLimit);
|
|
128
|
+
affected = Number(result.changes);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
catch (err) {
|
|
132
|
+
this.log.debug(`[retention] rag_vectors cleanup skipped: ${err.message}`);
|
|
133
|
+
}
|
|
134
|
+
// Estimate ~10KB per vector (embedding blob + metadata)
|
|
135
|
+
return { before, affected, protected: protectedCount, estimatedMB: +(affected * 10 / 1024).toFixed(2) };
|
|
136
|
+
}
|
|
137
|
+
// ── conversation_memories ─────────────────────────────────
|
|
138
|
+
cleanMemories(dryRun, protectedRows) {
|
|
139
|
+
const cfg = this.config;
|
|
140
|
+
const before = this.countRows('conversation_memories');
|
|
141
|
+
// Count protected rows (meeting any protection criterion but past TTL)
|
|
142
|
+
let protectedCount = 0;
|
|
143
|
+
try {
|
|
144
|
+
const protectedRow = this.db.prepare(`
|
|
145
|
+
SELECT COUNT(*) as cnt FROM conversation_memories
|
|
146
|
+
WHERE archive_candidate = 1
|
|
147
|
+
AND created_at < datetime('now', ?)
|
|
148
|
+
AND (importance >= ? OR use_count > 0 OR source = 'inferred'
|
|
149
|
+
OR (access_count > 0 AND last_accessed_at > datetime('now', '-30 days')))
|
|
150
|
+
`).get(`-${cfg.memoryTTLDays} days`, cfg.memoryProtectionImportance);
|
|
151
|
+
protectedCount = protectedRow.cnt;
|
|
152
|
+
if (protectedRows) {
|
|
153
|
+
const byImp = this.db.prepare(`
|
|
154
|
+
SELECT COUNT(*) as cnt FROM conversation_memories
|
|
155
|
+
WHERE archive_candidate = 1 AND created_at < datetime('now', ?) AND importance >= ?
|
|
156
|
+
`).get(`-${cfg.memoryTTLDays} days`, cfg.memoryProtectionImportance);
|
|
157
|
+
protectedRows.byImportance += byImp.cnt;
|
|
158
|
+
const byUse = this.db.prepare(`
|
|
159
|
+
SELECT COUNT(*) as cnt FROM conversation_memories
|
|
160
|
+
WHERE archive_candidate = 1 AND created_at < datetime('now', ?) AND use_count > 0
|
|
161
|
+
`).get(`-${cfg.memoryTTLDays} days`);
|
|
162
|
+
protectedRows.byUseCount += byUse.cnt;
|
|
163
|
+
const byRef = this.db.prepare(`
|
|
164
|
+
SELECT COUNT(*) as cnt FROM conversation_memories
|
|
165
|
+
WHERE archive_candidate = 1 AND created_at < datetime('now', ?)
|
|
166
|
+
AND access_count > 0 AND last_accessed_at > datetime('now', '-30 days')
|
|
167
|
+
`).get(`-${cfg.memoryTTLDays} days`);
|
|
168
|
+
protectedRows.byReferences += byRef.cnt;
|
|
169
|
+
const byCons = this.db.prepare(`
|
|
170
|
+
SELECT COUNT(*) as cnt FROM conversation_memories
|
|
171
|
+
WHERE archive_candidate = 1 AND created_at < datetime('now', ?) AND source = 'inferred'
|
|
172
|
+
`).get(`-${cfg.memoryTTLDays} days`);
|
|
173
|
+
protectedRows.byConsolidation += byCons.cnt;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
// table may not exist
|
|
178
|
+
}
|
|
179
|
+
// Deletable: archive_candidate=1, past TTL, no protection criteria met
|
|
180
|
+
let affected = 0;
|
|
181
|
+
try {
|
|
182
|
+
if (dryRun) {
|
|
183
|
+
const row = this.db.prepare(`
|
|
184
|
+
SELECT COUNT(*) as cnt FROM (
|
|
185
|
+
SELECT id FROM conversation_memories
|
|
186
|
+
WHERE archive_candidate = 1
|
|
187
|
+
AND created_at < datetime('now', ?)
|
|
188
|
+
AND importance < ?
|
|
189
|
+
AND use_count = 0
|
|
190
|
+
AND source != 'inferred'
|
|
191
|
+
AND NOT (access_count > 0 AND last_accessed_at > datetime('now', '-30 days'))
|
|
192
|
+
LIMIT ?
|
|
193
|
+
)
|
|
194
|
+
`).get(`-${cfg.memoryTTLDays} days`, cfg.memoryProtectionImportance, cfg.batchLimit);
|
|
195
|
+
affected = row.cnt;
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
const result = this.db.prepare(`
|
|
199
|
+
DELETE FROM conversation_memories WHERE id IN (
|
|
200
|
+
SELECT id FROM conversation_memories
|
|
201
|
+
WHERE archive_candidate = 1
|
|
202
|
+
AND created_at < datetime('now', ?)
|
|
203
|
+
AND importance < ?
|
|
204
|
+
AND use_count = 0
|
|
205
|
+
AND source != 'inferred'
|
|
206
|
+
AND NOT (access_count > 0 AND last_accessed_at > datetime('now', '-30 days'))
|
|
207
|
+
LIMIT ?
|
|
208
|
+
)
|
|
209
|
+
`).run(`-${cfg.memoryTTLDays} days`, cfg.memoryProtectionImportance, cfg.batchLimit);
|
|
210
|
+
affected = Number(result.changes);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
this.log.debug(`[retention] conversation_memories cleanup skipped: ${err.message}`);
|
|
215
|
+
}
|
|
216
|
+
// Estimate ~3KB per memory row (content + tags + metadata)
|
|
217
|
+
return { before, affected, protected: protectedCount, estimatedMB: +(affected * 3 / 1024).toFixed(2) };
|
|
218
|
+
}
|
|
219
|
+
// ── compressed_clusters ───────────────────────────────────
|
|
220
|
+
cleanClusters(dryRun) {
|
|
221
|
+
const cfg = this.config;
|
|
222
|
+
const before = this.countRows('compressed_clusters');
|
|
223
|
+
let affected = 0;
|
|
224
|
+
try {
|
|
225
|
+
if (dryRun) {
|
|
226
|
+
const row = this.db.prepare(`
|
|
227
|
+
SELECT COUNT(*) as cnt FROM (
|
|
228
|
+
SELECT id FROM compressed_clusters
|
|
229
|
+
WHERE created_at < datetime('now', ?)
|
|
230
|
+
LIMIT ?
|
|
231
|
+
)
|
|
232
|
+
`).get(`-${cfg.clusterTTLDays} days`, cfg.batchLimit);
|
|
233
|
+
affected = row.cnt;
|
|
234
|
+
}
|
|
235
|
+
else {
|
|
236
|
+
const result = this.db.prepare(`
|
|
237
|
+
DELETE FROM compressed_clusters WHERE id IN (
|
|
238
|
+
SELECT id FROM compressed_clusters
|
|
239
|
+
WHERE created_at < datetime('now', ?)
|
|
240
|
+
LIMIT ?
|
|
241
|
+
)
|
|
242
|
+
`).run(`-${cfg.clusterTTLDays} days`, cfg.batchLimit);
|
|
243
|
+
affected = Number(result.changes);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
catch (err) {
|
|
247
|
+
this.log.debug(`[retention] compressed_clusters cleanup skipped: ${err.message}`);
|
|
248
|
+
}
|
|
249
|
+
// Estimate ~2KB per cluster (summary + member_ids JSON)
|
|
250
|
+
return { before, affected, protected: 0, estimatedMB: +(affected * 2 / 1024).toFixed(2) };
|
|
251
|
+
}
|
|
252
|
+
// ── insights (supplement to lifecycle.ts) ─────────────────
|
|
253
|
+
cleanInsights(dryRun) {
|
|
254
|
+
const cfg = this.config;
|
|
255
|
+
const before = this.countRows('insights');
|
|
256
|
+
let affected = 0;
|
|
257
|
+
try {
|
|
258
|
+
if (dryRun) {
|
|
259
|
+
const row = this.db.prepare(`
|
|
260
|
+
SELECT COUNT(*) as cnt FROM (
|
|
261
|
+
SELECT id FROM insights
|
|
262
|
+
WHERE lifecycle = 'archived' AND created_at < datetime('now', ?)
|
|
263
|
+
LIMIT ?
|
|
264
|
+
)
|
|
265
|
+
`).get(`-${cfg.insightTTLDays} days`, cfg.batchLimit);
|
|
266
|
+
affected = row.cnt;
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
const result = this.db.prepare(`
|
|
270
|
+
DELETE FROM insights WHERE id IN (
|
|
271
|
+
SELECT id FROM insights
|
|
272
|
+
WHERE lifecycle = 'archived' AND created_at < datetime('now', ?)
|
|
273
|
+
LIMIT ?
|
|
274
|
+
)
|
|
275
|
+
`).run(`-${cfg.insightTTLDays} days`, cfg.batchLimit);
|
|
276
|
+
affected = Number(result.changes);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
catch (err) {
|
|
280
|
+
this.log.debug(`[retention] insights cleanup skipped: ${err.message}`);
|
|
281
|
+
}
|
|
282
|
+
// Estimate ~1KB per insight
|
|
283
|
+
return { before, affected, protected: 0, estimatedMB: +(affected * 1 / 1024).toFixed(2) };
|
|
284
|
+
}
|
|
285
|
+
// ── Status & Reports ──────────────────────────────────────
|
|
286
|
+
getStatus() {
|
|
287
|
+
return {
|
|
288
|
+
totalRuns: this.totalRuns,
|
|
289
|
+
lastReport: this.lastReport,
|
|
290
|
+
config: { ...this.config },
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
getLastReport() {
|
|
294
|
+
return this.lastReport;
|
|
295
|
+
}
|
|
296
|
+
getTableSizes() {
|
|
297
|
+
const tables = ['rag_vectors', 'conversation_memories', 'compressed_clusters', 'insights'];
|
|
298
|
+
const sizes = [];
|
|
299
|
+
for (const table of tables) {
|
|
300
|
+
const rowCount = this.countRows(table);
|
|
301
|
+
// Estimate per-row sizes
|
|
302
|
+
const kbPerRow = table === 'rag_vectors' ? 10
|
|
303
|
+
: table === 'conversation_memories' ? 3
|
|
304
|
+
: table === 'compressed_clusters' ? 2
|
|
305
|
+
: 1;
|
|
306
|
+
sizes.push({
|
|
307
|
+
table,
|
|
308
|
+
rowCount,
|
|
309
|
+
estimatedMB: +(rowCount * kbPerRow / 1024).toFixed(2),
|
|
310
|
+
});
|
|
311
|
+
}
|
|
312
|
+
return sizes;
|
|
313
|
+
}
|
|
314
|
+
// ── Helpers ───────────────────────────────────────────────
|
|
315
|
+
countRows(table) {
|
|
316
|
+
try {
|
|
317
|
+
const row = this.db.prepare(`SELECT COUNT(*) as cnt FROM ${table}`).get();
|
|
318
|
+
return row?.cnt ?? 0;
|
|
319
|
+
}
|
|
320
|
+
catch {
|
|
321
|
+
return 0;
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
//# sourceMappingURL=retention-engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retention-engine.js","sourceRoot":"","sources":["../../src/retention/retention-engine.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAqB/C,MAAM,cAAc,GAAoB;IACtC,gBAAgB,EAAE,EAAE;IACpB,6BAA6B,EAAE,CAAC;IAChC,aAAa,EAAE,EAAE;IACjB,0BAA0B,EAAE,CAAC;IAC7B,cAAc,EAAE,EAAE;IAClB,cAAc,EAAE,GAAG;IACnB,UAAU,EAAE,MAAM;CACnB,CAAC;AA6CF,+DAA+D;AAE/D,MAAM,OAAO,qBAAqB;IACf,EAAE,CAAoB;IACtB,MAAM,CAAkB;IACxB,GAAG,GAAG,SAAS,EAAE,CAAC;IAC3B,UAAU,GAA2B,IAAI,CAAC;IAC1C,SAAS,GAAG,CAAC,CAAC;IAEtB,YAAY,EAAqB,EAAE,MAAiC;QAClE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC;IACjD,CAAC;IAED,6DAA6D;IAE7D,GAAG,CAAC,MAAM,GAAG,IAAI;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,aAAa,GAAsB,EAAE,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;QAEjH,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAoB;YAC9B,MAAM;YACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE;gBACN,WAAW,EAAE,SAAS;gBACtB,qBAAqB,EAAE,SAAS;gBAChC,mBAAmB,EAAE,aAAa;gBAClC,QAAQ,EAAE,aAAa;aACxB;YACD,iBAAiB,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAQ,GAAG,aAAa,CAAC,QAAQ,GAAG,aAAa,CAAC,QAAQ;YAC5G,gBAAgB,EAAE,CAAC,CAAC,SAAS,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,GAAG,aAAa,CAAC,WAAW,GAAG,aAAa,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACrI,aAAa;YACb,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SAC/B,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACzC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,KAAK,MAAM,CAAC,iBAAiB,oBAAoB,MAAM,CAAC,gBAAgB,SAAS,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;QAEzI,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,6DAA6D;IAE7D,eAAe,CAAC,MAAe,EAAE,aAAiC;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;QAE7C,8DAA8D;QAC9D,mFAAmF;QACnF,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC;YACH,kHAAkH;YAClH,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;OAKpC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,OAAO,EAAE,GAAG,CAAC,6BAA6B,CAAgC,CAAC;YAC1G,cAAc,GAAG,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC;YAExC,IAAI,aAAa,EAAE,CAAC;gBAClB,gCAAgC;gBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,OAAO,EAAE,GAAG,CAAC,6BAA6B,CAAgC,CAAC;gBAC1G,aAAa,CAAC,YAAY,IAAI,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;gBAE9C,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,OAAO,CAAgC,CAAC;gBACvE,aAAa,CAAC,UAAU,IAAI,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qEAAqE;QACvE,CAAC;QAED,0DAA0D;QAC1D,6FAA6F;QAC7F,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;SAS3B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,OAAO,EAAE,GAAG,CAAC,6BAA6B,EAAE,GAAG,CAAC,UAAU,CAAoB,CAAC;gBAC9G,yEAAyE;gBACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;SAWhC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,OAAO,EAAE,GAAG,CAAC,6BAA6B,EAAE,GAAG,CAAC,UAAU,CAAoB,CAAC;gBAC9G,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;SAW9B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,gBAAgB,OAAO,EAAE,GAAG,CAAC,6BAA6B,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;gBAC3F,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,4CAA6C,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,wDAAwD;QACxD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1G,CAAC;IAED,6DAA6D;IAE7D,aAAa,CAAC,MAAe,EAAE,aAAiC;QAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;QAEvD,uEAAuE;QACvE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;OAMpC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,EAAE,GAAG,CAAC,0BAA0B,CAAoB,CAAC;YACxF,cAAc,GAAG,YAAY,CAAC,GAAG,CAAC;YAElC,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG7B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,EAAE,GAAG,CAAC,0BAA0B,CAAoB,CAAC;gBACxF,aAAa,CAAC,YAAY,IAAI,KAAK,CAAC,GAAG,CAAC;gBAExC,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG7B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,CAAoB,CAAC;gBACxD,aAAa,CAAC,UAAU,IAAI,KAAK,CAAC,GAAG,CAAC;gBAEtC,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;SAI7B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,CAAoB,CAAC;gBACxD,aAAa,CAAC,YAAY,IAAI,KAAK,CAAC,GAAG,CAAC;gBAExC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;SAG9B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,CAAoB,CAAC;gBACxD,aAAa,CAAC,eAAe,IAAI,MAAM,CAAC,GAAG,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;QAED,uEAAuE;QACvE,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;SAW3B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,EAAE,GAAG,CAAC,0BAA0B,EAAE,GAAG,CAAC,UAAU,CAAoB,CAAC;gBACxG,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;SAW9B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,aAAa,OAAO,EAAE,GAAG,CAAC,0BAA0B,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;gBACrF,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,sDAAuD,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACjG,CAAC;QAED,2DAA2D;QAC3D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IACzG,CAAC;IAED,6DAA6D;IAE7D,aAAa,CAAC,MAAe;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QAErD,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;SAM3B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,cAAc,OAAO,EAAE,GAAG,CAAC,UAAU,CAAoB,CAAC;gBACzE,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;SAM9B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,cAAc,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;gBACtD,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oDAAqD,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,wDAAwD;QACxD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,CAAC;IAED,6DAA6D;IAE7D,aAAa,CAAC,MAAe;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAE1C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;SAM3B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,cAAc,OAAO,EAAE,GAAG,CAAC,UAAU,CAAoB,CAAC;gBACzE,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;SAM9B,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,cAAc,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;gBACtD,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yCAA0C,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QACpF,CAAC;QAED,4BAA4B;QAC5B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,CAAC;IAED,6DAA6D;IAE7D,SAAS;QACP,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE;SAC3B,CAAC;IACJ,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,aAAa;QACX,MAAM,MAAM,GAAG,CAAC,aAAa,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,UAAU,CAAC,CAAC;QAC3F,MAAM,KAAK,GAAoB,EAAE,CAAC;QAElC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,yBAAyB;YACzB,MAAM,QAAQ,GAAG,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,EAAE;gBAC3C,CAAC,CAAC,KAAK,KAAK,uBAAuB,CAAC,CAAC,CAAC,CAAC;oBACvC,CAAC,CAAC,KAAK,KAAK,qBAAqB,CAAC,CAAC,CAAC,CAAC;wBACrC,CAAC,CAAC,CAAC,CAAC;YACN,KAAK,CAAC,IAAI,CAAC;gBACT,KAAK;gBACL,QAAQ;gBACR,WAAW,EAAE,CAAC,CAAC,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;aACtD,CAAC,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,6DAA6D;IAErD,SAAS,CAAC,KAAa;QAC7B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC,GAAG,EAAiC,CAAC;YACzG,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED