@icex-labs/openclaw-memory-engine 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +254 -0
- package/extras/auto-consolidation-crons.json +37 -0
- package/extras/memory-maintenance.sh +176 -0
- package/index.js +626 -0
- package/lib/archival.js +54 -0
- package/lib/backup.js +99 -0
- package/lib/consolidate.js +102 -0
- package/lib/core.js +76 -0
- package/lib/dashboard.js +235 -0
- package/lib/dedup.js +68 -0
- package/lib/embedding.js +70 -0
- package/lib/episodes.js +133 -0
- package/lib/graph.js +148 -0
- package/lib/paths.js +80 -0
- package/lib/reflection.js +188 -0
- package/lib/search.js +90 -0
- package/lib/store-sqlite.js +422 -0
- package/openclaw.plugin.json +23 -0
- package/package.json +40 -0
- package/setup.sh +368 -0
package/lib/episodes.js
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Episodic memory: conversation-level summaries with decisions, mood, and topics.
|
|
3
|
+
* Enables "what did we discuss last time about X?" queries.
|
|
4
|
+
*
|
|
5
|
+
* Storage: memory/episodes.jsonl — one episode per line.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { readFileSync, appendFileSync, existsSync, mkdirSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { episodesPath } from "./paths.js";
|
|
11
|
+
import { getEmbedding, cosineSimilarity, loadEmbeddingCache, saveEmbeddingCache } from "./embedding.js";
|
|
12
|
+
|
|
13
|
+
// ─── In-memory cache ───
|
|
14
|
+
|
|
15
|
+
const cache = new Map();
|
|
16
|
+
|
|
17
|
+
export function loadEpisodes(ws) {
|
|
18
|
+
if (cache.has(ws) && cache.get(ws).loaded) return cache.get(ws).episodes;
|
|
19
|
+
const p = episodesPath(ws);
|
|
20
|
+
let episodes = [];
|
|
21
|
+
if (existsSync(p)) {
|
|
22
|
+
episodes = readFileSync(p, "utf-8")
|
|
23
|
+
.trim().split("\n").filter(Boolean)
|
|
24
|
+
.map((l) => { try { return JSON.parse(l); } catch { return null; } })
|
|
25
|
+
.filter(Boolean);
|
|
26
|
+
}
|
|
27
|
+
cache.set(ws, { episodes, loaded: true });
|
|
28
|
+
return episodes;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Save a conversation episode.
|
|
33
|
+
* @returns {object} the saved episode record
|
|
34
|
+
*/
|
|
35
|
+
export function saveEpisode(ws, { summary, decisions = [], mood = "", topics = [], participants = [], duration_minutes = null }) {
|
|
36
|
+
const episode = {
|
|
37
|
+
id: `ep-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
38
|
+
type: "episode",
|
|
39
|
+
ts: new Date().toISOString(),
|
|
40
|
+
summary,
|
|
41
|
+
decisions,
|
|
42
|
+
mood,
|
|
43
|
+
topics,
|
|
44
|
+
participants,
|
|
45
|
+
duration_minutes,
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
mkdirSync(join(ws, "memory"), { recursive: true });
|
|
49
|
+
appendFileSync(episodesPath(ws), JSON.stringify(episode) + "\n", "utf-8");
|
|
50
|
+
|
|
51
|
+
if (cache.has(ws) && cache.get(ws).loaded) {
|
|
52
|
+
cache.get(ws).episodes.push(episode);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return episode;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Search episodes by query (hybrid: keyword + semantic) or get recent N.
|
|
60
|
+
* @param {string} [query] - search query (if null, returns recent episodes)
|
|
61
|
+
* @param {number} [lastN=5] - number of results
|
|
62
|
+
* @returns {Promise<object[]>}
|
|
63
|
+
*/
|
|
64
|
+
export async function recallEpisodes(ws, query = null, lastN = 5) {
|
|
65
|
+
const episodes = loadEpisodes(ws);
|
|
66
|
+
if (episodes.length === 0) return [];
|
|
67
|
+
|
|
68
|
+
// If no query, return most recent
|
|
69
|
+
if (!query) {
|
|
70
|
+
return episodes.slice(-lastN).reverse();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const queryLower = query.toLowerCase();
|
|
74
|
+
const queryTerms = queryLower.split(/\s+/).filter((t) => t.length > 1);
|
|
75
|
+
|
|
76
|
+
// Try semantic search
|
|
77
|
+
const queryEmb = await getEmbedding(query);
|
|
78
|
+
const embCache = loadEmbeddingCache(ws);
|
|
79
|
+
|
|
80
|
+
const scored = episodes.map((ep) => {
|
|
81
|
+
// Build searchable text from all episode fields
|
|
82
|
+
const text = [
|
|
83
|
+
ep.summary || "",
|
|
84
|
+
...(ep.decisions || []),
|
|
85
|
+
...(ep.topics || []),
|
|
86
|
+
ep.mood || "",
|
|
87
|
+
].join(" ").toLowerCase();
|
|
88
|
+
|
|
89
|
+
// Keyword score
|
|
90
|
+
let kwScore = 0;
|
|
91
|
+
for (const term of queryTerms) {
|
|
92
|
+
if (text.includes(term)) kwScore += 1;
|
|
93
|
+
}
|
|
94
|
+
if (queryTerms.length > 1 && text.includes(queryLower)) kwScore += 3;
|
|
95
|
+
|
|
96
|
+
// Semantic score
|
|
97
|
+
let semScore = 0;
|
|
98
|
+
if (queryEmb && ep.id && embCache[ep.id]) {
|
|
99
|
+
semScore = Math.max(0, cosineSimilarity(queryEmb, embCache[ep.id]));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// Recency bonus
|
|
103
|
+
let recencyScore = 0;
|
|
104
|
+
if (ep.ts) {
|
|
105
|
+
const ageDays = (Date.now() - new Date(ep.ts).getTime()) / 86400000;
|
|
106
|
+
recencyScore = Math.max(0, 1 - ageDays / 90); // episodes decay faster (90 days)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const total = kwScore * 2 + semScore * 5 + recencyScore;
|
|
110
|
+
return total > 0 ? { episode: ep, score: total } : null;
|
|
111
|
+
}).filter(Boolean);
|
|
112
|
+
|
|
113
|
+
scored.sort((a, b) => b.score - a.score);
|
|
114
|
+
return scored.slice(0, lastN).map((s) => s.episode);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Index episode embedding for semantic search.
|
|
119
|
+
*/
|
|
120
|
+
export async function indexEpisodeEmbedding(ws, episode) {
|
|
121
|
+
const text = [
|
|
122
|
+
episode.summary,
|
|
123
|
+
...(episode.decisions || []),
|
|
124
|
+
...(episode.topics || []),
|
|
125
|
+
].filter(Boolean).join(" ");
|
|
126
|
+
|
|
127
|
+
const emb = await getEmbedding(text);
|
|
128
|
+
if (emb) {
|
|
129
|
+
const embCache = loadEmbeddingCache(ws);
|
|
130
|
+
embCache[episode.id] = emb;
|
|
131
|
+
saveEmbeddingCache(ws);
|
|
132
|
+
}
|
|
133
|
+
}
|
package/lib/graph.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Knowledge Graph: triple store (subject, relation, object).
|
|
3
|
+
* Enables relational queries like "who is George's doctor" or "what treats 荨麻疹".
|
|
4
|
+
*
|
|
5
|
+
* Storage: memory/graph.jsonl — one triple per line.
|
|
6
|
+
* Auto-extraction: pattern-based extraction from archival insert content.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { readFileSync, writeFileSync, appendFileSync, existsSync, mkdirSync } from "node:fs";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { graphPath } from "./paths.js";
|
|
12
|
+
|
|
13
|
+
// ─── In-memory cache ───
|
|
14
|
+
|
|
15
|
+
const cache = new Map(); // ws → { triples: [], loaded: false }
|
|
16
|
+
|
|
17
|
+
export function loadGraph(ws) {
|
|
18
|
+
if (cache.has(ws) && cache.get(ws).loaded) return cache.get(ws).triples;
|
|
19
|
+
const p = graphPath(ws);
|
|
20
|
+
let triples = [];
|
|
21
|
+
if (existsSync(p)) {
|
|
22
|
+
triples = readFileSync(p, "utf-8")
|
|
23
|
+
.trim().split("\n").filter(Boolean)
|
|
24
|
+
.map((l) => { try { return JSON.parse(l); } catch { return null; } })
|
|
25
|
+
.filter(Boolean);
|
|
26
|
+
}
|
|
27
|
+
cache.set(ws, { triples, loaded: true });
|
|
28
|
+
return triples;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function addTriple(ws, subject, relation, object, sourceId = null) {
|
|
32
|
+
const triples = loadGraph(ws);
|
|
33
|
+
|
|
34
|
+
// Deduplicate: same subject+relation+object
|
|
35
|
+
const exists = triples.some(
|
|
36
|
+
(t) => t.s === subject && t.r === relation && t.o === object,
|
|
37
|
+
);
|
|
38
|
+
if (exists) return null;
|
|
39
|
+
|
|
40
|
+
const triple = {
|
|
41
|
+
id: `tri-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
42
|
+
s: subject,
|
|
43
|
+
r: relation,
|
|
44
|
+
o: object,
|
|
45
|
+
ts: new Date().toISOString(),
|
|
46
|
+
source: sourceId,
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
mkdirSync(join(ws, "memory"), { recursive: true });
|
|
50
|
+
appendFileSync(graphPath(ws), JSON.stringify(triple) + "\n", "utf-8");
|
|
51
|
+
triples.push(triple);
|
|
52
|
+
return triple;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function removeTriple(ws, tripleId) {
|
|
56
|
+
const triples = loadGraph(ws);
|
|
57
|
+
const idx = triples.findIndex((t) => t.id === tripleId);
|
|
58
|
+
if (idx === -1) return false;
|
|
59
|
+
triples.splice(idx, 1);
|
|
60
|
+
writeFileSync(graphPath(ws), triples.map((t) => JSON.stringify(t)).join("\n") + (triples.length ? "\n" : ""), "utf-8");
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Query the graph from a starting entity, optionally filtering by relation.
|
|
66
|
+
* @param {string} entity - starting node
|
|
67
|
+
* @param {string} [relation] - optional relation filter
|
|
68
|
+
* @param {number} [depth=2] - traversal depth
|
|
69
|
+
* @returns {Array<{ path: string[], triple: object }>}
|
|
70
|
+
*/
|
|
71
|
+
export function queryGraph(ws, entity, relation = null, depth = 2) {
|
|
72
|
+
const triples = loadGraph(ws);
|
|
73
|
+
const entityLower = entity.toLowerCase();
|
|
74
|
+
const results = [];
|
|
75
|
+
const visited = new Set();
|
|
76
|
+
|
|
77
|
+
function traverse(current, currentDepth, path) {
|
|
78
|
+
if (currentDepth > depth) return;
|
|
79
|
+
const key = `${current}:${currentDepth}`;
|
|
80
|
+
if (visited.has(key)) return;
|
|
81
|
+
visited.add(key);
|
|
82
|
+
|
|
83
|
+
const currentLower = current.toLowerCase();
|
|
84
|
+
for (const t of triples) {
|
|
85
|
+
// Forward: subject matches
|
|
86
|
+
if (t.s.toLowerCase() === currentLower) {
|
|
87
|
+
if (relation && t.r.toLowerCase() !== relation.toLowerCase()) continue;
|
|
88
|
+
results.push({ path: [...path, `--${t.r}-->`], node: t.o, triple: t });
|
|
89
|
+
traverse(t.o, currentDepth + 1, [...path, `--${t.r}-->`, t.o]);
|
|
90
|
+
}
|
|
91
|
+
// Reverse: object matches
|
|
92
|
+
if (t.o.toLowerCase() === currentLower) {
|
|
93
|
+
if (relation && t.r.toLowerCase() !== relation.toLowerCase()) continue;
|
|
94
|
+
results.push({ path: [...path, `<--${t.r}--`], node: t.s, triple: t });
|
|
95
|
+
traverse(t.s, currentDepth + 1, [...path, `<--${t.r}--`, t.s]);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
traverse(entity, 1, [entity]);
|
|
101
|
+
return results;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ─── Auto-extraction patterns ───
|
|
105
|
+
|
|
106
|
+
const EXTRACTION_PATTERNS = [
|
|
107
|
+
// "X's doctor is Y" / "X的医生是Y"
|
|
108
|
+
{ re: /(.+?)(?:'s|的)\s*(?:doctor|医生|主治医生)\s*(?:is|是|为)\s*(.+)/i, r: "has_doctor" },
|
|
109
|
+
// "X lives in Y" / "X住在Y"
|
|
110
|
+
{ re: /(.+?)\s*(?:lives? in|住在|位于)\s*(.+)/i, r: "lives_in" },
|
|
111
|
+
// "X works at Y" / "X在Y工作"
|
|
112
|
+
{ re: /(.+?)\s*(?:works? at|在(.+?)工作)/i, r: "works_at" },
|
|
113
|
+
// "X has condition Y" / disease / 疾病
|
|
114
|
+
{ re: /(.+?)\s*(?:has|有|患有)\s*(?:chronic |慢性)?\s*(.+?(?:症|病|urticaria|condition|disease))/i, r: "has_condition" },
|
|
115
|
+
// "X treated by/takes Y" / 用药
|
|
116
|
+
{ re: /(.+?)\s*(?:takes?|服用|用药|treated (?:by|with))\s*(.+)/i, r: "treated_by" },
|
|
117
|
+
// "X's wife/husband is Y"
|
|
118
|
+
{ re: /(.+?)(?:'s|的)\s*(wife|husband|妻子|丈夫|老婆|老公)\s*(?:is|是|为)\s*(.+)/i, r: "spouse", triple: true },
|
|
119
|
+
// "X's son/daughter is Y"
|
|
120
|
+
{ re: /(.+?)(?:'s|的)\s*(son|daughter|儿子|女儿)\s*(?:is|是|为)\s*(.+)/i, r: "has_child", triple: true },
|
|
121
|
+
// "X costs/price Y" / 定价
|
|
122
|
+
{ re: /(.+?)\s*(?:costs?|定价|售价|price[ds]?\s*(?:at)?)\s*\$?([\d,.]+)/i, r: "price" },
|
|
123
|
+
// "X's lawyer is Y"
|
|
124
|
+
{ re: /(.+?)(?:'s|的)\s*(?:lawyer|律师|attorney)\s*(?:is|是|为)\s*(.+)/i, r: "has_lawyer" },
|
|
125
|
+
// "X owns Y" / 拥有
|
|
126
|
+
{ re: /(.+?)\s*(?:owns?|拥有|有一辆|drives?)\s*(.+)/i, r: "owns" },
|
|
127
|
+
// "X studies/attends Y" / 就读
|
|
128
|
+
{ re: /(.+?)\s*(?:attends?|studies? at|就读于?|在(.+?)(?:上学|读书))/i, r: "attends" },
|
|
129
|
+
];
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Extract triples from a text string.
|
|
133
|
+
* @returns {Array<{ s: string, r: string, o: string }>}
|
|
134
|
+
*/
|
|
135
|
+
export function extractTriples(text) {
|
|
136
|
+
const results = [];
|
|
137
|
+
for (const pat of EXTRACTION_PATTERNS) {
|
|
138
|
+
const m = text.match(pat.re);
|
|
139
|
+
if (m) {
|
|
140
|
+
const s = (m[1] || "").trim();
|
|
141
|
+
const o = (pat.triple ? m[3] : m[2] || "").trim();
|
|
142
|
+
if (s && o && s.length < 50 && o.length < 80) {
|
|
143
|
+
results.push({ s, r: pat.r, o });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return results;
|
|
148
|
+
}
|
package/lib/paths.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared path resolution and config helpers.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_CORE_SIZE_LIMIT = 3072;
|
|
8
|
+
export const DEFAULT_TOP_K = 5;
|
|
9
|
+
export const MAX_TOP_K = 20;
|
|
10
|
+
export const EMBEDDING_MODEL = "text-embedding-3-small";
|
|
11
|
+
export const EMBEDDING_DIM = 512;
|
|
12
|
+
export const DEDUP_SIMILARITY_THRESHOLD = 0.92;
|
|
13
|
+
|
|
14
|
+
import { readFileSync as _readFileSync, existsSync as _existsSync } from "node:fs";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Agent→workspace mapping, loaded once from openclaw.json.
|
|
18
|
+
* Since OpenClaw v2026.3.x doesn't pass agentId/workspaceDir to plugin tools,
|
|
19
|
+
* we read the config and let agents pass their ID via tool params.
|
|
20
|
+
*/
|
|
21
|
+
let _agentMap = null;
|
|
22
|
+
|
|
23
|
+
function loadAgentMap() {
|
|
24
|
+
if (_agentMap) return _agentMap;
|
|
25
|
+
_agentMap = {};
|
|
26
|
+
try {
|
|
27
|
+
const configPath = join(process.env.HOME || "/tmp", ".openclaw", "openclaw.json");
|
|
28
|
+
if (_existsSync(configPath)) {
|
|
29
|
+
const cfg = JSON.parse(_readFileSync(configPath, "utf-8"));
|
|
30
|
+
const defaultWs = cfg?.agents?.defaults?.workspace || join(process.env.HOME || "/tmp", ".openclaw", "workspace");
|
|
31
|
+
_agentMap._default = defaultWs;
|
|
32
|
+
for (const agent of cfg?.agents?.list || []) {
|
|
33
|
+
_agentMap[agent.id] = agent.workspace || defaultWs;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
} catch { /* ignore */ }
|
|
37
|
+
return _agentMap;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Resolve workspace.
|
|
42
|
+
* @param {object} ctx - OpenClaw tool context
|
|
43
|
+
* @param {string} [agentId] - optional agent ID from tool params (workaround for OpenClaw not passing ctx)
|
|
44
|
+
*/
|
|
45
|
+
export function resolveWorkspace(ctx, agentId = null) {
|
|
46
|
+
// 1. ctx.workspaceDir (future OpenClaw versions)
|
|
47
|
+
if (ctx?.workspaceDir) return ctx.workspaceDir;
|
|
48
|
+
// 2. Explicit agentId → lookup from config
|
|
49
|
+
const aid = agentId || ctx?.agentId;
|
|
50
|
+
if (aid) {
|
|
51
|
+
const map = loadAgentMap();
|
|
52
|
+
if (map[aid]) return map[aid];
|
|
53
|
+
}
|
|
54
|
+
// 3. Plugin config workspace
|
|
55
|
+
if (ctx?.config?.workspace) return ctx.config.workspace;
|
|
56
|
+
// 4. Env
|
|
57
|
+
if (process.env.OPENCLAW_WORKSPACE) return process.env.OPENCLAW_WORKSPACE;
|
|
58
|
+
// 5. Default
|
|
59
|
+
return join(process.env.HOME || "/tmp", ".openclaw", "workspace");
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function getCoreSizeLimit(ctx) {
|
|
63
|
+
return ctx?.config?.coreSizeLimit || DEFAULT_CORE_SIZE_LIMIT;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function corePath(ws) { return join(ws, "memory", "core.json"); }
|
|
67
|
+
export function archivalPath(ws) { return join(ws, "memory", "archival.jsonl"); }
|
|
68
|
+
export function embeddingCachePath(ws) { return join(ws, "memory", "archival.embeddings.json"); }
|
|
69
|
+
export function graphPath(ws) { return join(ws, "memory", "graph.jsonl"); }
|
|
70
|
+
export function episodesPath(ws) { return join(ws, "memory", "episodes.jsonl"); }
|
|
71
|
+
|
|
72
|
+
export const DEFAULT_IMPORTANCE = 5;
|
|
73
|
+
export const FORGETTING_DECAY_RATE = 0.01; // importance decays by this per day without access
|
|
74
|
+
export const FORGETTING_THRESHOLD = 1.0; // below this = candidate for archiving
|
|
75
|
+
|
|
76
|
+
// Multi-agent sharing (default: on for single-user multi-agent setups)
|
|
77
|
+
// Set to false in config for privacy-sensitive scenarios (e.g., separate users per agent)
|
|
78
|
+
export function isSharingEnabled(ctx) {
|
|
79
|
+
return ctx?.config?.sharing !== false; // default true, explicitly set false to disable
|
|
80
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reflective memory: statistical analysis over recent archival + episodes
|
|
3
|
+
* to surface behavioral patterns, topic trends, and emotional shifts.
|
|
4
|
+
*
|
|
5
|
+
* This module does data analysis only — pattern interpretation is left
|
|
6
|
+
* to the agent (which has LLM reasoning the plugin doesn't).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { loadArchival } from "./archival.js";
|
|
10
|
+
import { loadEpisodes } from "./episodes.js";
|
|
11
|
+
import { loadGraph } from "./graph.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Analyze recent memory for patterns and trends.
|
|
15
|
+
* @param {string} ws - workspace path
|
|
16
|
+
* @param {number} [windowDays=7] - analysis window in days
|
|
17
|
+
* @returns {object} structured analysis
|
|
18
|
+
*/
|
|
19
|
+
export function analyzePatterns(ws, windowDays = 7) {
|
|
20
|
+
const allRecords = loadArchival(ws);
|
|
21
|
+
const allEpisodes = loadEpisodes(ws);
|
|
22
|
+
const graph = loadGraph(ws);
|
|
23
|
+
|
|
24
|
+
const cutoff = Date.now() - windowDays * 86400000;
|
|
25
|
+
|
|
26
|
+
const recentRecords = allRecords.filter(
|
|
27
|
+
(r) => r.ts && new Date(r.ts).getTime() > cutoff,
|
|
28
|
+
);
|
|
29
|
+
const recentEpisodes = allEpisodes.filter(
|
|
30
|
+
(e) => e.ts && new Date(e.ts).getTime() > cutoff,
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
// ─── 1. Topic frequency ───
|
|
34
|
+
const topicCounts = {};
|
|
35
|
+
for (const r of recentRecords) {
|
|
36
|
+
for (const tag of r.tags || []) {
|
|
37
|
+
topicCounts[tag] = (topicCounts[tag] || 0) + 1;
|
|
38
|
+
}
|
|
39
|
+
if (r.entity) {
|
|
40
|
+
topicCounts[`entity:${r.entity}`] = (topicCounts[`entity:${r.entity}`] || 0) + 1;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
for (const ep of recentEpisodes) {
|
|
44
|
+
for (const topic of ep.topics || []) {
|
|
45
|
+
topicCounts[`episode:${topic}`] = (topicCounts[`episode:${topic}`] || 0) + 1;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
const topTopics = Object.entries(topicCounts)
|
|
49
|
+
.sort((a, b) => b[1] - a[1])
|
|
50
|
+
.slice(0, 10);
|
|
51
|
+
|
|
52
|
+
// ─── 2. Time-of-day distribution ───
|
|
53
|
+
const hourBuckets = { morning: 0, afternoon: 0, evening: 0, night: 0 };
|
|
54
|
+
for (const r of recentRecords) {
|
|
55
|
+
if (!r.ts) continue;
|
|
56
|
+
const hour = new Date(r.ts).getHours();
|
|
57
|
+
if (hour >= 6 && hour < 12) hourBuckets.morning++;
|
|
58
|
+
else if (hour >= 12 && hour < 18) hourBuckets.afternoon++;
|
|
59
|
+
else if (hour >= 18 && hour < 23) hourBuckets.evening++;
|
|
60
|
+
else hourBuckets.night++;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ─── 3. Mood trend (from episodes) ───
|
|
64
|
+
const moodCounts = {};
|
|
65
|
+
for (const ep of recentEpisodes) {
|
|
66
|
+
if (ep.mood) {
|
|
67
|
+
moodCounts[ep.mood] = (moodCounts[ep.mood] || 0) + 1;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// ─── 4. Importance distribution ───
|
|
72
|
+
const importanceHigh = recentRecords.filter((r) => (r.importance || 5) >= 7).length;
|
|
73
|
+
const importanceLow = recentRecords.filter((r) => (r.importance || 5) <= 3).length;
|
|
74
|
+
const importanceMid = recentRecords.length - importanceHigh - importanceLow;
|
|
75
|
+
|
|
76
|
+
// ─── 5. Decision velocity (from episodes) ───
|
|
77
|
+
const totalDecisions = recentEpisodes.reduce(
|
|
78
|
+
(sum, ep) => sum + (ep.decisions?.length || 0), 0,
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
// ─── 6. Graph growth ───
|
|
82
|
+
const recentTriples = graph.filter(
|
|
83
|
+
(t) => t.ts && new Date(t.ts).getTime() > cutoff,
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
// ─── 7. Neglected entities (in graph but not accessed recently) ───
|
|
87
|
+
const graphEntities = new Set();
|
|
88
|
+
for (const t of graph) {
|
|
89
|
+
graphEntities.add(t.s);
|
|
90
|
+
graphEntities.add(t.o);
|
|
91
|
+
}
|
|
92
|
+
const recentEntities = new Set();
|
|
93
|
+
for (const r of recentRecords) {
|
|
94
|
+
if (r.entity) recentEntities.add(r.entity);
|
|
95
|
+
}
|
|
96
|
+
const neglected = [...graphEntities].filter((e) => !recentEntities.has(e));
|
|
97
|
+
|
|
98
|
+
// ─── 8. Forgetting candidates ───
|
|
99
|
+
const forgettingCandidates = allRecords.filter((r) => {
|
|
100
|
+
const importance = r.importance || 5;
|
|
101
|
+
const daysSinceAccess = r.last_accessed
|
|
102
|
+
? (Date.now() - new Date(r.last_accessed).getTime()) / 86400000
|
|
103
|
+
: (Date.now() - new Date(r.ts || Date.now()).getTime()) / 86400000;
|
|
104
|
+
const effective = importance * Math.exp(-0.01 * daysSinceAccess);
|
|
105
|
+
return effective < 1.0;
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
window_days: windowDays,
|
|
110
|
+
period: {
|
|
111
|
+
from: new Date(cutoff).toISOString().slice(0, 10),
|
|
112
|
+
to: new Date().toISOString().slice(0, 10),
|
|
113
|
+
},
|
|
114
|
+
activity: {
|
|
115
|
+
new_facts: recentRecords.length,
|
|
116
|
+
new_episodes: recentEpisodes.length,
|
|
117
|
+
new_graph_triples: recentTriples.length,
|
|
118
|
+
total_decisions: totalDecisions,
|
|
119
|
+
},
|
|
120
|
+
top_topics: topTopics,
|
|
121
|
+
time_distribution: hourBuckets,
|
|
122
|
+
mood_trend: moodCounts,
|
|
123
|
+
importance_distribution: {
|
|
124
|
+
high: importanceHigh,
|
|
125
|
+
medium: importanceMid,
|
|
126
|
+
low: importanceLow,
|
|
127
|
+
},
|
|
128
|
+
health: {
|
|
129
|
+
total_archival: allRecords.length,
|
|
130
|
+
total_episodes: allEpisodes.length,
|
|
131
|
+
total_graph: graph.length,
|
|
132
|
+
neglected_entities: neglected.slice(0, 10),
|
|
133
|
+
forgetting_candidates: forgettingCandidates.length,
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Format analysis into a human-readable report for the agent.
|
|
140
|
+
*/
|
|
141
|
+
export function formatReflection(analysis) {
|
|
142
|
+
const lines = [
|
|
143
|
+
`📊 Memory Reflection (${analysis.period.from} → ${analysis.period.to}, ${analysis.window_days}d window)`,
|
|
144
|
+
``,
|
|
145
|
+
`Activity:`,
|
|
146
|
+
` New facts: ${analysis.activity.new_facts}`,
|
|
147
|
+
` New episodes: ${analysis.activity.new_episodes}`,
|
|
148
|
+
` New graph relations: ${analysis.activity.new_graph_triples}`,
|
|
149
|
+
` Decisions made: ${analysis.activity.total_decisions}`,
|
|
150
|
+
];
|
|
151
|
+
|
|
152
|
+
if (analysis.top_topics.length > 0) {
|
|
153
|
+
lines.push(``, `Top topics:`);
|
|
154
|
+
for (const [topic, count] of analysis.top_topics) {
|
|
155
|
+
lines.push(` ${topic}: ${count}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const td = analysis.time_distribution;
|
|
160
|
+
const peakTime = Object.entries(td).sort((a, b) => b[1] - a[1])[0];
|
|
161
|
+
if (peakTime && peakTime[1] > 0) {
|
|
162
|
+
lines.push(``, `Time pattern: most active during ${peakTime[0]} (${peakTime[1]} events)`);
|
|
163
|
+
lines.push(` morning=${td.morning} afternoon=${td.afternoon} evening=${td.evening} night=${td.night}`);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if (Object.keys(analysis.mood_trend).length > 0) {
|
|
167
|
+
lines.push(``, `Mood trend:`);
|
|
168
|
+
for (const [mood, count] of Object.entries(analysis.mood_trend)) {
|
|
169
|
+
lines.push(` ${mood}: ${count}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const imp = analysis.importance_distribution;
|
|
174
|
+
lines.push(``, `Importance: high(≥7)=${imp.high} medium=${imp.medium} low(≤3)=${imp.low}`);
|
|
175
|
+
|
|
176
|
+
if (analysis.health.neglected_entities.length > 0) {
|
|
177
|
+
lines.push(``, `Neglected entities (in graph but no recent activity):`);
|
|
178
|
+
lines.push(` ${analysis.health.neglected_entities.join(", ")}`);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (analysis.health.forgetting_candidates > 0) {
|
|
182
|
+
lines.push(``, `⚠️ ${analysis.health.forgetting_candidates} facts below forgetting threshold — consider archival_deduplicate or cleanup`);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
lines.push(``, `Totals: ${analysis.health.total_archival} facts, ${analysis.health.total_episodes} episodes, ${analysis.health.total_graph} graph triples`);
|
|
186
|
+
|
|
187
|
+
return lines.join("\n");
|
|
188
|
+
}
|
package/lib/search.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hybrid search: keyword + embedding cosine similarity + recency + access decay.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import { loadArchival, rewriteArchival } from "./archival.js";
|
|
6
|
+
import { loadEmbeddingCache, getEmbedding, cosineSimilarity } from "./embedding.js";
|
|
7
|
+
import { DEFAULT_IMPORTANCE, FORGETTING_DECAY_RATE } from "./paths.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Search archival records with hybrid scoring.
|
|
11
|
+
* @param {string} ws - workspace path
|
|
12
|
+
* @param {string} query - search query
|
|
13
|
+
* @param {number} topK - max results
|
|
14
|
+
* @returns {Promise<object[]>} matched records
|
|
15
|
+
*/
|
|
16
|
+
export async function hybridSearch(ws, query, topK) {
|
|
17
|
+
const records = loadArchival(ws);
|
|
18
|
+
if (records.length === 0) return [];
|
|
19
|
+
|
|
20
|
+
const queryLower = query.toLowerCase();
|
|
21
|
+
const queryTerms = queryLower.split(/\s+/).filter((t) => t.length > 1);
|
|
22
|
+
|
|
23
|
+
const queryEmb = await getEmbedding(query);
|
|
24
|
+
const embCache = loadEmbeddingCache(ws);
|
|
25
|
+
|
|
26
|
+
const scored = records
|
|
27
|
+
.map((record) => {
|
|
28
|
+
const text = [record.content || "", record.entity || "", ...(record.tags || [])]
|
|
29
|
+
.join(" ")
|
|
30
|
+
.toLowerCase();
|
|
31
|
+
|
|
32
|
+
// Keyword score
|
|
33
|
+
let kwScore = 0;
|
|
34
|
+
for (const term of queryTerms) {
|
|
35
|
+
if (text.includes(term)) kwScore += 1;
|
|
36
|
+
}
|
|
37
|
+
if (queryTerms.length > 1 && text.includes(queryLower)) kwScore += 3;
|
|
38
|
+
|
|
39
|
+
// Semantic score
|
|
40
|
+
let semScore = 0;
|
|
41
|
+
if (queryEmb && record.id) {
|
|
42
|
+
const recEmb = embCache[record.id] || null;
|
|
43
|
+
if (recEmb) {
|
|
44
|
+
semScore = Math.max(0, cosineSimilarity(queryEmb, recEmb));
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Recency bonus (0–1, decays over 1 year)
|
|
49
|
+
let recencyScore = 0;
|
|
50
|
+
if (record.ts) {
|
|
51
|
+
const ageDays = (Date.now() - new Date(record.ts).getTime()) / 86400000;
|
|
52
|
+
recencyScore = Math.max(0, 1 - ageDays / 365);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Access decay bonus (0–0.5)
|
|
56
|
+
let accessScore = 0;
|
|
57
|
+
if (record.last_accessed) {
|
|
58
|
+
const accessAgeDays =
|
|
59
|
+
(Date.now() - new Date(record.last_accessed).getTime()) / 86400000;
|
|
60
|
+
accessScore = Math.max(0, 0.5 - accessAgeDays / 180);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// Importance score with forgetting curve
|
|
64
|
+
const importance = record.importance ?? DEFAULT_IMPORTANCE;
|
|
65
|
+
const daysSinceAccess = record.last_accessed
|
|
66
|
+
? (Date.now() - new Date(record.last_accessed).getTime()) / 86400000
|
|
67
|
+
: (Date.now() - new Date(record.ts || Date.now()).getTime()) / 86400000;
|
|
68
|
+
const effectiveImportance = importance * Math.exp(-FORGETTING_DECAY_RATE * daysSinceAccess);
|
|
69
|
+
const importanceScore = effectiveImportance * 0.5;
|
|
70
|
+
|
|
71
|
+
const total = kwScore * 2 + semScore * 5 + recencyScore + accessScore + importanceScore;
|
|
72
|
+
return total > 0 ? { record, score: total } : null;
|
|
73
|
+
})
|
|
74
|
+
.filter(Boolean);
|
|
75
|
+
|
|
76
|
+
scored.sort((a, b) => b.score - a.score);
|
|
77
|
+
const results = scored.slice(0, topK);
|
|
78
|
+
|
|
79
|
+
// Update access tracking
|
|
80
|
+
const now = new Date().toISOString();
|
|
81
|
+
let dirty = false;
|
|
82
|
+
for (const s of results) {
|
|
83
|
+
s.record.last_accessed = now;
|
|
84
|
+
s.record.access_count = (s.record.access_count || 0) + 1;
|
|
85
|
+
dirty = true;
|
|
86
|
+
}
|
|
87
|
+
if (dirty) rewriteArchival(ws, records);
|
|
88
|
+
|
|
89
|
+
return results.map((s) => s.record);
|
|
90
|
+
}
|