linksee-memory 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +144 -0
- package/dist/bin/import-sessions.d.ts +2 -0
- package/dist/bin/import-sessions.js +258 -0
- package/dist/bin/sync-session.d.ts +2 -0
- package/dist/bin/sync-session.js +97 -0
- package/dist/db/migrate.d.ts +4 -0
- package/dist/db/migrate.js +48 -0
- package/dist/db/schema.sql +199 -0
- package/dist/lib/consolidate.d.ts +12 -0
- package/dist/lib/consolidate.js +148 -0
- package/dist/lib/file-chunker.d.ts +12 -0
- package/dist/lib/file-chunker.js +250 -0
- package/dist/lib/forgetting.d.ts +12 -0
- package/dist/lib/forgetting.js +31 -0
- package/dist/lib/heat-index.d.ts +13 -0
- package/dist/lib/heat-index.js +29 -0
- package/dist/lib/momentum.d.ts +19 -0
- package/dist/lib/momentum.js +75 -0
- package/dist/lib/session-extractor.d.ts +34 -0
- package/dist/lib/session-extractor.js +263 -0
- package/dist/lib/session-parser.d.ts +40 -0
- package/dist/lib/session-parser.js +303 -0
- package/dist/mcp/read-smart.d.ts +5 -0
- package/dist/mcp/read-smart.js +132 -0
- package/dist/mcp/server.d.ts +2 -0
- package/dist/mcp/server.js +449 -0
- package/package.json +67 -0
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
-- linksee-memory schema v0.0.2
|
|
2
|
+
-- Single-file SQLite store for cross-agent structured memory.
|
|
3
|
+
-- Layers: 1=facts (entities), 2=associations (edges), 3=patterns (meanings), 4=events (time-series), 5=file-state (diff cache).
|
|
4
|
+
-- v2 adds: FTS5 full-text search, consolidations audit, momentum cache on entities.
|
|
5
|
+
|
|
6
|
+
-- ============================================================
|
|
7
|
+
-- Layer 1: Facts — entities (people / companies / projects / concepts)
|
|
8
|
+
-- ============================================================
|
|
9
|
+
CREATE TABLE IF NOT EXISTS entities (
|
|
10
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
11
|
+
kind TEXT NOT NULL CHECK (kind IN ('person', 'company', 'project', 'concept', 'file', 'other')),
|
|
12
|
+
name TEXT NOT NULL,
|
|
13
|
+
canonical_key TEXT UNIQUE,
|
|
14
|
+
attributes TEXT,
|
|
15
|
+
momentum_score REAL NOT NULL DEFAULT 0.0, -- 0-10 cached; refreshed on event insert
|
|
16
|
+
momentum_at INTEGER, -- when momentum was last computed
|
|
17
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
18
|
+
updated_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
19
|
+
);
|
|
20
|
+
|
|
21
|
+
CREATE INDEX IF NOT EXISTS idx_entities_kind ON entities(kind);
|
|
22
|
+
CREATE INDEX IF NOT EXISTS idx_entities_key ON entities(canonical_key);
|
|
23
|
+
|
|
24
|
+
-- ============================================================
|
|
25
|
+
-- Layer 3: Meanings — 6-layer structured memory per entity
|
|
26
|
+
-- ============================================================
|
|
27
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
28
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
29
|
+
entity_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
|
30
|
+
layer TEXT NOT NULL CHECK (layer IN (
|
|
31
|
+
'goal', 'context', 'emotion', 'implementation', 'caveat', 'learning'
|
|
32
|
+
)),
|
|
33
|
+
content TEXT NOT NULL,
|
|
34
|
+
importance REAL NOT NULL DEFAULT 0.5,
|
|
35
|
+
protected INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
source TEXT,
|
|
37
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
38
|
+
last_accessed_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
39
|
+
access_count INTEGER NOT NULL DEFAULT 0
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
CREATE INDEX IF NOT EXISTS idx_memories_entity ON memories(entity_id);
|
|
43
|
+
CREATE INDEX IF NOT EXISTS idx_memories_layer ON memories(layer);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS idx_memories_importance ON memories(importance DESC);
|
|
45
|
+
CREATE INDEX IF NOT EXISTS idx_memories_protected ON memories(protected);
|
|
46
|
+
|
|
47
|
+
CREATE TRIGGER IF NOT EXISTS trg_protect_caveat
|
|
48
|
+
AFTER INSERT ON memories
|
|
49
|
+
WHEN NEW.layer = 'caveat'
|
|
50
|
+
BEGIN
|
|
51
|
+
UPDATE memories SET protected = 1 WHERE id = NEW.id;
|
|
52
|
+
END;
|
|
53
|
+
|
|
54
|
+
-- ============================================================
|
|
55
|
+
-- FTS5 full-text search over memory content (Day 2)
|
|
56
|
+
-- BM25-ranked retrieval; combined with heat + momentum at query time.
|
|
57
|
+
-- ============================================================
|
|
58
|
+
-- trigram tokenizer indexes 3-character substrings — works for both English (case-insensitive
|
|
59
|
+
-- via remove_diacritics) AND Japanese/CJK (no word boundaries needed).
|
|
60
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
|
61
|
+
content,
|
|
62
|
+
content='memories',
|
|
63
|
+
content_rowid='id',
|
|
64
|
+
tokenize='trigram remove_diacritics 1'
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
CREATE TRIGGER IF NOT EXISTS trg_memories_fts_ai
|
|
68
|
+
AFTER INSERT ON memories BEGIN
|
|
69
|
+
INSERT INTO memories_fts(rowid, content) VALUES (NEW.id, NEW.content);
|
|
70
|
+
END;
|
|
71
|
+
|
|
72
|
+
CREATE TRIGGER IF NOT EXISTS trg_memories_fts_ad
|
|
73
|
+
AFTER DELETE ON memories BEGIN
|
|
74
|
+
INSERT INTO memories_fts(memories_fts, rowid, content) VALUES('delete', OLD.id, OLD.content);
|
|
75
|
+
END;
|
|
76
|
+
|
|
77
|
+
CREATE TRIGGER IF NOT EXISTS trg_memories_fts_au
|
|
78
|
+
AFTER UPDATE ON memories BEGIN
|
|
79
|
+
INSERT INTO memories_fts(memories_fts, rowid, content) VALUES('delete', OLD.id, OLD.content);
|
|
80
|
+
INSERT INTO memories_fts(rowid, content) VALUES (NEW.id, NEW.content);
|
|
81
|
+
END;
|
|
82
|
+
|
|
83
|
+
-- ============================================================
|
|
84
|
+
-- Layer 2: Associations — graph edges between entities
|
|
85
|
+
-- ============================================================
|
|
86
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
87
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
88
|
+
from_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
|
89
|
+
to_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
|
|
90
|
+
relation TEXT NOT NULL,
|
|
91
|
+
weight REAL NOT NULL DEFAULT 1.0,
|
|
92
|
+
attributes TEXT,
|
|
93
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
94
|
+
UNIQUE(from_id, to_id, relation)
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_id);
|
|
98
|
+
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_id);
|
|
99
|
+
CREATE INDEX IF NOT EXISTS idx_edges_rel ON edges(relation);
|
|
100
|
+
|
|
101
|
+
-- ============================================================
|
|
102
|
+
-- Layer 4: Events — time-series log with importance markers
|
|
103
|
+
-- ============================================================
|
|
104
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
105
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
106
|
+
entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE,
|
|
107
|
+
kind TEXT NOT NULL,
|
|
108
|
+
payload TEXT,
|
|
109
|
+
occurred_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
CREATE INDEX IF NOT EXISTS idx_events_entity ON events(entity_id);
|
|
113
|
+
CREATE INDEX IF NOT EXISTS idx_events_occurred ON events(occurred_at DESC);
|
|
114
|
+
CREATE INDEX IF NOT EXISTS idx_events_kind ON events(kind);
|
|
115
|
+
|
|
116
|
+
-- ============================================================
|
|
117
|
+
-- Layer 5: File snapshots — diff cache for read_smart (Day 3)
|
|
118
|
+
-- ============================================================
|
|
119
|
+
CREATE TABLE IF NOT EXISTS file_snapshots (
|
|
120
|
+
path TEXT PRIMARY KEY,
|
|
121
|
+
content_hash TEXT NOT NULL,
|
|
122
|
+
mtime INTEGER NOT NULL,
|
|
123
|
+
size_bytes INTEGER,
|
|
124
|
+
chunks TEXT,
|
|
125
|
+
last_read_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
126
|
+
read_count INTEGER NOT NULL DEFAULT 1
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
CREATE INDEX IF NOT EXISTS idx_file_mtime ON file_snapshots(mtime);
|
|
130
|
+
|
|
131
|
+
CREATE TABLE IF NOT EXISTS file_facts (
|
|
132
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
133
|
+
file_path TEXT NOT NULL REFERENCES file_snapshots(path) ON DELETE CASCADE,
|
|
134
|
+
chunk_hash TEXT,
|
|
135
|
+
fact TEXT NOT NULL,
|
|
136
|
+
layer TEXT CHECK (layer IN ('goal', 'context', 'emotion', 'implementation', 'caveat', 'learning')),
|
|
137
|
+
extracted_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
CREATE INDEX IF NOT EXISTS idx_file_facts_path ON file_facts(file_path);
|
|
141
|
+
|
|
142
|
+
-- ============================================================
|
|
143
|
+
-- Sessions — track which agent / conversation produced memories
|
|
144
|
+
-- ============================================================
|
|
145
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
146
|
+
id TEXT PRIMARY KEY,
|
|
147
|
+
agent_kind TEXT,
|
|
148
|
+
started_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
|
149
|
+
last_seen_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
-- ============================================================
|
|
153
|
+
-- Session file edits (v3) — conversation↔file linkage.
|
|
154
|
+
-- Each row: "during session S, at turn T, memory M mentions editing file F".
|
|
155
|
+
-- This is the table that breaks the Mem0 "flat metatag" wall:
|
|
156
|
+
-- memories describe WHY, file_edits tie the WHY to concrete filesystem changes.
|
|
157
|
+
-- ============================================================
|
|
158
|
+
CREATE TABLE IF NOT EXISTS session_file_edits (
|
|
159
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
160
|
+
session_id TEXT NOT NULL, -- Claude Code session uuid
|
|
161
|
+
memory_id INTEGER REFERENCES memories(id) ON DELETE SET NULL,
|
|
162
|
+
file_path TEXT NOT NULL,
|
|
163
|
+
operation TEXT NOT NULL CHECK (operation IN ('read', 'edit', 'write', 'bash', 'other')),
|
|
164
|
+
turn_uuid TEXT,
|
|
165
|
+
context_snippet TEXT, -- 1-2 line distilled "why this edit"
|
|
166
|
+
occurred_at INTEGER NOT NULL
|
|
167
|
+
);
|
|
168
|
+
|
|
169
|
+
CREATE INDEX IF NOT EXISTS idx_sfe_session ON session_file_edits(session_id);
|
|
170
|
+
CREATE INDEX IF NOT EXISTS idx_sfe_file ON session_file_edits(file_path);
|
|
171
|
+
CREATE INDEX IF NOT EXISTS idx_sfe_memory ON session_file_edits(memory_id);
|
|
172
|
+
CREATE INDEX IF NOT EXISTS idx_sfe_when ON session_file_edits(occurred_at DESC);
|
|
173
|
+
|
|
174
|
+
-- ============================================================
|
|
175
|
+
-- Consolidations audit (Day 2) — trail of what got compressed into what
|
|
176
|
+
-- ============================================================
|
|
177
|
+
CREATE TABLE IF NOT EXISTS consolidations (
|
|
178
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
179
|
+
learning_id INTEGER REFERENCES memories(id) ON DELETE SET NULL,
|
|
180
|
+
replaced_ids TEXT NOT NULL, -- JSON array of deleted memory ids
|
|
181
|
+
replaced_count INTEGER NOT NULL,
|
|
182
|
+
entity_id INTEGER REFERENCES entities(id) ON DELETE CASCADE,
|
|
183
|
+
original_layer TEXT NOT NULL,
|
|
184
|
+
created_at INTEGER NOT NULL DEFAULT (unixepoch())
|
|
185
|
+
);
|
|
186
|
+
|
|
187
|
+
CREATE INDEX IF NOT EXISTS idx_consolidations_entity ON consolidations(entity_id);
|
|
188
|
+
|
|
189
|
+
-- ============================================================
|
|
190
|
+
-- Meta — schema version tracking
|
|
191
|
+
-- ============================================================
|
|
192
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
193
|
+
key TEXT PRIMARY KEY,
|
|
194
|
+
value TEXT NOT NULL
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
INSERT OR IGNORE INTO meta (key, value) VALUES ('schema_version', '4');
|
|
198
|
+
INSERT OR IGNORE INTO meta (key, value) VALUES ('created_at', CAST(unixepoch() AS TEXT));
|
|
199
|
+
UPDATE meta SET value = '4' WHERE key = 'schema_version' AND value IN ('1', '2', '3');
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
export interface ConsolidateResult {
|
|
3
|
+
scanned: number;
|
|
4
|
+
clustersCompressed: number;
|
|
5
|
+
memoriesReplaced: number;
|
|
6
|
+
memoriesDropped: number;
|
|
7
|
+
learningIdsCreated: number[];
|
|
8
|
+
}
|
|
9
|
+
export declare function consolidate(db: Database.Database, opts?: {
|
|
10
|
+
scope?: 'all' | 'session';
|
|
11
|
+
min_age_days?: number;
|
|
12
|
+
}): ConsolidateResult;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// Sleep-mode consolidation: cluster stale low-importance memories,
|
|
2
|
+
// summarize into a single `learning`-layer entry, delete originals.
|
|
3
|
+
// Implements Michie's memory principle 4 (sleep consolidation).
|
|
4
|
+
//
|
|
5
|
+
// Strategy (rule-based, no LLM required):
|
|
6
|
+
// 1. Find candidates: layer IN (context, emotion, implementation),
|
|
7
|
+
// protected=0, cold (heat<30), older than 7 days, cluster size >= 2.
|
|
8
|
+
// 2. Group by (entity_id, layer).
|
|
9
|
+
// 3. Emit a structured summary into `learning` layer (protected=1),
|
|
10
|
+
// then delete originals.
|
|
11
|
+
// 4. Per-run forget-sweep drops expired memories that didn't cluster.
|
|
12
|
+
import { computeHeat } from './heat-index.js';
|
|
13
|
+
import { decideForgetting } from './forgetting.js';
|
|
14
|
+
const CLUSTER_LAYERS = ['context', 'emotion', 'implementation'];
|
|
15
|
+
const DEFAULT_MIN_AGE_DAYS = 7;
|
|
16
|
+
const MAX_HEAT = 30;
|
|
17
|
+
const MIN_CLUSTER_SIZE = 2;
|
|
18
|
+
export function consolidate(db, opts = {}) {
|
|
19
|
+
const now = Math.floor(Date.now() / 1000);
|
|
20
|
+
const minAgeDays = opts.min_age_days ?? DEFAULT_MIN_AGE_DAYS;
|
|
21
|
+
const ageCutoff = now - minAgeDays * 86400;
|
|
22
|
+
// Fetch candidates with age threshold. 'session' scope → same query currently
|
|
23
|
+
// (Day 2: session filter needs session_id plumbing; will add when sessions
|
|
24
|
+
// table is actually populated by the MCP server).
|
|
25
|
+
const layerPlaceholders = CLUSTER_LAYERS.map(() => '?').join(',');
|
|
26
|
+
const rows = db
|
|
27
|
+
.prepare(`
|
|
28
|
+
SELECT m.id, m.entity_id, m.layer, m.content, m.importance,
|
|
29
|
+
m.last_accessed_at, m.access_count, m.created_at, m.protected,
|
|
30
|
+
e.name as entity_name
|
|
31
|
+
FROM memories m
|
|
32
|
+
JOIN entities e ON e.id = m.entity_id
|
|
33
|
+
WHERE m.protected = 0
|
|
34
|
+
AND m.layer IN (${layerPlaceholders})
|
|
35
|
+
AND m.created_at <= ?
|
|
36
|
+
ORDER BY m.entity_id, m.layer, m.created_at
|
|
37
|
+
`)
|
|
38
|
+
.all(...CLUSTER_LAYERS, ageCutoff);
|
|
39
|
+
// Filter to cold-heat memories
|
|
40
|
+
const cold = rows.filter((r) => {
|
|
41
|
+
const daysSince = (now - r.last_accessed_at) / 86400;
|
|
42
|
+
const heat = computeHeat({
|
|
43
|
+
accessesLast30d: daysSince < 30 ? r.access_count : 0,
|
|
44
|
+
accessesLast90d: daysSince < 90 ? r.access_count : 0,
|
|
45
|
+
daysSinceLastAccess: daysSince,
|
|
46
|
+
totalAccesses: r.access_count,
|
|
47
|
+
baseImportance: r.importance,
|
|
48
|
+
});
|
|
49
|
+
return heat.score < MAX_HEAT;
|
|
50
|
+
});
|
|
51
|
+
// Group by (entity_id, layer)
|
|
52
|
+
const groups = new Map();
|
|
53
|
+
for (const r of cold) {
|
|
54
|
+
const key = `${r.entity_id}::${r.layer}`;
|
|
55
|
+
const arr = groups.get(key) ?? [];
|
|
56
|
+
arr.push(r);
|
|
57
|
+
groups.set(key, arr);
|
|
58
|
+
}
|
|
59
|
+
const result = {
|
|
60
|
+
scanned: rows.length,
|
|
61
|
+
clustersCompressed: 0,
|
|
62
|
+
memoriesReplaced: 0,
|
|
63
|
+
memoriesDropped: 0,
|
|
64
|
+
learningIdsCreated: [],
|
|
65
|
+
};
|
|
66
|
+
const insertLearning = db.prepare(`INSERT INTO memories (entity_id, layer, content, importance, protected, source)
|
|
67
|
+
VALUES (?, 'learning', ?, ?, 1, ?)`);
|
|
68
|
+
const deleteMemory = db.prepare('DELETE FROM memories WHERE id = ?');
|
|
69
|
+
const insertAudit = db.prepare(`INSERT INTO consolidations (learning_id, replaced_ids, replaced_count, entity_id, original_layer)
|
|
70
|
+
VALUES (?, ?, ?, ?, ?)`);
|
|
71
|
+
const insertEvent = db.prepare('INSERT INTO events (entity_id, kind, payload) VALUES (?, ?, ?)');
|
|
72
|
+
const tx = db.transaction(() => {
|
|
73
|
+
for (const [key, cluster] of groups) {
|
|
74
|
+
if (cluster.length < MIN_CLUSTER_SIZE)
|
|
75
|
+
continue;
|
|
76
|
+
const [entityIdStr, layer] = key.split('::');
|
|
77
|
+
const entityId = Number(entityIdStr);
|
|
78
|
+
const entityName = cluster[0].entity_name;
|
|
79
|
+
// Sort by importance desc, take top 3 as exemplars
|
|
80
|
+
const byImp = [...cluster].sort((a, b) => b.importance - a.importance);
|
|
81
|
+
const exemplars = byImp.slice(0, 3).map((c) => ({
|
|
82
|
+
fragment: c.content.length > 200 ? c.content.slice(0, 200) + '…' : c.content,
|
|
83
|
+
when: new Date(c.created_at * 1000).toISOString().slice(0, 10),
|
|
84
|
+
importance: c.importance,
|
|
85
|
+
}));
|
|
86
|
+
const timestamps = cluster.map((c) => c.created_at);
|
|
87
|
+
const earliest = new Date(Math.min(...timestamps) * 1000).toISOString().slice(0, 10);
|
|
88
|
+
const latest = new Date(Math.max(...timestamps) * 1000).toISOString().slice(0, 10);
|
|
89
|
+
const avgImp = cluster.reduce((s, c) => s + c.importance, 0) / cluster.length;
|
|
90
|
+
const summary = {
|
|
91
|
+
source: 'consolidate',
|
|
92
|
+
original_layer: layer,
|
|
93
|
+
count: cluster.length,
|
|
94
|
+
period: { from: earliest, to: latest },
|
|
95
|
+
pattern: `${cluster.length} cold observations on "${entityName}" (${layer}) consolidated during sleep`,
|
|
96
|
+
exemplars,
|
|
97
|
+
replaced_ids: cluster.map((c) => c.id),
|
|
98
|
+
};
|
|
99
|
+
const learningImportance = Math.max(avgImp, 0.55); // summaries are slightly more important than their avg source
|
|
100
|
+
const sourceMeta = JSON.stringify({ origin: 'consolidate', replaced: cluster.length });
|
|
101
|
+
const ins = insertLearning.run(entityId, JSON.stringify(summary, null, 2), learningImportance, sourceMeta);
|
|
102
|
+
const learningId = Number(ins.lastInsertRowid);
|
|
103
|
+
result.learningIdsCreated.push(learningId);
|
|
104
|
+
insertAudit.run(learningId, JSON.stringify(cluster.map((c) => c.id)), cluster.length, entityId, layer);
|
|
105
|
+
for (const c of cluster)
|
|
106
|
+
deleteMemory.run(c.id);
|
|
107
|
+
insertEvent.run(entityId, 'memory_consolidated', JSON.stringify({ learning_id: learningId, count: cluster.length, layer }));
|
|
108
|
+
result.clustersCompressed++;
|
|
109
|
+
result.memoriesReplaced += cluster.length;
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
tx();
|
|
113
|
+
// Post-consolidate: forget-sweep remaining non-clustered cold memories
|
|
114
|
+
const remaining = db
|
|
115
|
+
.prepare('SELECT id, layer, importance, access_count, last_accessed_at, protected FROM memories WHERE protected = 0')
|
|
116
|
+
.all();
|
|
117
|
+
const toDrop = [];
|
|
118
|
+
for (const r of remaining) {
|
|
119
|
+
const daysSince = (now - r.last_accessed_at) / 86400;
|
|
120
|
+
const heat = computeHeat({
|
|
121
|
+
accessesLast30d: daysSince < 30 ? r.access_count : 0,
|
|
122
|
+
accessesLast90d: daysSince < 90 ? r.access_count : 0,
|
|
123
|
+
daysSinceLastAccess: daysSince,
|
|
124
|
+
totalAccesses: r.access_count,
|
|
125
|
+
baseImportance: r.importance,
|
|
126
|
+
});
|
|
127
|
+
const action = decideForgetting({
|
|
128
|
+
daysSinceLastAccess: daysSince,
|
|
129
|
+
importance: r.importance,
|
|
130
|
+
heatScore: heat.score,
|
|
131
|
+
protected: r.protected === 1,
|
|
132
|
+
layer: r.layer,
|
|
133
|
+
});
|
|
134
|
+
if (action === 'drop')
|
|
135
|
+
toDrop.push(r.id);
|
|
136
|
+
}
|
|
137
|
+
if (toDrop.length > 0) {
|
|
138
|
+
const del = db.prepare('DELETE FROM memories WHERE id = ?');
|
|
139
|
+
const tx2 = db.transaction((ids) => {
|
|
140
|
+
for (const id of ids)
|
|
141
|
+
del.run(id);
|
|
142
|
+
});
|
|
143
|
+
tx2(toDrop);
|
|
144
|
+
result.memoriesDropped = toDrop.length;
|
|
145
|
+
}
|
|
146
|
+
return result;
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=consolidate.js.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type ChunkKind = 'function' | 'class' | 'variable' | 'import' | 'heading' | 'python_def' | 'python_class' | 'python_preamble' | 'fixed';
|
|
2
|
+
export interface Chunk {
|
|
3
|
+
id: string;
|
|
4
|
+
kind: ChunkKind;
|
|
5
|
+
start_line: number;
|
|
6
|
+
end_line: number;
|
|
7
|
+
hash: string;
|
|
8
|
+
content: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function shortHash(s: string): string;
|
|
11
|
+
export declare function hashFile(content: string): string;
|
|
12
|
+
export declare function chunkFile(path: string, content: string): Chunk[];
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
// File chunking for read_smart diff cache.
|
|
2
|
+
// Splits files into semantic chunks so re-reads can return only modified regions.
|
|
3
|
+
// .ts/.js/.jsx/.tsx/.mjs/.cjs → AST via @babel/parser (top-level decls)
|
|
4
|
+
// .py → indent-based (top-level def/class)
|
|
5
|
+
// .md → h2/h3 headings
|
|
6
|
+
// else → fixed 100-line windows
|
|
7
|
+
import { parse as babelParse } from '@babel/parser';
|
|
8
|
+
import { createHash } from 'node:crypto';
|
|
9
|
+
import { extname } from 'node:path';
|
|
10
|
+
export function shortHash(s) {
|
|
11
|
+
return createHash('sha256').update(s).digest('hex').slice(0, 16);
|
|
12
|
+
}
|
|
13
|
+
export function hashFile(content) {
|
|
14
|
+
return createHash('sha256').update(content).digest('hex');
|
|
15
|
+
}
|
|
16
|
+
function extractLines(content, start, end) {
|
|
17
|
+
return content.split('\n').slice(start - 1, end).join('\n');
|
|
18
|
+
}
|
|
19
|
+
export function chunkFile(path, content) {
|
|
20
|
+
const ext = extname(path).toLowerCase();
|
|
21
|
+
try {
|
|
22
|
+
if (['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs'].includes(ext)) {
|
|
23
|
+
return chunkTsJs(content);
|
|
24
|
+
}
|
|
25
|
+
if (ext === '.py')
|
|
26
|
+
return chunkPython(content);
|
|
27
|
+
if (ext === '.md' || ext === '.markdown')
|
|
28
|
+
return chunkMarkdown(content);
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
// parse failures fall back to fixed
|
|
32
|
+
}
|
|
33
|
+
return chunkFixed(content, 100);
|
|
34
|
+
}
|
|
35
|
+
// ============================================================
|
|
36
|
+
// TypeScript / JavaScript — AST-based
|
|
37
|
+
// ============================================================
|
|
38
|
+
function chunkTsJs(content) {
|
|
39
|
+
const ast = babelParse(content, {
|
|
40
|
+
sourceType: 'unambiguous',
|
|
41
|
+
plugins: ['typescript', 'jsx', 'decorators-legacy'],
|
|
42
|
+
errorRecovery: true,
|
|
43
|
+
allowImportExportEverywhere: true,
|
|
44
|
+
allowReturnOutsideFunction: true,
|
|
45
|
+
});
|
|
46
|
+
const chunks = [];
|
|
47
|
+
const importNodes = [];
|
|
48
|
+
for (const node of ast.program.body) {
|
|
49
|
+
if (!node.loc)
|
|
50
|
+
continue;
|
|
51
|
+
const start = node.loc.start.line;
|
|
52
|
+
const end = node.loc.end.line;
|
|
53
|
+
if (node.type === 'ImportDeclaration') {
|
|
54
|
+
importNodes.push({ start, end });
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
const name = extractDeclName(node);
|
|
58
|
+
const kind = mapNodeKind(node);
|
|
59
|
+
const text = extractLines(content, start, end);
|
|
60
|
+
chunks.push({
|
|
61
|
+
id: `${kind}:${name}`,
|
|
62
|
+
kind,
|
|
63
|
+
start_line: start,
|
|
64
|
+
end_line: end,
|
|
65
|
+
hash: shortHash(text),
|
|
66
|
+
content: text,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
if (importNodes.length > 0) {
|
|
70
|
+
const first = importNodes[0].start;
|
|
71
|
+
const last = importNodes[importNodes.length - 1].end;
|
|
72
|
+
const text = extractLines(content, first, last);
|
|
73
|
+
chunks.unshift({
|
|
74
|
+
id: 'import:_block',
|
|
75
|
+
kind: 'import',
|
|
76
|
+
start_line: first,
|
|
77
|
+
end_line: last,
|
|
78
|
+
hash: shortHash(text),
|
|
79
|
+
content: text,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
if (chunks.length === 0)
|
|
83
|
+
return chunkFixed(content, 100);
|
|
84
|
+
return chunks;
|
|
85
|
+
}
|
|
86
|
+
function extractDeclName(node) {
|
|
87
|
+
if (node.type === 'FunctionDeclaration')
|
|
88
|
+
return node.id?.name ?? 'anonymous';
|
|
89
|
+
if (node.type === 'ClassDeclaration')
|
|
90
|
+
return node.id?.name ?? 'anonymous';
|
|
91
|
+
if (node.type === 'VariableDeclaration') {
|
|
92
|
+
const d = node.declarations?.[0];
|
|
93
|
+
return d?.id?.name ?? 'anonymous';
|
|
94
|
+
}
|
|
95
|
+
if (node.type === 'ExportNamedDeclaration') {
|
|
96
|
+
if (node.declaration)
|
|
97
|
+
return extractDeclName(node.declaration);
|
|
98
|
+
const first = node.specifiers?.[0];
|
|
99
|
+
return first?.exported?.name ?? 'named_export';
|
|
100
|
+
}
|
|
101
|
+
if (node.type === 'ExportDefaultDeclaration') {
|
|
102
|
+
if (node.declaration?.id?.name)
|
|
103
|
+
return node.declaration.id.name;
|
|
104
|
+
return 'default';
|
|
105
|
+
}
|
|
106
|
+
if (node.type === 'TSInterfaceDeclaration')
|
|
107
|
+
return node.id?.name ?? 'interface';
|
|
108
|
+
if (node.type === 'TSTypeAliasDeclaration')
|
|
109
|
+
return node.id?.name ?? 'type';
|
|
110
|
+
if (node.type === 'TSEnumDeclaration')
|
|
111
|
+
return node.id?.name ?? 'enum';
|
|
112
|
+
if (node.type === 'TSModuleDeclaration')
|
|
113
|
+
return node.id?.name ?? 'module';
|
|
114
|
+
return node.type;
|
|
115
|
+
}
|
|
116
|
+
function mapNodeKind(node) {
|
|
117
|
+
const t = node.type;
|
|
118
|
+
if (t === 'FunctionDeclaration')
|
|
119
|
+
return 'function';
|
|
120
|
+
if (t === 'ClassDeclaration')
|
|
121
|
+
return 'class';
|
|
122
|
+
if (['VariableDeclaration', 'TSInterfaceDeclaration', 'TSTypeAliasDeclaration', 'TSEnumDeclaration', 'TSModuleDeclaration'].includes(t))
|
|
123
|
+
return 'variable';
|
|
124
|
+
if (t === 'ExportNamedDeclaration' || t === 'ExportDefaultDeclaration') {
|
|
125
|
+
if (node.declaration)
|
|
126
|
+
return mapNodeKind(node.declaration);
|
|
127
|
+
return 'variable';
|
|
128
|
+
}
|
|
129
|
+
return 'variable';
|
|
130
|
+
}
|
|
131
|
+
// ============================================================
|
|
132
|
+
// Python — indent-based
|
|
133
|
+
// ============================================================
|
|
134
|
+
function chunkPython(content) {
|
|
135
|
+
const lines = content.split('\n');
|
|
136
|
+
const chunks = [];
|
|
137
|
+
let current = null;
|
|
138
|
+
const pushCurrent = (end) => {
|
|
139
|
+
if (!current)
|
|
140
|
+
return;
|
|
141
|
+
const text = extractLines(content, current.start, end);
|
|
142
|
+
chunks.push({
|
|
143
|
+
id: `${current.kind}:${current.name}`,
|
|
144
|
+
kind: current.kind,
|
|
145
|
+
start_line: current.start,
|
|
146
|
+
end_line: end,
|
|
147
|
+
hash: shortHash(text),
|
|
148
|
+
content: text,
|
|
149
|
+
});
|
|
150
|
+
current = null;
|
|
151
|
+
};
|
|
152
|
+
for (let i = 0; i < lines.length; i++) {
|
|
153
|
+
const line = lines[i];
|
|
154
|
+
if (line.startsWith(' ') || line.startsWith('\t'))
|
|
155
|
+
continue;
|
|
156
|
+
const m = line.match(/^(async\s+def|def|class)\s+(\w+)/);
|
|
157
|
+
if (m) {
|
|
158
|
+
pushCurrent(i); // previous ends at the line before current
|
|
159
|
+
const kind = m[1] === 'class' ? 'python_class' : 'python_def';
|
|
160
|
+
current = { start: i + 1, name: m[2], kind };
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
pushCurrent(lines.length);
|
|
164
|
+
if (chunks.length === 0)
|
|
165
|
+
return chunkFixed(content, 100);
|
|
166
|
+
if (chunks[0].start_line > 1) {
|
|
167
|
+
const preambleText = extractLines(content, 1, chunks[0].start_line - 1);
|
|
168
|
+
if (preambleText.trim()) {
|
|
169
|
+
chunks.unshift({
|
|
170
|
+
id: 'python_preamble',
|
|
171
|
+
kind: 'python_preamble',
|
|
172
|
+
start_line: 1,
|
|
173
|
+
end_line: chunks[0].start_line - 1,
|
|
174
|
+
hash: shortHash(preambleText),
|
|
175
|
+
content: preambleText,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return chunks;
|
|
180
|
+
}
|
|
181
|
+
// ============================================================
|
|
182
|
+
// Markdown — h2/h3 boundaries
|
|
183
|
+
// ============================================================
|
|
184
|
+
function chunkMarkdown(content) {
|
|
185
|
+
const lines = content.split('\n');
|
|
186
|
+
const chunks = [];
|
|
187
|
+
let current = null;
|
|
188
|
+
const pushCurrent = (end) => {
|
|
189
|
+
if (!current)
|
|
190
|
+
return;
|
|
191
|
+
const text = extractLines(content, current.start, end);
|
|
192
|
+
chunks.push({
|
|
193
|
+
id: `heading:${current.heading}`,
|
|
194
|
+
kind: 'heading',
|
|
195
|
+
start_line: current.start,
|
|
196
|
+
end_line: end,
|
|
197
|
+
hash: shortHash(text),
|
|
198
|
+
content: text,
|
|
199
|
+
});
|
|
200
|
+
current = null;
|
|
201
|
+
};
|
|
202
|
+
for (let i = 0; i < lines.length; i++) {
|
|
203
|
+
const m = lines[i].match(/^(#{2,3})\s+(.+?)\s*$/);
|
|
204
|
+
if (m) {
|
|
205
|
+
pushCurrent(i);
|
|
206
|
+
current = { start: i + 1, heading: m[2] };
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
pushCurrent(lines.length);
|
|
210
|
+
if (chunks.length === 0)
|
|
211
|
+
return chunkFixed(content, 100);
|
|
212
|
+
if (chunks[0].start_line > 1) {
|
|
213
|
+
const preamble = extractLines(content, 1, chunks[0].start_line - 1);
|
|
214
|
+
if (preamble.trim()) {
|
|
215
|
+
chunks.unshift({
|
|
216
|
+
id: 'heading:_preamble',
|
|
217
|
+
kind: 'heading',
|
|
218
|
+
start_line: 1,
|
|
219
|
+
end_line: chunks[0].start_line - 1,
|
|
220
|
+
hash: shortHash(preamble),
|
|
221
|
+
content: preamble,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return chunks;
|
|
226
|
+
}
|
|
227
|
+
// ============================================================
|
|
228
|
+
// Fixed — 100-line windows (fallback)
|
|
229
|
+
// ============================================================
|
|
230
|
+
function chunkFixed(content, size) {
|
|
231
|
+
const lines = content.split('\n');
|
|
232
|
+
const chunks = [];
|
|
233
|
+
if (lines.length === 0)
|
|
234
|
+
return chunks;
|
|
235
|
+
for (let i = 0; i < lines.length; i += size) {
|
|
236
|
+
const slice = lines.slice(i, i + size);
|
|
237
|
+
const text = slice.join('\n');
|
|
238
|
+
const end = Math.min(i + size, lines.length);
|
|
239
|
+
chunks.push({
|
|
240
|
+
id: `lines:${i + 1}_${end}`,
|
|
241
|
+
kind: 'fixed',
|
|
242
|
+
start_line: i + 1,
|
|
243
|
+
end_line: end,
|
|
244
|
+
hash: shortHash(text),
|
|
245
|
+
content: text,
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
return chunks;
|
|
249
|
+
}
|
|
250
|
+
//# sourceMappingURL=file-chunker.js.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface ForgettingInput {
|
|
2
|
+
daysSinceLastAccess: number;
|
|
3
|
+
importance: number;
|
|
4
|
+
heatScore: number;
|
|
5
|
+
protected: boolean;
|
|
6
|
+
layer: string;
|
|
7
|
+
}
|
|
8
|
+
export declare function forgettingRisk(input: ForgettingInput): number;
|
|
9
|
+
export declare const COMPRESS_THRESHOLD = 50;
|
|
10
|
+
export declare const DROP_THRESHOLD = 200;
|
|
11
|
+
export type ForgettingAction = 'keep' | 'compress' | 'drop';
|
|
12
|
+
export declare function decideForgetting(input: ForgettingInput): ForgettingAction;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Ported from linksee-app/setup-learning-box.cjs:62-99 (Ebbinghaus forgetting curve).
|
|
2
|
+
// Implements Michie's memory principle 6: active forgetting.
|
|
3
|
+
// Protected memories (caveat layer) and goal layer bypass decay.
|
|
4
|
+
// Returns true if this memory should be forgotten (compressed to summary or deleted).
|
|
5
|
+
// Higher forgettingRisk → more likely to forget.
|
|
6
|
+
export function forgettingRisk(input) {
|
|
7
|
+
if (input.protected)
|
|
8
|
+
return 0;
|
|
9
|
+
if (input.layer === 'goal')
|
|
10
|
+
return 0; // Goals are WHY-anchors, never auto-forget while active
|
|
11
|
+
// Original formula from setup-learning-box.cjs:
|
|
12
|
+
// daysSinceContact * (heatScore/100) * (1 + daysSinceContact/30)
|
|
13
|
+
// We INVERT: high heat = low risk (hot memories should be kept).
|
|
14
|
+
const heatFactor = 1 - (input.heatScore / 100); // 0.0 = keep, 1.0 = drop
|
|
15
|
+
const importanceFactor = 1 - input.importance;
|
|
16
|
+
const timeFactor = input.daysSinceLastAccess * (1 + input.daysSinceLastAccess / 30);
|
|
17
|
+
return heatFactor * importanceFactor * timeFactor;
|
|
18
|
+
}
|
|
19
|
+
// Risk threshold above which memory is compressed (→ learning layer summary) and the original deleted.
|
|
20
|
+
export const COMPRESS_THRESHOLD = 50;
|
|
21
|
+
// Risk threshold above which memory is entirely dropped.
|
|
22
|
+
export const DROP_THRESHOLD = 200;
|
|
23
|
+
export function decideForgetting(input) {
|
|
24
|
+
const risk = forgettingRisk(input);
|
|
25
|
+
if (risk >= DROP_THRESHOLD)
|
|
26
|
+
return 'drop';
|
|
27
|
+
if (risk >= COMPRESS_THRESHOLD)
|
|
28
|
+
return 'compress';
|
|
29
|
+
return 'keep';
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=forgetting.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export type HeatBand = 'hot' | 'warm' | 'cold' | 'frozen';
|
|
2
|
+
export interface HeatInput {
|
|
3
|
+
accessesLast30d: number;
|
|
4
|
+
accessesLast90d: number;
|
|
5
|
+
daysSinceLastAccess: number;
|
|
6
|
+
totalAccesses: number;
|
|
7
|
+
baseImportance?: number;
|
|
8
|
+
}
|
|
9
|
+
export interface HeatResult {
|
|
10
|
+
score: number;
|
|
11
|
+
band: HeatBand;
|
|
12
|
+
}
|
|
13
|
+
export declare function computeHeat(input: HeatInput): HeatResult;
|