@modusensus/dsh-mneme 0.1.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +239 -207
- package/cordis.patch.yml +15 -15
- package/lib/api.js +277 -245
- package/lib/client.js +461 -461
- package/lib/commands.js +64 -64
- package/lib/config.js +49 -15
- package/lib/dream/clustering.js +118 -0
- package/lib/dream/decisions.js +128 -121
- package/lib/dream.js +428 -209
- package/lib/embedding.js +97 -97
- package/lib/index.js +176 -120
- package/lib/inject.js +47 -47
- package/lib/local-embedder.js +265 -0
- package/lib/mirror.js +131 -131
- package/lib/reranker.js +203 -0
- package/lib/service.js +268 -174
- package/lib/settings.js +142 -142
- package/lib/store.js +409 -310
- package/lib/summarize.js +171 -171
- package/lib/tools.js +230 -241
- package/lib/vector-index.js +106 -0
- package/package.json +7 -3
- package/src/api.js +277 -245
- package/src/commands.js +64 -64
- package/src/config.js +49 -15
- package/src/dream/clustering.js +118 -0
- package/src/dream/decisions.js +128 -121
- package/src/dream.js +428 -209
- package/src/embedding.js +97 -97
- package/src/index.js +176 -120
- package/src/inject.js +47 -47
- package/src/local-embedder.js +265 -0
- package/src/mirror.js +131 -131
- package/src/reranker.js +203 -0
- package/src/service.js +268 -174
- package/src/settings.js +142 -142
- package/src/store.js +409 -310
- package/src/summarize.js +171 -171
- package/src/tools.js +230 -241
- package/src/vector-index.js +106 -0
package/lib/store.js
CHANGED
|
@@ -1,310 +1,409 @@
|
|
|
1
|
-
import { DatabaseSync } from "node:sqlite";
|
|
2
|
-
import { randomUUID } from "node:crypto";
|
|
3
|
-
|
|
4
|
-
const SCHEMA = `
|
|
5
|
-
CREATE TABLE IF NOT EXISTS memories (
|
|
6
|
-
id TEXT PRIMARY KEY,
|
|
7
|
-
type TEXT NOT NULL,
|
|
8
|
-
title TEXT NOT NULL,
|
|
9
|
-
content TEXT NOT NULL,
|
|
10
|
-
tags TEXT NOT NULL DEFAULT '[]',
|
|
11
|
-
importance INTEGER NOT NULL DEFAULT 3,
|
|
12
|
-
forgotten INTEGER NOT NULL DEFAULT 0,
|
|
13
|
-
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
|
-
source TEXT,
|
|
15
|
-
embedding TEXT,
|
|
16
|
-
created_at TEXT NOT NULL,
|
|
17
|
-
updated_at TEXT NOT NULL
|
|
18
|
-
);
|
|
19
|
-
CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
|
|
20
|
-
CREATE INDEX IF NOT EXISTS idx_memories_importance ON memories(importance);
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
return
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
function
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
).
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
if (
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
return
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
return
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
if (!
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
const
|
|
234
|
-
const
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
return
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
function
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
}
|
|
310
|
-
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
|
|
4
|
+
const SCHEMA = `
|
|
5
|
+
CREATE TABLE IF NOT EXISTS memories (
|
|
6
|
+
id TEXT PRIMARY KEY,
|
|
7
|
+
type TEXT NOT NULL,
|
|
8
|
+
title TEXT NOT NULL,
|
|
9
|
+
content TEXT NOT NULL,
|
|
10
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
11
|
+
importance INTEGER NOT NULL DEFAULT 3,
|
|
12
|
+
forgotten INTEGER NOT NULL DEFAULT 0,
|
|
13
|
+
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
|
+
source TEXT,
|
|
15
|
+
embedding TEXT,
|
|
16
|
+
created_at TEXT NOT NULL,
|
|
17
|
+
updated_at TEXT NOT NULL
|
|
18
|
+
);
|
|
19
|
+
CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
|
|
20
|
+
CREATE INDEX IF NOT EXISTS idx_memories_importance ON memories(importance);
|
|
21
|
+
|
|
22
|
+
-- autoDream audit trail: one row per consolidation run, capturing the exact
|
|
23
|
+
-- input snapshot digest + the LLM decision list + per-id outcome + a compact
|
|
24
|
+
-- receipt. This makes every decision replayable so silent consolidation errors
|
|
25
|
+
-- (high pass rate but wrong merge/conflict) can be located after the fact.
|
|
26
|
+
CREATE TABLE IF NOT EXISTS dream_runs (
|
|
27
|
+
id TEXT PRIMARY KEY,
|
|
28
|
+
created_at TEXT NOT NULL,
|
|
29
|
+
status TEXT NOT NULL, -- ok | failed
|
|
30
|
+
error TEXT,
|
|
31
|
+
provider TEXT,
|
|
32
|
+
model TEXT,
|
|
33
|
+
snapshot_hash TEXT NOT NULL,
|
|
34
|
+
input_count INTEGER NOT NULL,
|
|
35
|
+
input TEXT, -- JSON: full input snapshot (id/type/title/content/importance/updated_at)
|
|
36
|
+
decisions TEXT, -- JSON: raw LLM decision list
|
|
37
|
+
outcome TEXT, -- JSON: { byId: {id: action} }
|
|
38
|
+
applied INTEGER NOT NULL DEFAULT 0,
|
|
39
|
+
summary_stored INTEGER NOT NULL DEFAULT 0,
|
|
40
|
+
receipt TEXT NOT NULL
|
|
41
|
+
);
|
|
42
|
+
CREATE INDEX IF NOT EXISTS idx_dream_runs_created ON dream_runs(created_at);
|
|
43
|
+
`;
|
|
44
|
+
|
|
45
|
+
const TYPES = new Set(["preference", "project", "decision", "history", "summary"]);
|
|
46
|
+
|
|
47
|
+
// Pure helpers: no shared module state.
|
|
48
|
+
|
|
49
|
+
function sanitizePage(limit, offset, defaultLimit) {
|
|
50
|
+
const lim = Number.isInteger(limit) && limit > 0 ? limit : defaultLimit;
|
|
51
|
+
const off = Number.isInteger(offset) && offset > 0 ? offset : 0;
|
|
52
|
+
return { limit: lim, offset: off };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function escapeLike(q) {
|
|
56
|
+
return q.replace(/[\\%_]/g, (c) => `\\${c}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function parseTags(raw) {
|
|
60
|
+
try {
|
|
61
|
+
const arr = JSON.parse(raw);
|
|
62
|
+
return Array.isArray(arr) ? arr : [];
|
|
63
|
+
} catch {
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function toRow(row) {
|
|
69
|
+
if (!row) return undefined;
|
|
70
|
+
return {
|
|
71
|
+
id: row.id,
|
|
72
|
+
type: row.type,
|
|
73
|
+
title: row.title,
|
|
74
|
+
content: row.content,
|
|
75
|
+
tags: parseTags(row.tags),
|
|
76
|
+
importance: row.importance,
|
|
77
|
+
forgotten: row.forgotten === 1,
|
|
78
|
+
archived: row.archived === 1,
|
|
79
|
+
source: row.source ?? undefined,
|
|
80
|
+
created_at: row.created_at,
|
|
81
|
+
updated_at: row.updated_at
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function toDreamRun(row) {
|
|
86
|
+
if (!row) return undefined;
|
|
87
|
+
return {
|
|
88
|
+
id: row.id,
|
|
89
|
+
created_at: row.created_at,
|
|
90
|
+
status: row.status,
|
|
91
|
+
error: row.error ?? undefined,
|
|
92
|
+
provider: row.provider ?? undefined,
|
|
93
|
+
model: row.model ?? undefined,
|
|
94
|
+
snapshot_hash: row.snapshot_hash,
|
|
95
|
+
input_count: row.input_count,
|
|
96
|
+
input: row.input ? JSON.parse(row.input) : undefined,
|
|
97
|
+
decisions: row.decisions ? JSON.parse(row.decisions) : undefined,
|
|
98
|
+
outcome: row.outcome ? JSON.parse(row.outcome) : undefined,
|
|
99
|
+
applied: row.applied,
|
|
100
|
+
summary_stored: row.summary_stored === 1,
|
|
101
|
+
receipt: row.receipt
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function createStore(path) {
|
|
106
|
+
const db = new DatabaseSync(path);
|
|
107
|
+
db.exec("PRAGMA journal_mode = WAL;");
|
|
108
|
+
db.exec(SCHEMA);
|
|
109
|
+
|
|
110
|
+
// Schema migrations for legacy databases (idempotent).
|
|
111
|
+
const columns = db.prepare("PRAGMA table_info(memories)").all().map((c) => c.name);
|
|
112
|
+
if (!columns.includes("archived")) {
|
|
113
|
+
db.exec("ALTER TABLE memories ADD COLUMN archived INTEGER NOT NULL DEFAULT 0");
|
|
114
|
+
}
|
|
115
|
+
if (!columns.includes("embedding")) {
|
|
116
|
+
db.exec("ALTER TABLE memories ADD COLUMN embedding TEXT");
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Per-instance monotonic timestamp guard: consecutive writes within the same
|
|
120
|
+
// millisecond must still produce strictly increasing timestamps (test asserts
|
|
121
|
+
// updated_at != created_at). State lives in the store closure, not module scope.
|
|
122
|
+
let lastTs = "";
|
|
123
|
+
function nowIso() {
|
|
124
|
+
let ts = new Date().toISOString();
|
|
125
|
+
if (lastTs && ts <= lastTs) {
|
|
126
|
+
const d = new Date(lastTs);
|
|
127
|
+
d.setMilliseconds(d.getMilliseconds() + 1);
|
|
128
|
+
ts = d.toISOString();
|
|
129
|
+
}
|
|
130
|
+
lastTs = ts;
|
|
131
|
+
return ts;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function count(type, { includeForgotten = false, includeArchived = false } = {}) {
|
|
135
|
+
const clauses = [];
|
|
136
|
+
const params = [];
|
|
137
|
+
if (type !== undefined) {
|
|
138
|
+
clauses.push("type = ?");
|
|
139
|
+
params.push(type);
|
|
140
|
+
}
|
|
141
|
+
if (!includeForgotten) {
|
|
142
|
+
clauses.push("forgotten = 0");
|
|
143
|
+
}
|
|
144
|
+
if (!includeArchived) {
|
|
145
|
+
clauses.push("archived = 0");
|
|
146
|
+
}
|
|
147
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
148
|
+
return db.prepare(`SELECT count(*) AS c FROM memories ${where}`).get(...params).c;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function getById(id) {
|
|
152
|
+
const row = db.prepare("SELECT * FROM memories WHERE id = ?").get(id);
|
|
153
|
+
return toRow(row);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function save(memory) {
|
|
157
|
+
const id = memory.id ?? randomUUID();
|
|
158
|
+
const type = memory.type;
|
|
159
|
+
if (!TYPES.has(type)) throw new Error(`invalid memory type: ${type}`);
|
|
160
|
+
if (memory.tags !== undefined && !Array.isArray(memory.tags)) {
|
|
161
|
+
throw new Error("tags must be an array");
|
|
162
|
+
}
|
|
163
|
+
const now = nowIso();
|
|
164
|
+
const tags = JSON.stringify(memory.tags ?? []);
|
|
165
|
+
const importance = Number.isInteger(memory.importance) ? memory.importance : 3;
|
|
166
|
+
const embedding = Array.isArray(memory.embedding) && memory.embedding.length
|
|
167
|
+
? JSON.stringify(memory.embedding)
|
|
168
|
+
: null;
|
|
169
|
+
db.prepare(
|
|
170
|
+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, embedding, created_at, updated_at)
|
|
171
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`
|
|
172
|
+
).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, embedding, now, now);
|
|
173
|
+
return getById(id);
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function update(id, patch) {
|
|
177
|
+
const existing = getById(id);
|
|
178
|
+
if (!existing) throw new Error(`memory not found: ${id}`);
|
|
179
|
+
const type = patch.type ?? existing.type;
|
|
180
|
+
if (!TYPES.has(type)) throw new Error(`invalid memory type: ${type}`);
|
|
181
|
+
if (patch.tags !== undefined && !Array.isArray(patch.tags)) {
|
|
182
|
+
throw new Error("tags must be an array");
|
|
183
|
+
}
|
|
184
|
+
const now = nowIso();
|
|
185
|
+
const embedding = patch.embedding !== undefined
|
|
186
|
+
? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
|
|
187
|
+
: existing.embedding ?? null;
|
|
188
|
+
db.prepare(
|
|
189
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=? WHERE id=?`
|
|
190
|
+
).run(
|
|
191
|
+
type,
|
|
192
|
+
patch.title ?? existing.title,
|
|
193
|
+
patch.content ?? existing.content,
|
|
194
|
+
JSON.stringify(patch.tags ?? existing.tags),
|
|
195
|
+
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
196
|
+
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
197
|
+
embedding,
|
|
198
|
+
now,
|
|
199
|
+
id
|
|
200
|
+
);
|
|
201
|
+
return getById(id);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function remove(id) {
|
|
205
|
+
db.prepare("DELETE FROM memories WHERE id = ?").run(id);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function setForget(id, forgotten) {
|
|
209
|
+
db.prepare("UPDATE memories SET forgotten = ?, updated_at = ? WHERE id = ?")
|
|
210
|
+
.run(forgotten === true || forgotten === 1 ? 1 : 0, nowIso(), id);
|
|
211
|
+
return getById(id);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function setArchived(id, archived) {
|
|
215
|
+
db.prepare("UPDATE memories SET archived = ?, updated_at = ? WHERE id = ?")
|
|
216
|
+
.run(archived ? 1 : 0, nowIso(), id);
|
|
217
|
+
return getById(id);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function list({ type, limit = 50, offset = 0, includeForgotten = false, includeArchived = false } = {}) {
|
|
221
|
+
const clauses = [];
|
|
222
|
+
const params = [];
|
|
223
|
+
if (type) {
|
|
224
|
+
clauses.push("type = ?");
|
|
225
|
+
params.push(type);
|
|
226
|
+
}
|
|
227
|
+
if (!includeForgotten) {
|
|
228
|
+
clauses.push("forgotten = 0");
|
|
229
|
+
}
|
|
230
|
+
if (!includeArchived) {
|
|
231
|
+
clauses.push("archived = 0");
|
|
232
|
+
}
|
|
233
|
+
const { limit: lim, offset: off } = sanitizePage(limit, offset, 50);
|
|
234
|
+
const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
235
|
+
const rows = db.prepare(
|
|
236
|
+
`SELECT * FROM memories ${where} ORDER BY importance DESC, updated_at DESC, id LIMIT ? OFFSET ?`
|
|
237
|
+
).all(...params, lim, off);
|
|
238
|
+
return rows.map(toRow);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function all() {
|
|
242
|
+
const rows = db.prepare("SELECT * FROM memories ORDER BY updated_at DESC").all();
|
|
243
|
+
return rows.map(toRow);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Set (or clear with null) the embedding vector of a memory. */
|
|
247
|
+
function setEmbedding(id, vector) {
|
|
248
|
+
const json = Array.isArray(vector) && vector.length ? JSON.stringify(vector) : null;
|
|
249
|
+
db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(json, id);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function embeddedCount() {
|
|
253
|
+
return db.prepare(
|
|
254
|
+
"SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != ''"
|
|
255
|
+
).get().c;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Candidate rows still missing an embedding, for incremental re-indexing. */
|
|
259
|
+
function needsEmbedding(limit = 50) {
|
|
260
|
+
return db.prepare(
|
|
261
|
+
`SELECT id, title, content FROM memories
|
|
262
|
+
WHERE embedding IS NULL OR embedding = ''
|
|
263
|
+
ORDER BY updated_at DESC LIMIT ?`
|
|
264
|
+
).all(limit);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function search(query, { limit = 20, includeArchived = false } = {}) {
|
|
268
|
+
const q = String(query).trim();
|
|
269
|
+
if (!q) return [];
|
|
270
|
+
// Plain LIKE substring scan over title/content/tags (wildcards escaped so
|
|
271
|
+
// user input matches literally). No FTS5: CJK substring matching needs
|
|
272
|
+
// LIKE, and typical memory stores are small enough that a scan is fine.
|
|
273
|
+
const like = `%${escapeLike(q)}%`;
|
|
274
|
+
const { limit: lim } = sanitizePage(limit, 0, 20);
|
|
275
|
+
const archivedFilter = includeArchived ? "" : "archived = 0 AND ";
|
|
276
|
+
const rows = db.prepare(
|
|
277
|
+
`SELECT * FROM memories
|
|
278
|
+
WHERE ${archivedFilter}forgotten = 0 AND (title LIKE ? ESCAPE '\\' OR content LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\')
|
|
279
|
+
ORDER BY
|
|
280
|
+
CASE WHEN title LIKE ? ESCAPE '\\' THEN 0 ELSE 1 END,
|
|
281
|
+
importance DESC,
|
|
282
|
+
updated_at DESC,
|
|
283
|
+
id
|
|
284
|
+
LIMIT ?`
|
|
285
|
+
).all(like, like, like, like, lim);
|
|
286
|
+
return rows.map(toRow);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// --- vector search ------------------------------------------------------
|
|
290
|
+
|
|
291
|
+
function cosine(a, b) {
|
|
292
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length || !a.length) return 0;
|
|
293
|
+
let dot = 0;
|
|
294
|
+
let na = 0;
|
|
295
|
+
let nb = 0;
|
|
296
|
+
for (let i = 0; i < a.length; i++) {
|
|
297
|
+
dot += a[i] * b[i];
|
|
298
|
+
na += a[i] * a[i];
|
|
299
|
+
nb += b[i] * b[i];
|
|
300
|
+
}
|
|
301
|
+
if (na === 0 || nb === 0) return 0;
|
|
302
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Brute-force cosine similarity over embedded rows. Returns rows decorated
|
|
307
|
+
* with a `score` (0..1). Only rows with a stored embedding participate.
|
|
308
|
+
*/
|
|
309
|
+
function searchVector(vector, { limit = 20, includeArchived = false, threshold = 0 } = {}) {
|
|
310
|
+
if (!Array.isArray(vector) || !vector.length) return [];
|
|
311
|
+
const archivedFilter = includeArchived ? "" : "archived = 0 AND ";
|
|
312
|
+
const rows = db.prepare(
|
|
313
|
+
`SELECT * FROM memories
|
|
314
|
+
WHERE ${archivedFilter}forgotten = 0 AND embedding IS NOT NULL AND embedding != ''`
|
|
315
|
+
).all();
|
|
316
|
+
const scored = [];
|
|
317
|
+
for (const row of rows) {
|
|
318
|
+
let v;
|
|
319
|
+
try {
|
|
320
|
+
v = JSON.parse(row.embedding);
|
|
321
|
+
} catch {
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
const score = cosine(vector, v);
|
|
325
|
+
if (score >= threshold) scored.push({ row, score });
|
|
326
|
+
}
|
|
327
|
+
scored.sort((a, b) => b.score - a.score);
|
|
328
|
+
const { limit: lim } = sanitizePage(limit, 0, 20);
|
|
329
|
+
return scored.slice(0, lim).map(({ row, score }) => ({ ...toRow(row), score }));
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// --- autoDream audit trail ----------------------------------------------
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* Persist one autoDream run. The audit row is machine-verifiable but never
|
|
336
|
+
* triggers write hooks (it is bookkeeping, not a memory mutation): dream
|
|
337
|
+
* records its own runs, and a notify here would loop back into the dream
|
|
338
|
+
* scheduler. Writes are idempotent on run id (replay overwrites, never
|
|
339
|
+
* duplicates) so the same logical run can be re-applied for verification.
|
|
340
|
+
*/
|
|
341
|
+
function saveDreamRun(run) {
|
|
342
|
+
const id = run.id ?? randomUUID();
|
|
343
|
+
const now = nowIso();
|
|
344
|
+
db.prepare(
|
|
345
|
+
`INSERT INTO dream_runs (id, created_at, status, error, provider, model, snapshot_hash,
|
|
346
|
+
input_count, input, decisions, outcome, applied, summary_stored, receipt)
|
|
347
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
348
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
349
|
+
created_at=excluded.created_at, status=excluded.status, error=excluded.error,
|
|
350
|
+
provider=excluded.provider, model=excluded.model, snapshot_hash=excluded.snapshot_hash,
|
|
351
|
+
input_count=excluded.input_count, input=excluded.input, decisions=excluded.decisions,
|
|
352
|
+
outcome=excluded.outcome, applied=excluded.applied, summary_stored=excluded.summary_stored,
|
|
353
|
+
receipt=excluded.receipt`
|
|
354
|
+
).run(
|
|
355
|
+
id,
|
|
356
|
+
run.created_at ?? now,
|
|
357
|
+
run.status,
|
|
358
|
+
run.error ?? null,
|
|
359
|
+
run.provider ?? null,
|
|
360
|
+
run.model ?? null,
|
|
361
|
+
run.snapshot_hash,
|
|
362
|
+
run.input_count,
|
|
363
|
+
run.input !== undefined ? JSON.stringify(run.input) : null,
|
|
364
|
+
run.decisions !== undefined ? JSON.stringify(run.decisions) : null,
|
|
365
|
+
run.outcome !== undefined ? JSON.stringify(run.outcome) : null,
|
|
366
|
+
run.applied ?? 0,
|
|
367
|
+
run.summary_stored ? 1 : 0,
|
|
368
|
+
run.receipt
|
|
369
|
+
);
|
|
370
|
+
return getDreamRun(id);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function getDreamRun(id) {
|
|
374
|
+
const row = db.prepare("SELECT * FROM dream_runs WHERE id = ?").get(id);
|
|
375
|
+
return toDreamRun(row);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
function listDreamRuns({ limit = 50, offset = 0 } = {}) {
|
|
379
|
+
const { limit: lim, offset: off } = sanitizePage(limit, offset, 50);
|
|
380
|
+
const rows = db.prepare(
|
|
381
|
+
"SELECT * FROM dream_runs ORDER BY created_at DESC, id LIMIT ? OFFSET ?"
|
|
382
|
+
).all(lim, off);
|
|
383
|
+
return rows.map(toDreamRun);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
return {
|
|
387
|
+
db,
|
|
388
|
+
count,
|
|
389
|
+
getById,
|
|
390
|
+
save,
|
|
391
|
+
update,
|
|
392
|
+
remove,
|
|
393
|
+
setForget,
|
|
394
|
+
setArchived,
|
|
395
|
+
list,
|
|
396
|
+
all,
|
|
397
|
+
search,
|
|
398
|
+
setEmbedding,
|
|
399
|
+
embeddedCount,
|
|
400
|
+
needsEmbedding,
|
|
401
|
+
searchVector,
|
|
402
|
+
saveDreamRun,
|
|
403
|
+
getDreamRun,
|
|
404
|
+
listDreamRuns,
|
|
405
|
+
close() {
|
|
406
|
+
db.close();
|
|
407
|
+
}
|
|
408
|
+
};
|
|
409
|
+
}
|