@modusensus/dsh-mneme 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -7
- package/lib/api.js +92 -3
- package/lib/client.js +135 -60
- package/lib/embedding.js +97 -0
- package/lib/index.js +8 -2
- package/lib/service.js +17 -0
- package/lib/settings.js +22 -0
- package/lib/store.js +84 -5
- package/lib/tools.js +24 -4
- package/package.json +58 -58
- package/src/api.js +92 -3
- package/src/embedding.js +97 -0
- package/src/index.js +8 -2
- package/src/service.js +17 -0
- package/src/settings.js +22 -0
- package/src/store.js +84 -5
- package/src/tools.js +24 -4
package/lib/service.js
CHANGED
|
@@ -6,6 +6,17 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
6
6
|
// passed in the constructor). Fired on the same write events as onWrite.
|
|
7
7
|
let dreamHook = null;
|
|
8
8
|
|
|
9
|
+
// Optional vector embedder, installed via setEmbedder after creation. After
|
|
10
|
+
// any content write it fire-and-forgets a re-embed of the row so vector
|
|
11
|
+
// search stays in sync; failures are swallowed inside the embedder.
|
|
12
|
+
let embedder = null;
|
|
13
|
+
|
|
14
|
+
function scheduleEmbed(memory) {
|
|
15
|
+
if (embedder && memory?.id) {
|
|
16
|
+
try { embedder.schedule(memory); } catch { /* ignore */ }
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
9
20
|
/**
|
|
10
21
|
* Fire-and-forget write notification; errors are swallowed to keep write
|
|
11
22
|
* paths clean. The store mutation has already committed, so a throwing
|
|
@@ -38,6 +49,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
38
49
|
});
|
|
39
50
|
syncMirror();
|
|
40
51
|
notifyWrite();
|
|
52
|
+
scheduleEmbed(merged);
|
|
41
53
|
return { action: "merged", memory: merged };
|
|
42
54
|
}
|
|
43
55
|
const created = store.save({
|
|
@@ -50,6 +62,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
50
62
|
});
|
|
51
63
|
syncMirror();
|
|
52
64
|
notifyWrite();
|
|
65
|
+
scheduleEmbed(created);
|
|
53
66
|
return { action: "created", memory: created };
|
|
54
67
|
}
|
|
55
68
|
|
|
@@ -126,8 +139,11 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
126
139
|
mergeHumanEdits,
|
|
127
140
|
toApiList,
|
|
128
141
|
setDreamHook(fn) { dreamHook = fn; },
|
|
142
|
+
setEmbedder(emb) { embedder = emb; },
|
|
129
143
|
// passthroughs used by tools and api layers; mutations keep the mirror in sync
|
|
130
144
|
search: (q, o) => store.search(q, o),
|
|
145
|
+
searchVector: (v, o) => store.searchVector(v, o),
|
|
146
|
+
embeddedCount: () => store.embeddedCount(),
|
|
131
147
|
list: (o) => store.list(o),
|
|
132
148
|
all: () => store.all(),
|
|
133
149
|
count: (type) => store.count(type),
|
|
@@ -141,6 +157,7 @@ export function createService({ store, mirror, config, onWrite }) {
|
|
|
141
157
|
const updated = store.update(id, p);
|
|
142
158
|
syncMirror();
|
|
143
159
|
notifyWrite();
|
|
160
|
+
scheduleEmbed(updated);
|
|
144
161
|
return updated;
|
|
145
162
|
},
|
|
146
163
|
setForget: (id, f) => {
|
package/lib/settings.js
CHANGED
|
@@ -115,6 +115,28 @@ export function createSettings(db) {
|
|
|
115
115
|
removeCommand(id) {
|
|
116
116
|
const result = db.prepare("DELETE FROM custom_commands WHERE id = ?").run(id);
|
|
117
117
|
return result.changes > 0;
|
|
118
|
+
},
|
|
119
|
+
|
|
120
|
+
/** Vector-search provider config (OpenAI-compatible embeddings endpoint). */
|
|
121
|
+
getVectorConfig() {
|
|
122
|
+
const raw = getSetting("vector");
|
|
123
|
+
if (!raw) return undefined;
|
|
124
|
+
try {
|
|
125
|
+
const cfg = JSON.parse(raw);
|
|
126
|
+
return typeof cfg === "object" && cfg !== null ? cfg : undefined;
|
|
127
|
+
} catch {
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
setVectorConfig({ enabled, baseUrl, apiKey, model }) {
|
|
132
|
+
const cfg = {
|
|
133
|
+
enabled: enabled === true || enabled === 1,
|
|
134
|
+
baseUrl: String(baseUrl ?? "").trim().replace(/\/+$/, ""),
|
|
135
|
+
apiKey: String(apiKey ?? "").trim(),
|
|
136
|
+
model: String(model ?? "").trim()
|
|
137
|
+
};
|
|
138
|
+
setSetting("vector", JSON.stringify(cfg));
|
|
139
|
+
return cfg;
|
|
118
140
|
}
|
|
119
141
|
};
|
|
120
142
|
}
|
package/lib/store.js
CHANGED
|
@@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS memories (
|
|
|
12
12
|
forgotten INTEGER NOT NULL DEFAULT 0,
|
|
13
13
|
archived INTEGER NOT NULL DEFAULT 0,
|
|
14
14
|
source TEXT,
|
|
15
|
+
embedding TEXT,
|
|
15
16
|
created_at TEXT NOT NULL,
|
|
16
17
|
updated_at TEXT NOT NULL
|
|
17
18
|
);
|
|
@@ -64,11 +65,14 @@ export function createStore(path) {
|
|
|
64
65
|
db.exec("PRAGMA journal_mode = WAL;");
|
|
65
66
|
db.exec(SCHEMA);
|
|
66
67
|
|
|
67
|
-
// Schema
|
|
68
|
+
// Schema migrations for legacy databases (idempotent).
|
|
68
69
|
const columns = db.prepare("PRAGMA table_info(memories)").all().map((c) => c.name);
|
|
69
70
|
if (!columns.includes("archived")) {
|
|
70
71
|
db.exec("ALTER TABLE memories ADD COLUMN archived INTEGER NOT NULL DEFAULT 0");
|
|
71
72
|
}
|
|
73
|
+
if (!columns.includes("embedding")) {
|
|
74
|
+
db.exec("ALTER TABLE memories ADD COLUMN embedding TEXT");
|
|
75
|
+
}
|
|
72
76
|
|
|
73
77
|
// Per-instance monotonic timestamp guard: consecutive writes within the same
|
|
74
78
|
// millisecond must still produce strictly increasing timestamps (test asserts
|
|
@@ -117,10 +121,13 @@ export function createStore(path) {
|
|
|
117
121
|
const now = nowIso();
|
|
118
122
|
const tags = JSON.stringify(memory.tags ?? []);
|
|
119
123
|
const importance = Number.isInteger(memory.importance) ? memory.importance : 3;
|
|
124
|
+
const embedding = Array.isArray(memory.embedding) && memory.embedding.length
|
|
125
|
+
? JSON.stringify(memory.embedding)
|
|
126
|
+
: null;
|
|
120
127
|
db.prepare(
|
|
121
|
-
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, created_at, updated_at)
|
|
122
|
-
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?)`
|
|
123
|
-
).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, now, now);
|
|
128
|
+
`INSERT INTO memories (id, type, title, content, tags, importance, forgotten, source, embedding, created_at, updated_at)
|
|
129
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`
|
|
130
|
+
).run(id, type, memory.title, memory.content, tags, importance, memory.source ?? null, embedding, now, now);
|
|
124
131
|
return getById(id);
|
|
125
132
|
}
|
|
126
133
|
|
|
@@ -133,8 +140,11 @@ export function createStore(path) {
|
|
|
133
140
|
throw new Error("tags must be an array");
|
|
134
141
|
}
|
|
135
142
|
const now = nowIso();
|
|
143
|
+
const embedding = patch.embedding !== undefined
|
|
144
|
+
? (Array.isArray(patch.embedding) && patch.embedding.length ? JSON.stringify(patch.embedding) : null)
|
|
145
|
+
: existing.embedding ?? null;
|
|
136
146
|
db.prepare(
|
|
137
|
-
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, updated_at=? WHERE id=?`
|
|
147
|
+
`UPDATE memories SET type=?, title=?, content=?, tags=?, importance=?, source=?, embedding=?, updated_at=? WHERE id=?`
|
|
138
148
|
).run(
|
|
139
149
|
type,
|
|
140
150
|
patch.title ?? existing.title,
|
|
@@ -142,6 +152,7 @@ export function createStore(path) {
|
|
|
142
152
|
JSON.stringify(patch.tags ?? existing.tags),
|
|
143
153
|
Number.isInteger(patch.importance) ? patch.importance : existing.importance,
|
|
144
154
|
patch.source !== undefined ? patch.source : (existing.source ?? null),
|
|
155
|
+
embedding,
|
|
145
156
|
now,
|
|
146
157
|
id
|
|
147
158
|
);
|
|
@@ -190,6 +201,27 @@ export function createStore(path) {
|
|
|
190
201
|
return rows.map(toRow);
|
|
191
202
|
}
|
|
192
203
|
|
|
204
|
+
/** Set (or clear with null) the embedding vector of a memory. */
|
|
205
|
+
function setEmbedding(id, vector) {
|
|
206
|
+
const json = Array.isArray(vector) && vector.length ? JSON.stringify(vector) : null;
|
|
207
|
+
db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(json, id);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function embeddedCount() {
|
|
211
|
+
return db.prepare(
|
|
212
|
+
"SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != ''"
|
|
213
|
+
).get().c;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Candidate rows still missing an embedding, for incremental re-indexing. */
|
|
217
|
+
function needsEmbedding(limit = 50) {
|
|
218
|
+
return db.prepare(
|
|
219
|
+
`SELECT id, title, content FROM memories
|
|
220
|
+
WHERE embedding IS NULL OR embedding = ''
|
|
221
|
+
ORDER BY updated_at DESC LIMIT ?`
|
|
222
|
+
).all(limit);
|
|
223
|
+
}
|
|
224
|
+
|
|
193
225
|
function search(query, { limit = 20, includeArchived = false } = {}) {
|
|
194
226
|
const q = String(query).trim();
|
|
195
227
|
if (!q) return [];
|
|
@@ -212,6 +244,49 @@ export function createStore(path) {
|
|
|
212
244
|
return rows.map(toRow);
|
|
213
245
|
}
|
|
214
246
|
|
|
247
|
+
// --- vector search ------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
function cosine(a, b) {
|
|
250
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length || !a.length) return 0;
|
|
251
|
+
let dot = 0;
|
|
252
|
+
let na = 0;
|
|
253
|
+
let nb = 0;
|
|
254
|
+
for (let i = 0; i < a.length; i++) {
|
|
255
|
+
dot += a[i] * b[i];
|
|
256
|
+
na += a[i] * a[i];
|
|
257
|
+
nb += b[i] * b[i];
|
|
258
|
+
}
|
|
259
|
+
if (na === 0 || nb === 0) return 0;
|
|
260
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Brute-force cosine similarity over embedded rows. Returns rows decorated
|
|
265
|
+
* with a `score` (0..1). Only rows with a stored embedding participate.
|
|
266
|
+
*/
|
|
267
|
+
function searchVector(vector, { limit = 20, includeArchived = false, threshold = 0 } = {}) {
|
|
268
|
+
if (!Array.isArray(vector) || !vector.length) return [];
|
|
269
|
+
const archivedFilter = includeArchived ? "" : "archived = 0 AND ";
|
|
270
|
+
const rows = db.prepare(
|
|
271
|
+
`SELECT * FROM memories
|
|
272
|
+
WHERE ${archivedFilter}forgotten = 0 AND embedding IS NOT NULL AND embedding != ''`
|
|
273
|
+
).all();
|
|
274
|
+
const scored = [];
|
|
275
|
+
for (const row of rows) {
|
|
276
|
+
let v;
|
|
277
|
+
try {
|
|
278
|
+
v = JSON.parse(row.embedding);
|
|
279
|
+
} catch {
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
const score = cosine(vector, v);
|
|
283
|
+
if (score >= threshold) scored.push({ row, score });
|
|
284
|
+
}
|
|
285
|
+
scored.sort((a, b) => b.score - a.score);
|
|
286
|
+
const { limit: lim } = sanitizePage(limit, 0, 20);
|
|
287
|
+
return scored.slice(0, lim).map(({ row, score }) => ({ ...toRow(row), score }));
|
|
288
|
+
}
|
|
289
|
+
|
|
215
290
|
return {
|
|
216
291
|
db,
|
|
217
292
|
count,
|
|
@@ -224,6 +299,10 @@ export function createStore(path) {
|
|
|
224
299
|
list,
|
|
225
300
|
all,
|
|
226
301
|
search,
|
|
302
|
+
setEmbedding,
|
|
303
|
+
embeddedCount,
|
|
304
|
+
needsEmbedding,
|
|
305
|
+
searchVector,
|
|
227
306
|
close() {
|
|
228
307
|
db.close();
|
|
229
308
|
}
|
package/lib/tools.js
CHANGED
|
@@ -21,7 +21,7 @@ const MEMORY_ITEM_SCHEMA = {
|
|
|
21
21
|
}
|
|
22
22
|
};
|
|
23
23
|
|
|
24
|
-
export function createTools(ctx, service, config) {
|
|
24
|
+
export function createTools(ctx, service, config, embedder) {
|
|
25
25
|
const tools = [
|
|
26
26
|
defineTool({
|
|
27
27
|
name: "memory_save",
|
|
@@ -63,10 +63,12 @@ export function createTools(ctx, service, config) {
|
|
|
63
63
|
|
|
64
64
|
defineTool({
|
|
65
65
|
name: "memory_search",
|
|
66
|
-
description: "
|
|
66
|
+
description: "Search the cross-session memory store. Use when you need past context: how a problem was solved, user preferences, project decisions. Substring-matches title/content/tags, and optionally augments results with semantic (vector) recall when an embeddings provider is configured. Returns matching entries with source and timestamps.",
|
|
67
67
|
parameters: {
|
|
68
68
|
query: { type: "string", required: true, description: "Search text; substring match over title/content/tags" },
|
|
69
|
-
limit: { type: "integer", description: "Max results (default 20)" }
|
|
69
|
+
limit: { type: "integer", description: "Max results (default 20)" },
|
|
70
|
+
mode: { type: "string", enum: ["auto", "keyword", "vector"], description: "auto (default) = keyword hits first + vector fill when enabled; keyword = text only; vector = semantic recall first (falls back to keyword)" },
|
|
71
|
+
semantic: { type: "boolean", description: "Shorthand: enable semantic (vector) recall (same as mode=vector when true)" }
|
|
70
72
|
},
|
|
71
73
|
output: {
|
|
72
74
|
schema: {
|
|
@@ -82,7 +84,25 @@ export function createTools(ctx, service, config) {
|
|
|
82
84
|
render: (_args, value) => TEXT_OUTPUT(`Found ${value.items.length} memory entr${value.items.length === 1 ? "y" : "ies"}.`)
|
|
83
85
|
},
|
|
84
86
|
async execute(args) {
|
|
85
|
-
const
|
|
87
|
+
const limit = args.limit ?? 20;
|
|
88
|
+
const rows = service.toApiList(service.search(args.query, { limit }));
|
|
89
|
+
const wantVector = args.mode === "vector" || args.semantic === true || args.mode === "auto";
|
|
90
|
+
if (wantVector && embedder) {
|
|
91
|
+
try {
|
|
92
|
+
const vector = await embedder.embed(args.query);
|
|
93
|
+
if (vector) {
|
|
94
|
+
const scored = service.toApiList(service.searchVector(vector, { limit }));
|
|
95
|
+
const seen = new Set(rows.map((m) => m.id));
|
|
96
|
+
for (const m of scored) {
|
|
97
|
+
if (rows.length >= limit) break;
|
|
98
|
+
if (!seen.has(m.id)) {
|
|
99
|
+
seen.add(m.id);
|
|
100
|
+
rows.push(m);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
} catch { /* vector unavailable: keep keyword results */ }
|
|
105
|
+
}
|
|
86
106
|
return { items: rows };
|
|
87
107
|
}
|
|
88
108
|
}),
|
package/package.json
CHANGED
|
@@ -1,58 +1,58 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@modusensus/dsh-mneme",
|
|
3
|
-
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, and a Web GUI panel",
|
|
4
|
-
"version": "0.1.
|
|
5
|
-
"license": "MIT",
|
|
6
|
-
"type": "module",
|
|
7
|
-
"main": "lib/index.js",
|
|
8
|
-
"exports": {
|
|
9
|
-
".": {
|
|
10
|
-
"default": "./lib/index.js"
|
|
11
|
-
},
|
|
12
|
-
"./client": {
|
|
13
|
-
"default": "./lib/client.js"
|
|
14
|
-
},
|
|
15
|
-
"./package.json": "./package.json"
|
|
16
|
-
},
|
|
17
|
-
"files": [
|
|
18
|
-
"lib",
|
|
19
|
-
"src",
|
|
20
|
-
"cordis.patch.yml"
|
|
21
|
-
],
|
|
22
|
-
"dsh": {
|
|
23
|
-
"client": {
|
|
24
|
-
"inject": [
|
|
25
|
-
"slots",
|
|
26
|
-
"locale",
|
|
27
|
-
"layout",
|
|
28
|
-
"connection"
|
|
29
|
-
],
|
|
30
|
-
"platform": "web"
|
|
31
|
-
},
|
|
32
|
-
"bundle": {
|
|
33
|
-
"patch": "./cordis.patch.yml"
|
|
34
|
-
}
|
|
35
|
-
},
|
|
36
|
-
"peerDependencies": {
|
|
37
|
-
"@deepseek-ai/cordis": "^4.0.1",
|
|
38
|
-
"@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
|
|
39
|
-
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
|
40
|
-
"@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
|
|
41
|
-
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
|
42
|
-
"@deepseek-ai/schemastery": "^3.18.1"
|
|
43
|
-
},
|
|
44
|
-
"devDependencies": {
|
|
45
|
-
"@deepseek-ai/cordis": "^4.0.1",
|
|
46
|
-
"@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
|
|
47
|
-
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
|
48
|
-
"@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
|
|
49
|
-
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
|
50
|
-
"@deepseek-ai/schemastery": "^3.18.1"
|
|
51
|
-
},
|
|
52
|
-
"scripts": {
|
|
53
|
-
"sync": "node scripts/sync-lib.js",
|
|
54
|
-
"prepack": "npm run sync",
|
|
55
|
-
"test": "node --test --test-isolation=none test/*.test.js",
|
|
56
|
-
"e2e": "node scripts/e2e-dsh.js"
|
|
57
|
-
}
|
|
58
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "@modusensus/dsh-mneme",
|
|
3
|
+
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 6 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
+
"version": "0.1.6",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"main": "lib/index.js",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"default": "./lib/index.js"
|
|
11
|
+
},
|
|
12
|
+
"./client": {
|
|
13
|
+
"default": "./lib/client.js"
|
|
14
|
+
},
|
|
15
|
+
"./package.json": "./package.json"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"lib",
|
|
19
|
+
"src",
|
|
20
|
+
"cordis.patch.yml"
|
|
21
|
+
],
|
|
22
|
+
"dsh": {
|
|
23
|
+
"client": {
|
|
24
|
+
"inject": [
|
|
25
|
+
"slots",
|
|
26
|
+
"locale",
|
|
27
|
+
"layout",
|
|
28
|
+
"connection"
|
|
29
|
+
],
|
|
30
|
+
"platform": "web"
|
|
31
|
+
},
|
|
32
|
+
"bundle": {
|
|
33
|
+
"patch": "./cordis.patch.yml"
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"peerDependencies": {
|
|
37
|
+
"@deepseek-ai/cordis": "^4.0.1",
|
|
38
|
+
"@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
|
|
39
|
+
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
|
40
|
+
"@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
|
|
41
|
+
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
|
42
|
+
"@deepseek-ai/schemastery": "^3.18.1"
|
|
43
|
+
},
|
|
44
|
+
"devDependencies": {
|
|
45
|
+
"@deepseek-ai/cordis": "^4.0.1",
|
|
46
|
+
"@deepseek-ai/dsh-host-webserver": "^0.1.0-rc.6",
|
|
47
|
+
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
|
48
|
+
"@deepseek-ai/dsh-system-prompt": "^0.1.0-rc.6",
|
|
49
|
+
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
|
50
|
+
"@deepseek-ai/schemastery": "^3.18.1"
|
|
51
|
+
},
|
|
52
|
+
"scripts": {
|
|
53
|
+
"sync": "node scripts/sync-lib.js",
|
|
54
|
+
"prepack": "npm run sync",
|
|
55
|
+
"test": "node --test --test-isolation=none test/*.test.js",
|
|
56
|
+
"e2e": "node scripts/e2e-dsh.js"
|
|
57
|
+
}
|
|
58
|
+
}
|
package/src/api.js
CHANGED
|
@@ -23,7 +23,7 @@ function parseBody(text) {
|
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
export function createApi(ctx, service, settings, commands) {
|
|
26
|
+
export function createApi(ctx, service, settings, commands, embedder) {
|
|
27
27
|
const disposers = [];
|
|
28
28
|
|
|
29
29
|
const register = (route) => {
|
|
@@ -64,8 +64,49 @@ export function createApi(ctx, service, settings, commands) {
|
|
|
64
64
|
const url = new URL(req.url, "http://localhost");
|
|
65
65
|
const q = url.searchParams.get("q") ?? "";
|
|
66
66
|
const limit = Number(url.searchParams.get("limit") ?? 20);
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
// mode: auto (default) | keyword | vector
|
|
68
|
+
const mode = url.searchParams.get("mode") ?? "auto";
|
|
69
|
+
const query = q.trim();
|
|
70
|
+
if (!query) {
|
|
71
|
+
sendJson(res, 200, { items: [], mode: "keyword" });
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
// Keyword results (existing behavior) always computed; used as a
|
|
75
|
+
// fallback and as the primary ranking when vector is unavailable.
|
|
76
|
+
const keyword = service.toApiList(service.search(query, { limit }));
|
|
77
|
+
if (mode === "keyword" || !embedder) {
|
|
78
|
+
sendJson(res, 200, { items: keyword, mode: "keyword" });
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const cfg = settings.getVectorConfig();
|
|
82
|
+
if (mode === "vector" && !cfg?.enabled) {
|
|
83
|
+
sendJson(res, 200, { items: keyword, mode: "keyword", error: "vector-disabled" });
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
// Try vector search; on any failure fall back to keyword results.
|
|
87
|
+
return embedder.embed(query).then(async (vector) => {
|
|
88
|
+
let items = keyword;
|
|
89
|
+
let used = "keyword";
|
|
90
|
+
if (vector) {
|
|
91
|
+
const scored = service.toApiList(service.searchVector(vector, { limit }));
|
|
92
|
+
// Merge: keyword exact hits first (they are the user's literal
|
|
93
|
+
// words), then vector results fill the remaining slots, deduped.
|
|
94
|
+
const seen = new Set(keyword.map((m) => m.id));
|
|
95
|
+
const merged = [...keyword];
|
|
96
|
+
for (const m of scored) {
|
|
97
|
+
if (merged.length >= limit) break;
|
|
98
|
+
if (!seen.has(m.id)) {
|
|
99
|
+
seen.add(m.id);
|
|
100
|
+
merged.push(m);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
items = merged;
|
|
104
|
+
used = "vector";
|
|
105
|
+
}
|
|
106
|
+
sendJson(res, 200, { items, mode: used });
|
|
107
|
+
}).catch(() => {
|
|
108
|
+
sendJson(res, 200, { items: keyword, mode: "keyword" });
|
|
109
|
+
});
|
|
69
110
|
} catch {
|
|
70
111
|
sendJson(res, 500, { error: "internal" });
|
|
71
112
|
}
|
|
@@ -112,6 +153,54 @@ export function createApi(ctx, service, settings, commands) {
|
|
|
112
153
|
}
|
|
113
154
|
});
|
|
114
155
|
|
|
156
|
+
// --- vector search config ---
|
|
157
|
+
register({
|
|
158
|
+
kind: "exact",
|
|
159
|
+
path: "/api/dsh-mneme/vector-config",
|
|
160
|
+
handler(req, res) {
|
|
161
|
+
try {
|
|
162
|
+
if (req.method === "PUT" || req.method === "POST") {
|
|
163
|
+
return readBody(req).then((text) => {
|
|
164
|
+
const body = parseBody(text);
|
|
165
|
+
const cfg = settings.setVectorConfig({
|
|
166
|
+
enabled: body.enabled,
|
|
167
|
+
baseUrl: body.baseUrl,
|
|
168
|
+
apiKey: body.apiKey,
|
|
169
|
+
model: body.model
|
|
170
|
+
});
|
|
171
|
+
sendJson(res, 200, { config: cfg });
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
sendJson(res, 200, { config: settings.getVectorConfig() ?? { enabled: false, baseUrl: "", apiKey: "", model: "" } });
|
|
175
|
+
} catch {
|
|
176
|
+
sendJson(res, 500, { error: "internal" });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
// --- vector re-index (backfill embeddings for rows missing them) ---
|
|
182
|
+
register({
|
|
183
|
+
kind: "exact",
|
|
184
|
+
path: "/api/dsh-mneme/vector-reindex",
|
|
185
|
+
handler(req, res) {
|
|
186
|
+
try {
|
|
187
|
+
if (!embedder) {
|
|
188
|
+
sendJson(res, 200, { indexed: 0, skipped: 0, error: "vector-unavailable" });
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
const url = new URL(req.url, "http://localhost");
|
|
192
|
+
const limit = Number(url.searchParams.get("limit") ?? 100);
|
|
193
|
+
embedder.reindexMissing(limit).then((result) => {
|
|
194
|
+
sendJson(res, 200, result);
|
|
195
|
+
}).catch(() => {
|
|
196
|
+
sendJson(res, 200, { indexed: 0, skipped: 0, error: "vector-failed" });
|
|
197
|
+
});
|
|
198
|
+
} catch {
|
|
199
|
+
sendJson(res, 500, { error: "internal" });
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
|
|
115
204
|
// --- custom commands ---
|
|
116
205
|
register({
|
|
117
206
|
kind: "exact",
|
package/src/embedding.js
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// OpenAI-compatible embedding client for vector search. DSH's LLM service is
|
|
2
|
+
// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
|
|
3
|
+
// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
|
|
4
|
+
// proxy) and any provider exposing the standard embeddings API.
|
|
5
|
+
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
|
+
|
|
7
|
+
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
8
|
+
function embeddingsUrl(baseUrl) {
|
|
9
|
+
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
10
|
+
if (!base) return "";
|
|
11
|
+
// Accept both "https://host/v1" and a full path ending in /embeddings.
|
|
12
|
+
if (/\/embeddings$/i.test(base)) return base;
|
|
13
|
+
return `${base}/embeddings`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Call the embeddings API for one text. Resolves to a Float64 array, or null
|
|
18
|
+
* when the provider is not configured, the call fails, or the response is
|
|
19
|
+
* unusable. Never throws: failures degrade to keyword search.
|
|
20
|
+
*/
|
|
21
|
+
export async function embedText({ baseUrl, apiKey, model }, text) {
|
|
22
|
+
const url = embeddingsUrl(baseUrl);
|
|
23
|
+
if (!url || !apiKey || !model || !text) return null;
|
|
24
|
+
let res;
|
|
25
|
+
try {
|
|
26
|
+
res = await fetch(url, {
|
|
27
|
+
method: "POST",
|
|
28
|
+
headers: {
|
|
29
|
+
"Content-Type": "application/json",
|
|
30
|
+
"Authorization": `Bearer ${apiKey}`
|
|
31
|
+
},
|
|
32
|
+
body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
|
|
33
|
+
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
|
|
34
|
+
});
|
|
35
|
+
} catch {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
if (!res.ok) return null;
|
|
39
|
+
let body;
|
|
40
|
+
try {
|
|
41
|
+
body = await res.json();
|
|
42
|
+
} catch {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
const vec = body?.data?.[0]?.embedding;
|
|
46
|
+
return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
51
|
+
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
52
|
+
* flaky embedding endpoint never breaks memory writes.
|
|
53
|
+
*/
|
|
54
|
+
export function createEmbedder({ store, settings, logger }) {
|
|
55
|
+
async function embedFor(id, title, content) {
|
|
56
|
+
const cfg = settings.getVectorConfig();
|
|
57
|
+
if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return;
|
|
58
|
+
const text = [title, content].filter(Boolean).join("\n");
|
|
59
|
+
const vector = await embedText(cfg, text);
|
|
60
|
+
if (vector) {
|
|
61
|
+
store.setEmbedding(id, vector);
|
|
62
|
+
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
/** Fire-and-forget re-embed of a memory after any write. */
|
|
68
|
+
schedule(memory) {
|
|
69
|
+
if (!memory?.id) return;
|
|
70
|
+
embedFor(memory.id, memory.title, memory.content).catch(() => {});
|
|
71
|
+
},
|
|
72
|
+
|
|
73
|
+
/** Embed one text and return its vector (null on failure/disabled). */
|
|
74
|
+
async embed(query) {
|
|
75
|
+
const cfg = settings.getVectorConfig();
|
|
76
|
+
if (!cfg?.enabled || !cfg.baseUrl || !cfg.apiKey || !cfg.model) return null;
|
|
77
|
+
return embedText(cfg, query);
|
|
78
|
+
},
|
|
79
|
+
|
|
80
|
+
/** Batch re-index rows still missing an embedding. */
|
|
81
|
+
async reindexMissing(limit = 50) {
|
|
82
|
+
const cfg = settings.getVectorConfig();
|
|
83
|
+
if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
|
|
84
|
+
const rows = store.needsEmbedding(limit);
|
|
85
|
+
let indexed = 0;
|
|
86
|
+
for (const row of rows) {
|
|
87
|
+
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
88
|
+
const vector = await embedText(cfg, text);
|
|
89
|
+
if (vector) {
|
|
90
|
+
store.setEmbedding(row.id, vector);
|
|
91
|
+
indexed++;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return { indexed, skipped: rows.length - indexed };
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
}
|
package/src/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import { createDreamScheduler } from "./dream.js";
|
|
|
8
8
|
import { createApi } from "./api.js";
|
|
9
9
|
import { createSettings } from "./settings.js";
|
|
10
10
|
import { createCommandManager } from "./commands.js";
|
|
11
|
+
import { createEmbedder } from "./embedding.js";
|
|
11
12
|
import { Config } from "./config.js";
|
|
12
13
|
import { mkdirSync } from "node:fs";
|
|
13
14
|
import { join } from "node:path";
|
|
@@ -39,6 +40,11 @@ export const apply = (ctx, config) => {
|
|
|
39
40
|
// same SQLite file but live in dedicated tables, isolated from memories.
|
|
40
41
|
const settings = createSettings(store.db);
|
|
41
42
|
|
|
43
|
+
// Vector search: embedder calls the configured OpenAI-compatible embeddings
|
|
44
|
+
// endpoint on writes and for queries. service re-embeds after each write.
|
|
45
|
+
const embedder = createEmbedder({ store, settings, logger: ctx.logger });
|
|
46
|
+
service.setEmbedder(embedder);
|
|
47
|
+
|
|
42
48
|
// Custom commands: register persisted commands into the DSH command registry
|
|
43
49
|
// on boot; add/remove re-register live through the API.
|
|
44
50
|
let commands = null;
|
|
@@ -85,7 +91,7 @@ export const apply = (ctx, config) => {
|
|
|
85
91
|
});
|
|
86
92
|
|
|
87
93
|
ctx.inject(["tools"], (toolsCtx) => {
|
|
88
|
-
disposers.push(createTools(toolsCtx, service, cfg));
|
|
94
|
+
disposers.push(createTools(toolsCtx, service, cfg, embedder));
|
|
89
95
|
});
|
|
90
96
|
|
|
91
97
|
const summarizer = createSummarizer(ctx, service, cfg);
|
|
@@ -96,7 +102,7 @@ export const apply = (ctx, config) => {
|
|
|
96
102
|
add: () => { throw new Error("commands unavailable"); },
|
|
97
103
|
remove: () => false,
|
|
98
104
|
list: () => []
|
|
99
|
-
});
|
|
105
|
+
}, embedder);
|
|
100
106
|
disposers.push(api.dispose);
|
|
101
107
|
}
|
|
102
108
|
|