@modusensus/dsh-mneme 0.4.7 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -5
- package/lib/api.js +117 -1
- package/lib/client.js +1013 -216
- package/lib/config.js +30 -1
- package/lib/dream.js +41 -12
- package/lib/hot-memory.js +53 -0
- package/lib/inject.js +84 -3
- package/lib/local-embedder.js +7 -1
- package/lib/reranker.js +4 -1
- package/lib/search/adaptive.js +22 -0
- package/lib/search/bm25.js +96 -0
- package/lib/service.js +138 -9
- package/lib/store.js +24 -0
- package/package.json +9 -1
- package/scripts/benchmark-recall.js +133 -0
- package/scripts/sync-lib.js +7 -2
- package/src/api.js +117 -1
- package/src/config.js +30 -1
- package/src/dream.js +41 -12
- package/src/hot-memory.js +53 -0
- package/src/inject.js +84 -3
- package/src/local-embedder.js +7 -1
- package/src/reranker.js +4 -1
- package/src/search/adaptive.js +22 -0
- package/src/search/bm25.js +96 -0
- package/src/service.js +138 -9
- package/src/store.js +24 -0
- package/test/benchmark.test.js +35 -0
- package/test/client.test.js +205 -15
- package/test/graph-api.test.js +175 -0
- package/test/hot-memory.test.js +174 -0
- package/test/reasoning-effort.test.js +1 -1
- package/test/recall-layer.test.js +2 -2
- package/test/reranker.test.js +43 -0
- package/test/search-fusion.test.js +90 -0
- package/test/service-search.test.js +31 -2
package/lib/store.js
CHANGED
|
@@ -911,6 +911,29 @@ export function createStore(path) {
|
|
|
911
911
|
db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(json, id);
|
|
912
912
|
}
|
|
913
913
|
|
|
914
|
+
/** Batch fetch stored embeddings by id (v0.5.0 search-time semantic dedup).
|
|
915
|
+
* Returns a Map(id → number[]); rows without a parseable embedding are
|
|
916
|
+
* simply absent from the map. */
|
|
917
|
+
function getEmbeddings(ids) {
|
|
918
|
+
const out = new Map();
|
|
919
|
+
const list = (Array.isArray(ids) ? ids : []).filter(Boolean);
|
|
920
|
+
for (let i = 0; i < list.length; i += 100) {
|
|
921
|
+
const chunk = list.slice(i, i + 100);
|
|
922
|
+
const rows = db.prepare(
|
|
923
|
+
`SELECT id, embedding FROM memories
|
|
924
|
+
WHERE embedding IS NOT NULL AND embedding != ''
|
|
925
|
+
AND id IN (${chunk.map(() => "?").join(",")})`
|
|
926
|
+
).all(...chunk);
|
|
927
|
+
for (const row of rows) {
|
|
928
|
+
try {
|
|
929
|
+
const vec = JSON.parse(row.embedding);
|
|
930
|
+
if (Array.isArray(vec) && vec.length) out.set(row.id, vec);
|
|
931
|
+
} catch { /* corrupt row: skip */ }
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
return out;
|
|
935
|
+
}
|
|
936
|
+
|
|
914
937
|
function embeddedCount() {
|
|
915
938
|
return db.prepare(
|
|
916
939
|
"SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != ''"
|
|
@@ -1852,6 +1875,7 @@ export function createStore(path) {
|
|
|
1852
1875
|
all,
|
|
1853
1876
|
search,
|
|
1854
1877
|
setEmbedding,
|
|
1878
|
+
getEmbeddings,
|
|
1855
1879
|
embeddedCount,
|
|
1856
1880
|
needsEmbedding,
|
|
1857
1881
|
searchVector,
|
package/package.json
CHANGED
|
@@ -1,8 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modusensus/dsh-mneme",
|
|
3
3
|
"description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 7 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.5.1",
|
|
5
5
|
"license": "MIT",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/modusensus/dsh-mneme.git"
|
|
9
|
+
},
|
|
10
|
+
"homepage": "https://github.com/modusensus/dsh-mneme#readme",
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/modusensus/dsh-mneme/issues"
|
|
13
|
+
},
|
|
6
14
|
"type": "module",
|
|
7
15
|
"main": "lib/index.js",
|
|
8
16
|
"exports": {
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// Recall benchmark (v0.5.0 评测体系): a self-contained harness that seeds an
|
|
2
|
+
// in-memory store with labelled memories, runs a standard query set through
|
|
3
|
+
// the real searchMemories pipeline, and reports Recall@K / MRR per case and
|
|
4
|
+
// in aggregate. Runs in two configurations so the BM25/third-path lift is
|
|
5
|
+
// visible: `legacy` (bm25 + adaptive + dedup off) vs `fused` (defaults on).
|
|
6
|
+
//
|
|
7
|
+
// Usage:
|
|
8
|
+
// node scripts/benchmark-recall.js # run both configurations
|
|
9
|
+
// node scripts/benchmark-recall.js --json # machine-readable output
|
|
10
|
+
// The harness exports runBenchmark()/TEST_CASES for the test suite; the CLI
|
|
11
|
+
// path below only executes when invoked directly.
|
|
12
|
+
import { createStore } from "../src/store.js";
|
|
13
|
+
import { createService } from "../src/service.js";
|
|
14
|
+
import { createVectorIndex } from "../src/vector-index.js";
|
|
15
|
+
|
|
16
|
+
// Deterministic toy embedder: bag-of-words hashed into a fixed-dimension
|
|
17
|
+
// vector, so cosine similarity ≈ lexical overlap. Good enough to exercise
|
|
18
|
+
// the vector path mechanically — semantic quality is not under test here.
|
|
19
|
+
const DIM = 256;
|
|
20
|
+
function hashVec(text) {
|
|
21
|
+
const v = new Array(DIM).fill(0);
|
|
22
|
+
const tokens = String(text ?? "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
|
23
|
+
for (const t of tokens) {
|
|
24
|
+
let h = 0;
|
|
25
|
+
for (const ch of t) h = (h * 31 + ch.codePointAt(0)) >>> 0;
|
|
26
|
+
v[h % DIM] += 1;
|
|
27
|
+
}
|
|
28
|
+
const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
|
|
29
|
+
return v.map((x) => x / norm);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const SEED = [
|
|
33
|
+
{ id: "mem_user_pref", type: "preference", title: "编辑器偏好", content: "用户偏好 VS Code,深色主题,等宽字体 JetBrains Mono", importance: 4, tags: ["editor"] },
|
|
34
|
+
{ id: "mem_user_project", type: "project", title: "dsh-mneme 插件项目", content: "用户在开发 dsh-mneme 记忆插件,TypeScript 与 cordis 框架", importance: 5, tags: ["plugin"] },
|
|
35
|
+
{ id: "mem_rust_switch", type: "decision", title: "语言迁移决策", content: "项目编译模块从 Go 迁移到 Rust,理由是内存安全", importance: 4, tags: ["rust"] },
|
|
36
|
+
{ id: "mem_zfs_bug", type: "project", title: "ZFS-4421 数据损坏", content: "线上池 ZFS-4421 出现 checksum 错误,根因是 HBA 固件 bug", importance: 5, tags: ["ops"] },
|
|
37
|
+
{ id: "mem_city_thesis", type: "project", title: "湿地论文", content: "毕业论文研究城市湿地公园周边开发案例,ArcGIS 空间分析", importance: 4, tags: ["thesis"] },
|
|
38
|
+
{ id: "mem_async_pattern", type: "decision", title: "异步并发模式", content: "async runtime 选用 tokio,任务用 spawn 管理,channel 通信", importance: 3, tags: ["rust"] },
|
|
39
|
+
{ id: "mem_python_etl", type: "project", title: "ETL 脚本", content: "夜间 ETL 用 Python 编写,pandas 清洗,SQLite 落地", importance: 3, tags: ["etl"] },
|
|
40
|
+
{ id: "mem_ui_style", type: "preference", title: "界面审美", content: "喜欢编辑风 brutalism 排版,低饱和度配色,衬线标题", importance: 3, tags: ["design"] }
|
|
41
|
+
];
|
|
42
|
+
|
|
43
|
+
// Standard query set: each case is a query plus the ids that MUST appear in
|
|
44
|
+
// the top-K for the case to count as a hit. Covers the three recall paths —
|
|
45
|
+
// multi-term lexical (BM25's home turf), identifier lookup, and semantic.
|
|
46
|
+
export const TEST_CASES = [
|
|
47
|
+
{ query: "rust 异步", expected: ["mem_async_pattern", "mem_rust_switch"], note: "scattered terms — BM25 territory" },
|
|
48
|
+
{ query: "ZFS-4421 checksum", expected: ["mem_zfs_bug"], note: "identifier + keyword" },
|
|
49
|
+
{ query: "插件 开发", expected: ["mem_user_project"], note: "multi-term CJK" },
|
|
50
|
+
{ query: "论文 空间分析", expected: ["mem_city_thesis"], note: "scattered CJK terms" },
|
|
51
|
+
{ query: "ETL 脚本", expected: ["mem_python_etl"], note: "mixed" },
|
|
52
|
+
{ query: "深色主题", expected: ["mem_user_pref"], note: "substring match" },
|
|
53
|
+
{ query: "channel 通信 任务", expected: ["mem_async_pattern"], note: "scattered terms" },
|
|
54
|
+
{ query: "内存安全 语言", expected: ["mem_rust_switch"], note: "scattered terms" },
|
|
55
|
+
{ query: "配色 审美", expected: ["mem_ui_style"], note: "scattered CJK" },
|
|
56
|
+
{ query: "HBA 固件", expected: ["mem_zfs_bug"], note: "scattered terms" }
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
function seedService(overrides = {}) {
|
|
60
|
+
const store = createStore(":memory:");
|
|
61
|
+
const config = {
|
|
62
|
+
bm25SearchEnabled: true,
|
|
63
|
+
adaptiveThresholdEnabled: true,
|
|
64
|
+
searchSemanticDedup: true,
|
|
65
|
+
searchSemanticDedupThreshold: 0.95,
|
|
66
|
+
selectiveInjectEnabled: true,
|
|
67
|
+
entitySearchEnabled: false,
|
|
68
|
+
...overrides
|
|
69
|
+
};
|
|
70
|
+
const service = createService({ store, mirror: null, config, logger: null });
|
|
71
|
+
const vectorIndex = createVectorIndex({ store, logger: null });
|
|
72
|
+
service.setVectorIndex(vectorIndex);
|
|
73
|
+
service.setEmbedder({
|
|
74
|
+
embedSingle: async (text) => hashVec(text)
|
|
75
|
+
});
|
|
76
|
+
for (const m of SEED) {
|
|
77
|
+
const row = store.save({ type: m.type, title: m.title, content: m.content, tags: m.tags, importance: m.importance, source: "seed" });
|
|
78
|
+
store.setEmbedding(row.id, hashVec(`${m.title} ${m.content}`));
|
|
79
|
+
}
|
|
80
|
+
return service;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export async function runBenchmark({ topK = 5, mode = "auto" } = {}) {
|
|
84
|
+
const configs = [
|
|
85
|
+
{ name: "legacy", overrides: { bm25SearchEnabled: false, adaptiveThresholdEnabled: false, searchSemanticDedup: false } },
|
|
86
|
+
{ name: "fused", overrides: {} }
|
|
87
|
+
];
|
|
88
|
+
const runs = [];
|
|
89
|
+
for (const cfg of configs) {
|
|
90
|
+
const service = seedService(cfg.overrides);
|
|
91
|
+
const rows = [];
|
|
92
|
+
let hits = 0;
|
|
93
|
+
let mrrSum = 0;
|
|
94
|
+
for (const tc of TEST_CASES) {
|
|
95
|
+
const results = await service.searchMemories(tc.query, { mode, topK, useRerank: false });
|
|
96
|
+
const ids = results.map((r) => r.id);
|
|
97
|
+
const metrics = service.computeRetrievalMetrics(ids, tc.expected);
|
|
98
|
+
if (metrics.recall === 1) hits++;
|
|
99
|
+
mrrSum += metrics.mrr;
|
|
100
|
+
rows.push({ query: tc.query, note: tc.note, expected: tc.expected, got: ids, ...metrics });
|
|
101
|
+
}
|
|
102
|
+
runs.push({
|
|
103
|
+
config: cfg.name,
|
|
104
|
+
recallAtK: +(hits / TEST_CASES.length).toFixed(3),
|
|
105
|
+
avgMrr: +(mrrSum / TEST_CASES.length).toFixed(3),
|
|
106
|
+
rows
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
return { topK, mode, runs };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function printReport(report) {
|
|
113
|
+
for (const run of report.runs) {
|
|
114
|
+
console.log(`\n=== ${run.config} (topK=${report.topK}, mode=${report.mode}) ===`);
|
|
115
|
+
for (const r of run.rows) {
|
|
116
|
+
const ok = r.recall === 1 ? "PASS" : "MISS";
|
|
117
|
+
console.log(` [${ok}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
|
|
118
|
+
if (r.recall < 1) console.log(` expected ⊇ ${r.expected.join(", ")} got: ${r.got.join(", ") || "—"}`);
|
|
119
|
+
}
|
|
120
|
+
console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}`);
|
|
121
|
+
}
|
|
122
|
+
const [legacy, fused] = report.runs;
|
|
123
|
+
const lift = ((fused.recallAtK - legacy.recallAtK) * 100).toFixed(1);
|
|
124
|
+
console.log(`\n三路融合 vs 旧两路: Recall@${report.topK} ${legacy.recallAtK * 100}% → ${fused.recallAtK * 100}% (${lift >= 0 ? "+" : ""}${lift}pp)`);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const invokedDirectly = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/").split("/").pop() ?? "");
|
|
128
|
+
if (invokedDirectly) {
|
|
129
|
+
const asJson = process.argv.includes("--json");
|
|
130
|
+
const report = await runBenchmark({});
|
|
131
|
+
if (asJson) console.log(JSON.stringify(report, null, 2));
|
|
132
|
+
else printReport(report);
|
|
133
|
+
}
|
package/scripts/sync-lib.js
CHANGED
|
@@ -5,7 +5,12 @@
|
|
|
5
5
|
//
|
|
6
6
|
// Usage: npm run sync (also run automatically by `npm pack`/`npm publish`
|
|
7
7
|
// via the prepack hook, so a published tarball always ships a fresh lib/).
|
|
8
|
-
|
|
8
|
+
//
|
|
9
|
+
// Note: copyFileSync (not cpSync) is used on purpose — cpSync removes the
|
|
10
|
+
// destination first, which fails with EPERM/unlink on Windows when the path
|
|
11
|
+
// is long enough to trigger the \\?\ extended-prefix (observed on publish).
|
|
12
|
+
// copyFileSync truncates and rewrites in place, so it survives long paths.
|
|
13
|
+
import { copyFileSync, existsSync, mkdirSync, readdirSync } from "node:fs";
|
|
9
14
|
import { join, relative } from "node:path";
|
|
10
15
|
import { fileURLToPath } from "node:url";
|
|
11
16
|
|
|
@@ -31,7 +36,7 @@ for (const file of walk(srcDir)) {
|
|
|
31
36
|
const rel = relative(srcDir, file);
|
|
32
37
|
const dest = join(libDir, rel);
|
|
33
38
|
mkdirSync(join(dest, ".."), { recursive: true });
|
|
34
|
-
|
|
39
|
+
copyFileSync(file, dest);
|
|
35
40
|
copied++;
|
|
36
41
|
console.log(`synced ${rel}`);
|
|
37
42
|
}
|
package/src/api.js
CHANGED
|
@@ -103,7 +103,7 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
103
103
|
try {
|
|
104
104
|
const url = new URL(req.url, "http://localhost");
|
|
105
105
|
const q = url.searchParams.get("q") ?? "";
|
|
106
|
-
const limit = Number(url.searchParams.get("limit") ?? 20);
|
|
106
|
+
const limit = Number(url.searchParams.get("topK") ?? url.searchParams.get("limit") ?? 20);
|
|
107
107
|
// mode selects the recall strategy (defaults to auto):
|
|
108
108
|
// auto (default) keyword first, vector fills remaining slots
|
|
109
109
|
// hybrid vector first, keyword fills remaining slots; scores of
|
|
@@ -306,6 +306,122 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
306
306
|
}
|
|
307
307
|
});
|
|
308
308
|
|
|
309
|
+
// --- ego graph: 1-2 hop neighborhood of one entity (graph panel P1) ---
|
|
310
|
+
// Read-only like list/search/semantic, so it stays open when apiToken is set.
|
|
311
|
+
// BFS from the root entity over entity_relations (both directions; the
|
|
312
|
+
// idx_relations_from/to indexes keep a 2-hop walk in the tens of ms even
|
|
313
|
+
// for a few thousand nodes). `distance` on each node is the hop count from
|
|
314
|
+
// the root so the UI can shade the frontier. The API is graph-traversal
|
|
315
|
+
// only — nodes carry no attr payload; hover summaries come from
|
|
316
|
+
// /semantic/graph/entity-attrs.
|
|
317
|
+
register({
|
|
318
|
+
kind: "exact",
|
|
319
|
+
path: "/api/dsh-mneme/semantic/graph/ego",
|
|
320
|
+
handler(req, res) {
|
|
321
|
+
try {
|
|
322
|
+
const url = new URL(req.url, "http://localhost");
|
|
323
|
+
const name = (url.searchParams.get("entity") ?? "").trim();
|
|
324
|
+
if (!name) {
|
|
325
|
+
sendJson(res, 400, { error: "missing-entity" });
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
const root = service.findEntityByName?.(name);
|
|
329
|
+
if (!root) {
|
|
330
|
+
sendJson(res, 404, { error: "entity-not-found" });
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
const depth = Math.max(1, Math.min(2, Number(url.searchParams.get("depth") ?? 1) || 1));
|
|
334
|
+
const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 40) || 40));
|
|
335
|
+
|
|
336
|
+
const nodes = new Map([[root.id, { ...root, distance: 0 }]]);
|
|
337
|
+
let frontier = [root.id];
|
|
338
|
+
for (let d = 1; d <= depth && nodes.size < limit; d++) {
|
|
339
|
+
const next = [];
|
|
340
|
+
for (const id of frontier) {
|
|
341
|
+
for (const rel of service.getRelations?.(id) ?? []) {
|
|
342
|
+
const other = rel.from_entity === id ? rel.to_entity : rel.from_entity;
|
|
343
|
+
if (nodes.has(other) || nodes.size >= limit) continue;
|
|
344
|
+
const entity = service.findEntityById?.(other);
|
|
345
|
+
if (!entity) continue;
|
|
346
|
+
nodes.set(other, { ...entity, distance: d });
|
|
347
|
+
next.push(other);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
frontier = next;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// Collect every relation whose endpoints both survived the limit cut;
|
|
354
|
+
// each edge is visited twice (once per endpoint) so dedupe by id.
|
|
355
|
+
const edgeMap = new Map();
|
|
356
|
+
for (const id of nodes.keys()) {
|
|
357
|
+
for (const rel of service.getRelations?.(id) ?? []) {
|
|
358
|
+
if (nodes.has(rel.from_entity) && nodes.has(rel.to_entity)) {
|
|
359
|
+
edgeMap.set(rel.id, rel);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
sendJson(res, 200, {
|
|
365
|
+
root: { id: root.id, name: root.name, type: root.type ?? null, mention_count: root.mention_count ?? 1 },
|
|
366
|
+
nodes: [...nodes.values()].map((n) => ({
|
|
367
|
+
id: n.id,
|
|
368
|
+
name: n.name,
|
|
369
|
+
type: n.type ?? null,
|
|
370
|
+
mention_count: n.mention_count ?? 1,
|
|
371
|
+
distance: n.distance
|
|
372
|
+
})),
|
|
373
|
+
edges: [...edgeMap.values()].map((e) => ({
|
|
374
|
+
id: e.id,
|
|
375
|
+
from: e.from_entity,
|
|
376
|
+
to: e.to_entity,
|
|
377
|
+
relation_type: e.relation_type,
|
|
378
|
+
memory_id: e.memory_id ?? null,
|
|
379
|
+
created_at: e.created_at
|
|
380
|
+
}))
|
|
381
|
+
});
|
|
382
|
+
} catch {
|
|
383
|
+
sendJson(res, 500, { error: "internal" });
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
// --- entity attrs: current valid attrs for one entity (graph hover panel) ---
|
|
389
|
+
// Read-only; mirrors getCurrentAttrs (valid_until IS NULL). Also used as the
|
|
390
|
+
// graph panel's fallback list when the ego graph is too sparse to draw.
|
|
391
|
+
register({
|
|
392
|
+
kind: "exact",
|
|
393
|
+
path: "/api/dsh-mneme/semantic/graph/entity-attrs",
|
|
394
|
+
handler(req, res) {
|
|
395
|
+
try {
|
|
396
|
+
const url = new URL(req.url, "http://localhost");
|
|
397
|
+
const name = (url.searchParams.get("entity") ?? "").trim();
|
|
398
|
+
if (!name) {
|
|
399
|
+
sendJson(res, 400, { error: "missing-entity" });
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
const entity = service.findEntityByName?.(name);
|
|
403
|
+
if (!entity) {
|
|
404
|
+
sendJson(res, 404, { error: "entity-not-found" });
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
const attrs = service.getCurrentAttrs?.(entity.id) ?? [];
|
|
408
|
+
sendJson(res, 200, {
|
|
409
|
+
entity: { id: entity.id, name: entity.name, type: entity.type ?? null, mention_count: entity.mention_count ?? 1 },
|
|
410
|
+
attrs: Array.isArray(attrs)
|
|
411
|
+
? attrs.map((a) => ({
|
|
412
|
+
key: a.attr_key,
|
|
413
|
+
value: a.attr_value,
|
|
414
|
+
confidence: a.confidence ?? null,
|
|
415
|
+
valid_from: a.valid_from ?? null
|
|
416
|
+
}))
|
|
417
|
+
: []
|
|
418
|
+
});
|
|
419
|
+
} catch {
|
|
420
|
+
sendJson(res, 500, { error: "internal" });
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
});
|
|
424
|
+
|
|
309
425
|
// --- health: mirror sync state (F-NEW-03 / v0.3.6) ---
|
|
310
426
|
// Auth-gated; only returns a sanitized error code (never raw last_error which
|
|
311
427
|
// may leak paths/token-like strings/internal hosts). On state read failure it
|
package/src/config.js
CHANGED
|
@@ -17,7 +17,7 @@ export const Config = z.object({
|
|
|
17
17
|
dreamDelayMs: z.natural().min(0).max(60000).default(2000),
|
|
18
18
|
dreamProvider: z.string(),
|
|
19
19
|
dreamModel: z.string(),
|
|
20
|
-
dreamMaxTokens: z.natural().min(256).max(131072).default(
|
|
20
|
+
dreamMaxTokens: z.natural().min(256).max(131072).default(8192),
|
|
21
21
|
// Pass-through reasoning effort for dream's LLM calls. 'none' (default)
|
|
22
22
|
// omits the field so the provider's own default applies; low/medium/high
|
|
23
23
|
// are forwarded verbatim. Useful to cap reasoning spend on thinking-type
|
|
@@ -92,6 +92,35 @@ export const Config = z.object({
|
|
|
92
92
|
// rule-based pick to fill/dedupe. Empty query / no vector → legacy behavior.
|
|
93
93
|
hybridInject: z.boolean().default(true),
|
|
94
94
|
|
|
95
|
+
// --- recall optimization (v0.5.0) ----------------------------------------
|
|
96
|
+
// BM25 third recall path beside vector + LIKE keyword (1.1): per-token IDF
|
|
97
|
+
// scoring recalls rows whose query terms are scattered — identifiers, code
|
|
98
|
+
// fragments, mixed CJK/ASCII — where substring LIKE cannot match.
|
|
99
|
+
bm25SearchEnabled: z.boolean().default(true),
|
|
100
|
+
// Query-aware vector cutoff (1.2) replacing the fixed 0.65: entity:/attr:
|
|
101
|
+
// prefixes loosen to 0.5, short queries tighten to 0.7, long queries loosen
|
|
102
|
+
// to 0.6, and a decisive top-1/top-5 score gap loosens to 0.5 so the tail
|
|
103
|
+
// still reaches the reranker. Off = legacy fixed threshold behavior.
|
|
104
|
+
adaptiveThresholdEnabled: z.boolean().default(true),
|
|
105
|
+
// Session-scoped hot memory (1.3): the latest N dialogue rounds rendered
|
|
106
|
+
// ahead of the long-term recall block — short-term context that never
|
|
107
|
+
// enters the memory store.
|
|
108
|
+
hotMemoryEnabled: z.boolean().default(true),
|
|
109
|
+
hotMemoryRounds: z.natural().min(1).max(50).default(5),
|
|
110
|
+
hotMemoryMaxTokens: z.natural().min(200).max(32000).default(2000),
|
|
111
|
+
// Topic-ranked injection (2.2): when a query vector is available the whole
|
|
112
|
+
// injection candidate list is re-ordered by similarity to the current
|
|
113
|
+
// query instead of keeping the rule-based order.
|
|
114
|
+
selectiveInjectEnabled: z.boolean().default(true),
|
|
115
|
+
// Search-time semantic dedup (2.3): greedy pass over the merged candidate
|
|
116
|
+
// list dropping rows whose embedding cosine-similarity to an already-kept
|
|
117
|
+
// row exceeds the threshold — duplicates are filtered at recall time
|
|
118
|
+
// instead of waiting for a dream consolidation. Opt-in aggressive mode:
|
|
119
|
+
// small embedding models can collapse legitimately distinct rows, so the
|
|
120
|
+
// default keeps every recalled row.
|
|
121
|
+
searchSemanticDedup: z.boolean().default(false),
|
|
122
|
+
searchSemanticDedupThreshold: z.number().min(0.5).max(1).default(0.95),
|
|
123
|
+
|
|
95
124
|
// --- semantic: rerank layer (v0.2) --------------------------------------
|
|
96
125
|
// Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
|
|
97
126
|
// (transformers.js) at init, so a bare install must not load it. Only an
|
package/src/dream.js
CHANGED
|
@@ -3,6 +3,43 @@ import { clusterMemories, findPotentialConflicts } from "./dream/clustering.js";
|
|
|
3
3
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
4
|
export { validateDecisions, applyDecisions };
|
|
5
5
|
|
|
6
|
+
|
|
7
|
+
// Extract the first JSON array from LLM output, tolerating markdown fences,
|
|
8
|
+
// leading/trailing prose, and common wrapper noise. Returns an array or null.
|
|
9
|
+
function extractJsonArray(text) {
|
|
10
|
+
if (typeof text !== "string" || text.trim().length === 0) return null;
|
|
11
|
+
|
|
12
|
+
// 1. Strip markdown code fences (```json ... ``` or ``` ... ```).
|
|
13
|
+
let cleaned = text.replace(/```(?:json)?\s*([\s\S]*?)```/gi, "$1");
|
|
14
|
+
cleaned = cleaned.trim();
|
|
15
|
+
|
|
16
|
+
// 2. Find the first '[' and the matching last ']' that yields valid JSON.
|
|
17
|
+
const start = cleaned.indexOf("[");
|
|
18
|
+
if (start === -1) return null;
|
|
19
|
+
for (let end = cleaned.lastIndexOf("]"); end > start; end = cleaned.lastIndexOf("]", end - 1)) {
|
|
20
|
+
const candidate = cleaned.slice(start, end + 1);
|
|
21
|
+
try {
|
|
22
|
+
return JSON.parse(candidate);
|
|
23
|
+
} catch {
|
|
24
|
+
// Light repair: remove trailing commas before ] or }.
|
|
25
|
+
try {
|
|
26
|
+
const repaired = candidate.replace(/,(\s*[}\]])/g, "$1");
|
|
27
|
+
return JSON.parse(repaired);
|
|
28
|
+
} catch {
|
|
29
|
+
// keep searching backwards
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// 3. Fallback: a broader regex extraction.
|
|
35
|
+
try {
|
|
36
|
+
const match = cleaned.match(/\[[\s\S]*\]/);
|
|
37
|
+
if (match) return JSON.parse(match[0]);
|
|
38
|
+
} catch {
|
|
39
|
+
// fall through
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
6
43
|
const SUMMARY_PROMPT = `你是记忆库摘要助手。根据整理后的记忆,生成一段 150-200 字的记忆库总览,覆盖:用户偏好、活跃项目、关键决策。之后作为会话上下文注入。只输出摘要文本,不要其他内容。`;
|
|
7
44
|
|
|
8
45
|
const CONSOLIDATION_PROMPT = `你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。
|
|
@@ -579,18 +616,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
579
616
|
return finish({ ok: false, error: "llm failed", summary: false });
|
|
580
617
|
}
|
|
581
618
|
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
if (start === -1 || end <= start) {
|
|
587
|
-
logger?.warn?.("dsh-mneme dream: no json array in llm output");
|
|
588
|
-
return finish({ ok: false, error: "no json array in llm output", summary: false });
|
|
589
|
-
}
|
|
590
|
-
decisions = JSON.parse(decisionText.slice(start, end + 1));
|
|
591
|
-
} catch {
|
|
592
|
-
logger?.warn?.("dsh-mneme dream: invalid decisions json");
|
|
593
|
-
return finish({ ok: false, error: "invalid decisions json", summary: false });
|
|
619
|
+
const decisions = extractJsonArray(decisionText);
|
|
620
|
+
if (!Array.isArray(decisions)) {
|
|
621
|
+
logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0})`);
|
|
622
|
+
return finish({ ok: false, error: "no json array in llm output", summary: false });
|
|
594
623
|
}
|
|
595
624
|
const { ok, errors } = validateDecisions(decisions, snapshot, {
|
|
596
625
|
maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// Session-scoped hot memory (v0.5.0 召回率优化 1.3): a short-term buffer of
|
|
2
|
+
// the latest dialogue rounds, kept strictly apart from the long-term memory
|
|
3
|
+
// store. The injector renders it ahead of the long-term recall block so the
|
|
4
|
+
// agent sees "what we were just talking about" without those rounds ever
|
|
5
|
+
// being persisted as memories. Bounded two ways: maxRounds (count) and
|
|
6
|
+
// maxTokens (budget) — whichever evicts first.
|
|
7
|
+
|
|
8
|
+
// CJK-aware token estimate: one Chinese character ≈ 0.6 tokens (clustering
|
|
9
|
+
// behavior of mainstream tokenizers), one ASCII char ≈ 0.25.
|
|
10
|
+
export function estimateTokens(text) {
|
|
11
|
+
const s = String(text ?? "");
|
|
12
|
+
let cjk = 0;
|
|
13
|
+
for (const ch of s) if (ch >= "\u4e00" && ch <= "\u9fff") cjk++;
|
|
14
|
+
return Math.ceil(cjk * 0.6 + (s.length - cjk) * 0.25);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {{maxRounds?: number, maxTokens?: number}} opts
|
|
19
|
+
* @returns {{add(round: {query: string, response?: string}): void,
|
|
20
|
+
* getContext(): string,
|
|
21
|
+
* rounds(): Array, clear(): void}}
|
|
22
|
+
*/
|
|
23
|
+
export function createHotMemory({ maxRounds = 5, maxTokens = 2000 } = {}) {
|
|
24
|
+
// Entry defense: a non-positive or non-integer maxRounds (0, -1, 1.5, NaN,
|
|
25
|
+
// null, "2") would make the eviction while-loop unbounded — the buffer can
|
|
26
|
+
// never shrink below `buffer.length > maxRounds`, so `add` would spin forever.
|
|
27
|
+
// Fall back to the defaults so a hostile/buggy caller can never wedge the
|
|
28
|
+
// hot-memory buffer in an infinite loop.
|
|
29
|
+
maxRounds = (Number.isInteger(maxRounds) && maxRounds > 0) ? maxRounds : 5;
|
|
30
|
+
maxTokens = (Number.isFinite(maxTokens) && maxTokens > 0) ? maxTokens : 2000;
|
|
31
|
+
const buffer = [];
|
|
32
|
+
|
|
33
|
+
function totalTokens() {
|
|
34
|
+
return buffer.reduce(
|
|
35
|
+
(sum, r) => sum + estimateTokens(`Q: ${r.query}\nA: ${r.response ?? ""}`),
|
|
36
|
+
0
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
add(round) {
|
|
42
|
+
if (!round?.query) return;
|
|
43
|
+
buffer.push({ query: String(round.query), response: String(round.response ?? "") });
|
|
44
|
+
while (buffer.length > maxRounds) buffer.shift();
|
|
45
|
+
while (buffer.length > 1 && totalTokens() > maxTokens) buffer.shift();
|
|
46
|
+
},
|
|
47
|
+
getContext() {
|
|
48
|
+
return buffer.map((r) => `Q: ${r.query}\nA: ${r.response ?? ""}`).join("\n\n");
|
|
49
|
+
},
|
|
50
|
+
rounds: () => [...buffer],
|
|
51
|
+
clear() { buffer.length = 0; }
|
|
52
|
+
};
|
|
53
|
+
}
|
package/src/inject.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { createHotMemory } from "./hot-memory.js";
|
|
2
|
+
|
|
1
3
|
// Best-effort extraction of the current user's latest message text from the
|
|
2
4
|
// live session, for semantic-first injection (Bug4). The system-prompt
|
|
3
5
|
// interpolator renders synchronously, so this walks the already-materialized
|
|
@@ -25,6 +27,50 @@ function lastUserQuery(ctx) {
|
|
|
25
27
|
return "";
|
|
26
28
|
}
|
|
27
29
|
|
|
30
|
+
// Hot-memory round extraction (v0.5.0 1.3): pairs each user/message with the
|
|
31
|
+
// next assistant reply from the materialized session log. Tolerates shapes
|
|
32
|
+
// where assistant events carry a different type tag — anything whose payload
|
|
33
|
+
// has content parts and is not a user message counts as a reply. Best-effort:
|
|
34
|
+
// returns [] on any failure, and the hot block simply does not render.
|
|
35
|
+
function extractRounds(ctx, maxRounds) {
|
|
36
|
+
try {
|
|
37
|
+
const events = ctx?.agent?.session?.events;
|
|
38
|
+
if (!Array.isArray(events) || events.length === 0) return [];
|
|
39
|
+
const rounds = [];
|
|
40
|
+
let pendingQuery = null;
|
|
41
|
+
const textOf = (event) => {
|
|
42
|
+
const parts = event?.data?.content;
|
|
43
|
+
if (!Array.isArray(parts)) return "";
|
|
44
|
+
return parts
|
|
45
|
+
.map((p) => (typeof p === "string" ? p : p?.text ?? ""))
|
|
46
|
+
.filter(Boolean)
|
|
47
|
+
.join("\n")
|
|
48
|
+
.trim();
|
|
49
|
+
};
|
|
50
|
+
for (const event of events) {
|
|
51
|
+
const kind = event?.data?.source?.kind;
|
|
52
|
+
const isUser = event?.type === "user/message" && (kind === undefined || kind === "user");
|
|
53
|
+
if (isUser) {
|
|
54
|
+
if (pendingQuery) rounds.push({ query: pendingQuery, response: "" });
|
|
55
|
+
pendingQuery = textOf(event).slice(0, 500);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
// Only assistant-originated events close a round; tool/system events
|
|
59
|
+
// carrying text must not be mistaken for the model's reply.
|
|
60
|
+
const isAssistant = typeof event?.type === "string" && event.type.includes("assistant")
|
|
61
|
+
|| kind === "assistant";
|
|
62
|
+
const body = isAssistant ? textOf(event) : "";
|
|
63
|
+
if (!body || !pendingQuery) continue;
|
|
64
|
+
rounds.push({ query: pendingQuery, response: body.slice(0, 800) });
|
|
65
|
+
pendingQuery = null;
|
|
66
|
+
}
|
|
67
|
+
if (pendingQuery) rounds.push({ query: pendingQuery, response: "" });
|
|
68
|
+
return rounds.slice(-maxRounds);
|
|
69
|
+
} catch {
|
|
70
|
+
return [];
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
28
74
|
export function createInjector(ctx, service, settings, config) {
|
|
29
75
|
const maxItems = config.maxInjectedItems ?? 5;
|
|
30
76
|
const threshold = config.importanceThreshold ?? 3;
|
|
@@ -36,6 +82,35 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
36
82
|
const MAX_CONTENT = 300;
|
|
37
83
|
const MAX_BLOCK = 1500;
|
|
38
84
|
|
|
85
|
+
// Compressed injection (v0.5.0 2.1): a sleep-demoted row already carries its
|
|
86
|
+
// summary in `content` with the original parked in `_full_content` — inject
|
|
87
|
+
// the summary verbatim instead of re-truncating the (already short) text.
|
|
88
|
+
// Regular long rows keep the hard truncate.
|
|
89
|
+
function injectMemory(m, maxLength = MAX_CONTENT) {
|
|
90
|
+
if (m?._full_content) return String(m.content ?? "");
|
|
91
|
+
const text = String(m?.content ?? "");
|
|
92
|
+
return text.length <= maxLength ? text : `${text.slice(0, maxLength)}…`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Hot memory (v0.5.0 1.3): the latest rounds of THIS session, rebuilt from
|
|
96
|
+
// the materialized event log on every render — stateless, so it survives
|
|
97
|
+
// session switches and never persists anywhere.
|
|
98
|
+
const hot = createHotMemory({
|
|
99
|
+
maxRounds: config.hotMemoryRounds ?? 5,
|
|
100
|
+
maxTokens: config.hotMemoryMaxTokens ?? 2000
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
function renderHotContext(ctx) {
|
|
104
|
+
if (config.hotMemoryEnabled === false) return "";
|
|
105
|
+
const rounds = extractRounds(ctx, config.hotMemoryRounds ?? 5);
|
|
106
|
+
if (!rounds.length) return "";
|
|
107
|
+
hot.clear();
|
|
108
|
+
for (const r of rounds) hot.add(r);
|
|
109
|
+
const body = hot.getContext();
|
|
110
|
+
if (!body) return "";
|
|
111
|
+
return `[短期上下文] 最近对话(共 ${rounds.length} 轮):\n${body}`;
|
|
112
|
+
}
|
|
113
|
+
|
|
39
114
|
function render(candidates) {
|
|
40
115
|
if (!candidates.length) return "";
|
|
41
116
|
const header = "[记忆库] 来自 dsh-mneme 的跨会话记忆(用户偏好与高优先级项目/决策):";
|
|
@@ -48,8 +123,7 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
48
123
|
? "[verified] "
|
|
49
124
|
: "";
|
|
50
125
|
const title = `${m.title}(重要性 ${m.importance})`;
|
|
51
|
-
|
|
52
|
-
if (content.length > MAX_CONTENT) content = `${content.slice(0, MAX_CONTENT)}…`;
|
|
126
|
+
const content = injectMemory(m);
|
|
53
127
|
const full = `- [${m.type}] ${verified}${title}:${content}`;
|
|
54
128
|
if (budget - full.length >= 0) {
|
|
55
129
|
lines.push(full);
|
|
@@ -108,7 +182,14 @@ export function createInjector(ctx, service, settings, config) {
|
|
|
108
182
|
if (query) prefetchQueryVector(query);
|
|
109
183
|
const queryVector = queryVectorCache.get(query);
|
|
110
184
|
const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold });
|
|
111
|
-
|
|
185
|
+
// Hot memory (v0.5.0 1.3) leads the single memory block: the agent
|
|
186
|
+
// sees the short-term rounds first, then the cross-session recall —
|
|
187
|
+
// the documented injection order 1→2. Folding it here (instead of a
|
|
188
|
+
// separate context) keeps the prompt assembly stable at two blocks.
|
|
189
|
+
const hotText = renderHotContext(ctx);
|
|
190
|
+
const body = render(candidates);
|
|
191
|
+
if (!hotText) return body;
|
|
192
|
+
return body ? `${hotText}\n\n${body}` : hotText;
|
|
112
193
|
}
|
|
113
194
|
}),
|
|
114
195
|
ctx.systemPrompt.context({
|
package/src/local-embedder.js
CHANGED
|
@@ -22,7 +22,13 @@ function modelHash(model) {
|
|
|
22
22
|
|
|
23
23
|
/** Lazy default loader: dynamic import keeps module load cheap. */
|
|
24
24
|
async function defaultPipelineLoader(task, model, options) {
|
|
25
|
-
const { pipeline } = await import("@huggingface/transformers");
|
|
25
|
+
const { env, pipeline } = await import("@huggingface/transformers");
|
|
26
|
+
// issue #13: transformers.js's get_tokenizer_files() drops the caller's
|
|
27
|
+
// cache_dir when it pre-checks tokenizer_config.json metadata, so the HEAD
|
|
28
|
+
// request falls back to env.cacheDir and hits the network even when the
|
|
29
|
+
// model is fully cached locally. Mirroring the cache_dir onto env.cacheDir
|
|
30
|
+
// makes that pre-check resolve locally too — fully offline loading.
|
|
31
|
+
if (options?.cache_dir) env.cacheDir = options.cache_dir;
|
|
26
32
|
return pipeline(task, model, options);
|
|
27
33
|
}
|
|
28
34
|
|
package/src/reranker.js
CHANGED
|
@@ -21,7 +21,10 @@ function modelHash(model) {
|
|
|
21
21
|
|
|
22
22
|
/** Lazy default pipeline factory: dynamic import keeps module load cheap. */
|
|
23
23
|
async function defaultPipelineLoader(task, model, options) {
|
|
24
|
-
const { pipeline } = await import("@huggingface/transformers");
|
|
24
|
+
const { env, pipeline } = await import("@huggingface/transformers");
|
|
25
|
+
// issue #13: mirror cache_dir onto env.cacheDir so the tokenizer pre-check
|
|
26
|
+
// resolves locally too (same fix as local-embedder.js).
|
|
27
|
+
if (options?.cache_dir) env.cacheDir = options.cache_dir;
|
|
25
28
|
return pipeline(task, model, options);
|
|
26
29
|
}
|
|
27
30
|
|