@modusensus/dsh-mneme 0.4.7 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/store.js CHANGED
@@ -911,6 +911,29 @@ export function createStore(path) {
911
911
  db.prepare("UPDATE memories SET embedding = ? WHERE id = ?").run(json, id);
912
912
  }
913
913
 
914
+ /** Batch fetch stored embeddings by id (v0.5.0 search-time semantic dedup).
915
+ * Returns a Map(id → number[]); rows without a parseable embedding are
916
+ * simply absent from the map. */
917
+ function getEmbeddings(ids) {
918
+ const out = new Map();
919
+ const list = (Array.isArray(ids) ? ids : []).filter(Boolean);
920
+ for (let i = 0; i < list.length; i += 100) {
921
+ const chunk = list.slice(i, i + 100);
922
+ const rows = db.prepare(
923
+ `SELECT id, embedding FROM memories
924
+ WHERE embedding IS NOT NULL AND embedding != ''
925
+ AND id IN (${chunk.map(() => "?").join(",")})`
926
+ ).all(...chunk);
927
+ for (const row of rows) {
928
+ try {
929
+ const vec = JSON.parse(row.embedding);
930
+ if (Array.isArray(vec) && vec.length) out.set(row.id, vec);
931
+ } catch { /* corrupt row: skip */ }
932
+ }
933
+ }
934
+ return out;
935
+ }
936
+
914
937
  function embeddedCount() {
915
938
  return db.prepare(
916
939
  "SELECT count(*) AS c FROM memories WHERE embedding IS NOT NULL AND embedding != ''"
@@ -1852,6 +1875,7 @@ export function createStore(path) {
1852
1875
  all,
1853
1876
  search,
1854
1877
  setEmbedding,
1878
+ getEmbeddings,
1855
1879
  embeddedCount,
1856
1880
  needsEmbedding,
1857
1881
  searchVector,
package/package.json CHANGED
@@ -1,8 +1,16 @@
1
1
  {
2
2
  "name": "@modusensus/dsh-mneme",
3
3
  "description": "Cross-session memory plugin for DeepSeek Harness with autoDream consolidation: SQLite store, Markdown mirrors, 7 model tools, automatic injection, session summarization, user profile/rules, custom slash commands, vector (semantic) search, and a Web GUI panel",
4
- "version": "0.4.7",
4
+ "version": "0.5.1",
5
5
  "license": "MIT",
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "git+https://github.com/modusensus/dsh-mneme.git"
9
+ },
10
+ "homepage": "https://github.com/modusensus/dsh-mneme#readme",
11
+ "bugs": {
12
+ "url": "https://github.com/modusensus/dsh-mneme/issues"
13
+ },
6
14
  "type": "module",
7
15
  "main": "lib/index.js",
8
16
  "exports": {
@@ -0,0 +1,133 @@
1
+ // Recall benchmark (v0.5.0 评测体系): a self-contained harness that seeds an
2
+ // in-memory store with labelled memories, runs a standard query set through
3
+ // the real searchMemories pipeline, and reports Recall@K / MRR per case and
4
+ // in aggregate. Runs in two configurations so the BM25/third-path lift is
5
+ // visible: `legacy` (bm25 + adaptive + dedup off) vs `fused` (defaults on).
6
+ //
7
+ // Usage:
8
+ // node scripts/benchmark-recall.js # run both configurations
9
+ // node scripts/benchmark-recall.js --json # machine-readable output
10
+ // The harness exports runBenchmark()/TEST_CASES for the test suite; the CLI
11
+ // path below only executes when invoked directly.
12
+ import { createStore } from "../src/store.js";
13
+ import { createService } from "../src/service.js";
14
+ import { createVectorIndex } from "../src/vector-index.js";
15
+
16
+ // Deterministic toy embedder: bag-of-words hashed into a fixed-dimension
17
+ // vector, so cosine similarity ≈ lexical overlap. Good enough to exercise
18
+ // the vector path mechanically — semantic quality is not under test here.
19
+ const DIM = 256;
20
+ function hashVec(text) {
21
+ const v = new Array(DIM).fill(0);
22
+ const tokens = String(text ?? "").toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean);
23
+ for (const t of tokens) {
24
+ let h = 0;
25
+ for (const ch of t) h = (h * 31 + ch.codePointAt(0)) >>> 0;
26
+ v[h % DIM] += 1;
27
+ }
28
+ const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
29
+ return v.map((x) => x / norm);
30
+ }
31
+
32
+ const SEED = [
33
+ { id: "mem_user_pref", type: "preference", title: "编辑器偏好", content: "用户偏好 VS Code,深色主题,等宽字体 JetBrains Mono", importance: 4, tags: ["editor"] },
34
+ { id: "mem_user_project", type: "project", title: "dsh-mneme 插件项目", content: "用户在开发 dsh-mneme 记忆插件,TypeScript 与 cordis 框架", importance: 5, tags: ["plugin"] },
35
+ { id: "mem_rust_switch", type: "decision", title: "语言迁移决策", content: "项目编译模块从 Go 迁移到 Rust,理由是内存安全", importance: 4, tags: ["rust"] },
36
+ { id: "mem_zfs_bug", type: "project", title: "ZFS-4421 数据损坏", content: "线上池 ZFS-4421 出现 checksum 错误,根因是 HBA 固件 bug", importance: 5, tags: ["ops"] },
37
+ { id: "mem_city_thesis", type: "project", title: "湿地论文", content: "毕业论文研究城市湿地公园周边开发案例,ArcGIS 空间分析", importance: 4, tags: ["thesis"] },
38
+ { id: "mem_async_pattern", type: "decision", title: "异步并发模式", content: "async runtime 选用 tokio,任务用 spawn 管理,channel 通信", importance: 3, tags: ["rust"] },
39
+ { id: "mem_python_etl", type: "project", title: "ETL 脚本", content: "夜间 ETL 用 Python 编写,pandas 清洗,SQLite 落地", importance: 3, tags: ["etl"] },
40
+ { id: "mem_ui_style", type: "preference", title: "界面审美", content: "喜欢编辑风 brutalism 排版,低饱和度配色,衬线标题", importance: 3, tags: ["design"] }
41
+ ];
42
+
43
+ // Standard query set: each case is a query plus the ids that MUST appear in
44
+ // the top-K for the case to count as a hit. Covers the three recall paths —
45
+ // multi-term lexical (BM25's home turf), identifier lookup, and semantic.
46
+ export const TEST_CASES = [
47
+ { query: "rust 异步", expected: ["mem_async_pattern", "mem_rust_switch"], note: "scattered terms — BM25 territory" },
48
+ { query: "ZFS-4421 checksum", expected: ["mem_zfs_bug"], note: "identifier + keyword" },
49
+ { query: "插件 开发", expected: ["mem_user_project"], note: "multi-term CJK" },
50
+ { query: "论文 空间分析", expected: ["mem_city_thesis"], note: "scattered CJK terms" },
51
+ { query: "ETL 脚本", expected: ["mem_python_etl"], note: "mixed" },
52
+ { query: "深色主题", expected: ["mem_user_pref"], note: "substring match" },
53
+ { query: "channel 通信 任务", expected: ["mem_async_pattern"], note: "scattered terms" },
54
+ { query: "内存安全 语言", expected: ["mem_rust_switch"], note: "scattered terms" },
55
+ { query: "配色 审美", expected: ["mem_ui_style"], note: "scattered CJK" },
56
+ { query: "HBA 固件", expected: ["mem_zfs_bug"], note: "scattered terms" }
57
+ ];
58
+
59
+ function seedService(overrides = {}) {
60
+ const store = createStore(":memory:");
61
+ const config = {
62
+ bm25SearchEnabled: true,
63
+ adaptiveThresholdEnabled: true,
64
+ searchSemanticDedup: true,
65
+ searchSemanticDedupThreshold: 0.95,
66
+ selectiveInjectEnabled: true,
67
+ entitySearchEnabled: false,
68
+ ...overrides
69
+ };
70
+ const service = createService({ store, mirror: null, config, logger: null });
71
+ const vectorIndex = createVectorIndex({ store, logger: null });
72
+ service.setVectorIndex(vectorIndex);
73
+ service.setEmbedder({
74
+ embedSingle: async (text) => hashVec(text)
75
+ });
76
+ for (const m of SEED) {
77
+ const row = store.save({ type: m.type, title: m.title, content: m.content, tags: m.tags, importance: m.importance, source: "seed" });
78
+ store.setEmbedding(row.id, hashVec(`${m.title} ${m.content}`));
79
+ }
80
+ return service;
81
+ }
82
+
83
+ export async function runBenchmark({ topK = 5, mode = "auto" } = {}) {
84
+ const configs = [
85
+ { name: "legacy", overrides: { bm25SearchEnabled: false, adaptiveThresholdEnabled: false, searchSemanticDedup: false } },
86
+ { name: "fused", overrides: {} }
87
+ ];
88
+ const runs = [];
89
+ for (const cfg of configs) {
90
+ const service = seedService(cfg.overrides);
91
+ const rows = [];
92
+ let hits = 0;
93
+ let mrrSum = 0;
94
+ for (const tc of TEST_CASES) {
95
+ const results = await service.searchMemories(tc.query, { mode, topK, useRerank: false });
96
+ const ids = results.map((r) => r.id);
97
+ const metrics = service.computeRetrievalMetrics(ids, tc.expected);
98
+ if (metrics.recall === 1) hits++;
99
+ mrrSum += metrics.mrr;
100
+ rows.push({ query: tc.query, note: tc.note, expected: tc.expected, got: ids, ...metrics });
101
+ }
102
+ runs.push({
103
+ config: cfg.name,
104
+ recallAtK: +(hits / TEST_CASES.length).toFixed(3),
105
+ avgMrr: +(mrrSum / TEST_CASES.length).toFixed(3),
106
+ rows
107
+ });
108
+ }
109
+ return { topK, mode, runs };
110
+ }
111
+
112
+ function printReport(report) {
113
+ for (const run of report.runs) {
114
+ console.log(`\n=== ${run.config} (topK=${report.topK}, mode=${report.mode}) ===`);
115
+ for (const r of run.rows) {
116
+ const ok = r.recall === 1 ? "PASS" : "MISS";
117
+ console.log(` [${ok}] "${r.query}" (${r.note}) recall=${r.recall} mrr=${r.mrr}`);
118
+ if (r.recall < 1) console.log(` expected ⊇ ${r.expected.join(", ")} got: ${r.got.join(", ") || "—"}`);
119
+ }
120
+ console.log(` → Recall@${report.topK}: ${(run.recallAtK * 100).toFixed(1)}% avg MRR: ${run.avgMrr}`);
121
+ }
122
+ const [legacy, fused] = report.runs;
123
+ const lift = ((fused.recallAtK - legacy.recallAtK) * 100).toFixed(1);
124
+ console.log(`\n三路融合 vs 旧两路: Recall@${report.topK} ${legacy.recallAtK * 100}% → ${fused.recallAtK * 100}% (${lift >= 0 ? "+" : ""}${lift}pp)`);
125
+ }
126
+
127
+ const invokedDirectly = process.argv[1] && import.meta.url.endsWith(process.argv[1].replace(/\\/g, "/").split("/").pop() ?? "");
128
+ if (invokedDirectly) {
129
+ const asJson = process.argv.includes("--json");
130
+ const report = await runBenchmark({});
131
+ if (asJson) console.log(JSON.stringify(report, null, 2));
132
+ else printReport(report);
133
+ }
@@ -5,7 +5,12 @@
5
5
  //
6
6
  // Usage: npm run sync (also run automatically by `npm pack`/`npm publish`
7
7
  // via the prepack hook, so a published tarball always ships a fresh lib/).
8
- import { cpSync, existsSync, mkdirSync, readdirSync } from "node:fs";
8
+ //
9
+ // Note: copyFileSync (not cpSync) is used on purpose — cpSync removes the
10
+ // destination first, which fails with EPERM/unlink on Windows when the path
11
+ // is long enough to trigger the \\?\ extended-prefix (observed on publish).
12
+ // copyFileSync truncates and rewrites in place, so it survives long paths.
13
+ import { copyFileSync, existsSync, mkdirSync, readdirSync } from "node:fs";
9
14
  import { join, relative } from "node:path";
10
15
  import { fileURLToPath } from "node:url";
11
16
 
@@ -31,7 +36,7 @@ for (const file of walk(srcDir)) {
31
36
  const rel = relative(srcDir, file);
32
37
  const dest = join(libDir, rel);
33
38
  mkdirSync(join(dest, ".."), { recursive: true });
34
- cpSync(file, dest);
39
+ copyFileSync(file, dest);
35
40
  copied++;
36
41
  console.log(`synced ${rel}`);
37
42
  }
package/src/api.js CHANGED
@@ -103,7 +103,7 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
103
103
  try {
104
104
  const url = new URL(req.url, "http://localhost");
105
105
  const q = url.searchParams.get("q") ?? "";
106
- const limit = Number(url.searchParams.get("limit") ?? 20);
106
+ const limit = Number(url.searchParams.get("topK") ?? url.searchParams.get("limit") ?? 20);
107
107
  // mode selects the recall strategy (defaults to auto):
108
108
  // auto (default) keyword first, vector fills remaining slots
109
109
  // hybrid vector first, keyword fills remaining slots; scores of
@@ -306,6 +306,122 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
306
306
  }
307
307
  });
308
308
 
309
+ // --- ego graph: 1-2 hop neighborhood of one entity (graph panel P1) ---
310
+ // Read-only like list/search/semantic, so it stays open when apiToken is set.
311
+ // BFS from the root entity over entity_relations (both directions; the
312
+ // idx_relations_from/to indexes keep a 2-hop walk in the tens of ms even
313
+ // for a few thousand nodes). `distance` on each node is the hop count from
314
+ // the root so the UI can shade the frontier. The API is graph-traversal
315
+ // only — nodes carry no attr payload; hover summaries come from
316
+ // /semantic/graph/entity-attrs.
317
+ register({
318
+ kind: "exact",
319
+ path: "/api/dsh-mneme/semantic/graph/ego",
320
+ handler(req, res) {
321
+ try {
322
+ const url = new URL(req.url, "http://localhost");
323
+ const name = (url.searchParams.get("entity") ?? "").trim();
324
+ if (!name) {
325
+ sendJson(res, 400, { error: "missing-entity" });
326
+ return;
327
+ }
328
+ const root = service.findEntityByName?.(name);
329
+ if (!root) {
330
+ sendJson(res, 404, { error: "entity-not-found" });
331
+ return;
332
+ }
333
+ const depth = Math.max(1, Math.min(2, Number(url.searchParams.get("depth") ?? 1) || 1));
334
+ const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 40) || 40));
335
+
336
+ const nodes = new Map([[root.id, { ...root, distance: 0 }]]);
337
+ let frontier = [root.id];
338
+ for (let d = 1; d <= depth && nodes.size < limit; d++) {
339
+ const next = [];
340
+ for (const id of frontier) {
341
+ for (const rel of service.getRelations?.(id) ?? []) {
342
+ const other = rel.from_entity === id ? rel.to_entity : rel.from_entity;
343
+ if (nodes.has(other) || nodes.size >= limit) continue;
344
+ const entity = service.findEntityById?.(other);
345
+ if (!entity) continue;
346
+ nodes.set(other, { ...entity, distance: d });
347
+ next.push(other);
348
+ }
349
+ }
350
+ frontier = next;
351
+ }
352
+
353
+ // Collect every relation whose endpoints both survived the limit cut;
354
+ // each edge is visited twice (once per endpoint) so dedupe by id.
355
+ const edgeMap = new Map();
356
+ for (const id of nodes.keys()) {
357
+ for (const rel of service.getRelations?.(id) ?? []) {
358
+ if (nodes.has(rel.from_entity) && nodes.has(rel.to_entity)) {
359
+ edgeMap.set(rel.id, rel);
360
+ }
361
+ }
362
+ }
363
+
364
+ sendJson(res, 200, {
365
+ root: { id: root.id, name: root.name, type: root.type ?? null, mention_count: root.mention_count ?? 1 },
366
+ nodes: [...nodes.values()].map((n) => ({
367
+ id: n.id,
368
+ name: n.name,
369
+ type: n.type ?? null,
370
+ mention_count: n.mention_count ?? 1,
371
+ distance: n.distance
372
+ })),
373
+ edges: [...edgeMap.values()].map((e) => ({
374
+ id: e.id,
375
+ from: e.from_entity,
376
+ to: e.to_entity,
377
+ relation_type: e.relation_type,
378
+ memory_id: e.memory_id ?? null,
379
+ created_at: e.created_at
380
+ }))
381
+ });
382
+ } catch {
383
+ sendJson(res, 500, { error: "internal" });
384
+ }
385
+ }
386
+ });
387
+
388
+ // --- entity attrs: current valid attrs for one entity (graph hover panel) ---
389
+ // Read-only; mirrors getCurrentAttrs (valid_until IS NULL). Also used as the
390
+ // graph panel's fallback list when the ego graph is too sparse to draw.
391
+ register({
392
+ kind: "exact",
393
+ path: "/api/dsh-mneme/semantic/graph/entity-attrs",
394
+ handler(req, res) {
395
+ try {
396
+ const url = new URL(req.url, "http://localhost");
397
+ const name = (url.searchParams.get("entity") ?? "").trim();
398
+ if (!name) {
399
+ sendJson(res, 400, { error: "missing-entity" });
400
+ return;
401
+ }
402
+ const entity = service.findEntityByName?.(name);
403
+ if (!entity) {
404
+ sendJson(res, 404, { error: "entity-not-found" });
405
+ return;
406
+ }
407
+ const attrs = service.getCurrentAttrs?.(entity.id) ?? [];
408
+ sendJson(res, 200, {
409
+ entity: { id: entity.id, name: entity.name, type: entity.type ?? null, mention_count: entity.mention_count ?? 1 },
410
+ attrs: Array.isArray(attrs)
411
+ ? attrs.map((a) => ({
412
+ key: a.attr_key,
413
+ value: a.attr_value,
414
+ confidence: a.confidence ?? null,
415
+ valid_from: a.valid_from ?? null
416
+ }))
417
+ : []
418
+ });
419
+ } catch {
420
+ sendJson(res, 500, { error: "internal" });
421
+ }
422
+ }
423
+ });
424
+
309
425
  // --- health: mirror sync state (F-NEW-03 / v0.3.6) ---
310
426
  // Auth-gated; only returns a sanitized error code (never raw last_error which
311
427
  // may leak paths/token-like strings/internal hosts). On state read failure it
package/src/config.js CHANGED
@@ -17,7 +17,7 @@ export const Config = z.object({
17
17
  dreamDelayMs: z.natural().min(0).max(60000).default(2000),
18
18
  dreamProvider: z.string(),
19
19
  dreamModel: z.string(),
20
- dreamMaxTokens: z.natural().min(256).max(131072).default(4096),
20
+ dreamMaxTokens: z.natural().min(256).max(131072).default(8192),
21
21
  // Pass-through reasoning effort for dream's LLM calls. 'none' (default)
22
22
  // omits the field so the provider's own default applies; low/medium/high
23
23
  // are forwarded verbatim. Useful to cap reasoning spend on thinking-type
@@ -92,6 +92,35 @@ export const Config = z.object({
92
92
  // rule-based pick to fill/dedupe. Empty query / no vector → legacy behavior.
93
93
  hybridInject: z.boolean().default(true),
94
94
 
95
+ // --- recall optimization (v0.5.0) ----------------------------------------
96
+ // BM25 third recall path beside vector + LIKE keyword (1.1): per-token IDF
97
+ // scoring recalls rows whose query terms are scattered — identifiers, code
98
+ // fragments, mixed CJK/ASCII — where substring LIKE cannot match.
99
+ bm25SearchEnabled: z.boolean().default(true),
100
+ // Query-aware vector cutoff (1.2) replacing the fixed 0.65: entity:/attr:
101
+ // prefixes loosen to 0.5, short queries tighten to 0.7, long queries loosen
102
+ // to 0.6, and a decisive top-1/top-5 score gap loosens to 0.5 so the tail
103
+ // still reaches the reranker. Off = legacy fixed threshold behavior.
104
+ adaptiveThresholdEnabled: z.boolean().default(true),
105
+ // Session-scoped hot memory (1.3): the latest N dialogue rounds rendered
106
+ // ahead of the long-term recall block — short-term context that never
107
+ // enters the memory store.
108
+ hotMemoryEnabled: z.boolean().default(true),
109
+ hotMemoryRounds: z.natural().min(1).max(50).default(5),
110
+ hotMemoryMaxTokens: z.natural().min(200).max(32000).default(2000),
111
+ // Topic-ranked injection (2.2): when a query vector is available the whole
112
+ // injection candidate list is re-ordered by similarity to the current
113
+ // query instead of keeping the rule-based order.
114
+ selectiveInjectEnabled: z.boolean().default(true),
115
+ // Search-time semantic dedup (2.3): greedy pass over the merged candidate
116
+ // list dropping rows whose embedding cosine-similarity to an already-kept
117
+ // row exceeds the threshold — duplicates are filtered at recall time
118
+ // instead of waiting for a dream consolidation. Opt-in aggressive mode:
119
+ // small embedding models can collapse legitimately distinct rows, so the
120
+ // default keeps every recalled row.
121
+ searchSemanticDedup: z.boolean().default(false),
122
+ searchSemanticDedupThreshold: z.number().min(0.5).max(1).default(0.95),
123
+
95
124
  // --- semantic: rerank layer (v0.2) --------------------------------------
96
125
  // Opt-in by default (item ⑥): the local cross-encoder pulls in onnxruntime
97
126
  // (transformers.js) at init, so a bare install must not load it. Only an
package/src/dream.js CHANGED
@@ -3,6 +3,43 @@ import { clusterMemories, findPotentialConflicts } from "./dream/clustering.js";
3
3
  import { createHash, randomUUID } from "node:crypto";
4
4
  export { validateDecisions, applyDecisions };
5
5
 
6
+
7
+ // Extract the first JSON array from LLM output, tolerating markdown fences,
8
+ // leading/trailing prose, and common wrapper noise. Returns an array or null.
9
+ function extractJsonArray(text) {
10
+ if (typeof text !== "string" || text.trim().length === 0) return null;
11
+
12
+ // 1. Strip markdown code fences (```json ... ``` or ``` ... ```).
13
+ let cleaned = text.replace(/```(?:json)?\s*([\s\S]*?)```/gi, "$1");
14
+ cleaned = cleaned.trim();
15
+
16
+ // 2. Find the first '[' and the matching last ']' that yields valid JSON.
17
+ const start = cleaned.indexOf("[");
18
+ if (start === -1) return null;
19
+ for (let end = cleaned.lastIndexOf("]"); end > start; end = cleaned.lastIndexOf("]", end - 1)) {
20
+ const candidate = cleaned.slice(start, end + 1);
21
+ try {
22
+ return JSON.parse(candidate);
23
+ } catch {
24
+ // Light repair: remove trailing commas before ] or }.
25
+ try {
26
+ const repaired = candidate.replace(/,(\s*[}\]])/g, "$1");
27
+ return JSON.parse(repaired);
28
+ } catch {
29
+ // keep searching backwards
30
+ }
31
+ }
32
+ }
33
+
34
+ // 3. Fallback: a broader regex extraction.
35
+ try {
36
+ const match = cleaned.match(/\[[\s\S]*\]/);
37
+ if (match) return JSON.parse(match[0]);
38
+ } catch {
39
+ // fall through
40
+ }
41
+ return null;
42
+ }
6
43
  const SUMMARY_PROMPT = `你是记忆库摘要助手。根据整理后的记忆,生成一段 150-200 字的记忆库总览,覆盖:用户偏好、活跃项目、关键决策。之后作为会话上下文注入。只输出摘要文本,不要其他内容。`;
7
44
 
8
45
  const CONSOLIDATION_PROMPT = `你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。
@@ -579,18 +616,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
579
616
  return finish({ ok: false, error: "llm failed", summary: false });
580
617
  }
581
618
 
582
- let decisions;
583
- try {
584
- const start = decisionText.indexOf("[");
585
- const end = decisionText.lastIndexOf("]");
586
- if (start === -1 || end <= start) {
587
- logger?.warn?.("dsh-mneme dream: no json array in llm output");
588
- return finish({ ok: false, error: "no json array in llm output", summary: false });
589
- }
590
- decisions = JSON.parse(decisionText.slice(start, end + 1));
591
- } catch {
592
- logger?.warn?.("dsh-mneme dream: invalid decisions json");
593
- return finish({ ok: false, error: "invalid decisions json", summary: false });
619
+ const decisions = extractJsonArray(decisionText);
620
+ if (!Array.isArray(decisions)) {
621
+ logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0})`);
622
+ return finish({ ok: false, error: "no json array in llm output", summary: false });
594
623
  }
595
624
  const { ok, errors } = validateDecisions(decisions, snapshot, {
596
625
  maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
@@ -0,0 +1,53 @@
1
+ // Session-scoped hot memory (v0.5.0 召回率优化 1.3): a short-term buffer of
2
+ // the latest dialogue rounds, kept strictly apart from the long-term memory
3
+ // store. The injector renders it ahead of the long-term recall block so the
4
+ // agent sees "what we were just talking about" without those rounds ever
5
+ // being persisted as memories. Bounded two ways: maxRounds (count) and
6
+ // maxTokens (budget) — whichever evicts first.
7
+
8
+ // CJK-aware token estimate: one Chinese character ≈ 0.6 tokens (clustering
9
+ // behavior of mainstream tokenizers), one ASCII char ≈ 0.25.
10
+ export function estimateTokens(text) {
11
+ const s = String(text ?? "");
12
+ let cjk = 0;
13
+ for (const ch of s) if (ch >= "\u4e00" && ch <= "\u9fff") cjk++;
14
+ return Math.ceil(cjk * 0.6 + (s.length - cjk) * 0.25);
15
+ }
16
+
17
+ /**
18
+ * @param {{maxRounds?: number, maxTokens?: number}} opts
19
+ * @returns {{add(round: {query: string, response?: string}): void,
20
+ * getContext(): string,
21
+ * rounds(): Array, clear(): void}}
22
+ */
23
+ export function createHotMemory({ maxRounds = 5, maxTokens = 2000 } = {}) {
24
+ // Entry defense: a non-positive or non-integer maxRounds (0, -1, 1.5, NaN,
25
+ // null, "2") would make the eviction while-loop unbounded — the buffer can
26
+ // never shrink below `buffer.length > maxRounds`, so `add` would spin forever.
27
+ // Fall back to the defaults so a hostile/buggy caller can never wedge the
28
+ // hot-memory buffer in an infinite loop.
29
+ maxRounds = (Number.isInteger(maxRounds) && maxRounds > 0) ? maxRounds : 5;
30
+ maxTokens = (Number.isFinite(maxTokens) && maxTokens > 0) ? maxTokens : 2000;
31
+ const buffer = [];
32
+
33
+ function totalTokens() {
34
+ return buffer.reduce(
35
+ (sum, r) => sum + estimateTokens(`Q: ${r.query}\nA: ${r.response ?? ""}`),
36
+ 0
37
+ );
38
+ }
39
+
40
+ return {
41
+ add(round) {
42
+ if (!round?.query) return;
43
+ buffer.push({ query: String(round.query), response: String(round.response ?? "") });
44
+ while (buffer.length > maxRounds) buffer.shift();
45
+ while (buffer.length > 1 && totalTokens() > maxTokens) buffer.shift();
46
+ },
47
+ getContext() {
48
+ return buffer.map((r) => `Q: ${r.query}\nA: ${r.response ?? ""}`).join("\n\n");
49
+ },
50
+ rounds: () => [...buffer],
51
+ clear() { buffer.length = 0; }
52
+ };
53
+ }
package/src/inject.js CHANGED
@@ -1,3 +1,5 @@
1
+ import { createHotMemory } from "./hot-memory.js";
2
+
1
3
  // Best-effort extraction of the current user's latest message text from the
2
4
  // live session, for semantic-first injection (Bug4). The system-prompt
3
5
  // interpolator renders synchronously, so this walks the already-materialized
@@ -25,6 +27,50 @@ function lastUserQuery(ctx) {
25
27
  return "";
26
28
  }
27
29
 
30
+ // Hot-memory round extraction (v0.5.0 1.3): pairs each user/message with the
31
+ // next assistant reply from the materialized session log. Tolerates shapes
32
+ // where assistant events carry a different type tag — anything whose payload
33
+ // has content parts and is not a user message counts as a reply. Best-effort:
34
+ // returns [] on any failure, and the hot block simply does not render.
35
+ function extractRounds(ctx, maxRounds) {
36
+ try {
37
+ const events = ctx?.agent?.session?.events;
38
+ if (!Array.isArray(events) || events.length === 0) return [];
39
+ const rounds = [];
40
+ let pendingQuery = null;
41
+ const textOf = (event) => {
42
+ const parts = event?.data?.content;
43
+ if (!Array.isArray(parts)) return "";
44
+ return parts
45
+ .map((p) => (typeof p === "string" ? p : p?.text ?? ""))
46
+ .filter(Boolean)
47
+ .join("\n")
48
+ .trim();
49
+ };
50
+ for (const event of events) {
51
+ const kind = event?.data?.source?.kind;
52
+ const isUser = event?.type === "user/message" && (kind === undefined || kind === "user");
53
+ if (isUser) {
54
+ if (pendingQuery) rounds.push({ query: pendingQuery, response: "" });
55
+ pendingQuery = textOf(event).slice(0, 500);
56
+ continue;
57
+ }
58
+ // Only assistant-originated events close a round; tool/system events
59
+ // carrying text must not be mistaken for the model's reply.
60
+ const isAssistant = typeof event?.type === "string" && event.type.includes("assistant")
61
+ || kind === "assistant";
62
+ const body = isAssistant ? textOf(event) : "";
63
+ if (!body || !pendingQuery) continue;
64
+ rounds.push({ query: pendingQuery, response: body.slice(0, 800) });
65
+ pendingQuery = null;
66
+ }
67
+ if (pendingQuery) rounds.push({ query: pendingQuery, response: "" });
68
+ return rounds.slice(-maxRounds);
69
+ } catch {
70
+ return [];
71
+ }
72
+ }
73
+
28
74
  export function createInjector(ctx, service, settings, config) {
29
75
  const maxItems = config.maxInjectedItems ?? 5;
30
76
  const threshold = config.importanceThreshold ?? 3;
@@ -36,6 +82,35 @@ export function createInjector(ctx, service, settings, config) {
36
82
  const MAX_CONTENT = 300;
37
83
  const MAX_BLOCK = 1500;
38
84
 
85
+ // Compressed injection (v0.5.0 2.1): a sleep-demoted row already carries its
86
+ // summary in `content` with the original parked in `_full_content` — inject
87
+ // the summary verbatim instead of re-truncating the (already short) text.
88
+ // Regular long rows keep the hard truncate.
89
+ function injectMemory(m, maxLength = MAX_CONTENT) {
90
+ if (m?._full_content) return String(m.content ?? "");
91
+ const text = String(m?.content ?? "");
92
+ return text.length <= maxLength ? text : `${text.slice(0, maxLength)}…`;
93
+ }
94
+
95
+ // Hot memory (v0.5.0 1.3): the latest rounds of THIS session, rebuilt from
96
+ // the materialized event log on every render — stateless, so it survives
97
+ // session switches and never persists anywhere.
98
+ const hot = createHotMemory({
99
+ maxRounds: config.hotMemoryRounds ?? 5,
100
+ maxTokens: config.hotMemoryMaxTokens ?? 2000
101
+ });
102
+
103
+ function renderHotContext(ctx) {
104
+ if (config.hotMemoryEnabled === false) return "";
105
+ const rounds = extractRounds(ctx, config.hotMemoryRounds ?? 5);
106
+ if (!rounds.length) return "";
107
+ hot.clear();
108
+ for (const r of rounds) hot.add(r);
109
+ const body = hot.getContext();
110
+ if (!body) return "";
111
+ return `[短期上下文] 最近对话(共 ${rounds.length} 轮):\n${body}`;
112
+ }
113
+
39
114
  function render(candidates) {
40
115
  if (!candidates.length) return "";
41
116
  const header = "[记忆库] 来自 dsh-mneme 的跨会话记忆(用户偏好与高优先级项目/决策):";
@@ -48,8 +123,7 @@ export function createInjector(ctx, service, settings, config) {
48
123
  ? "[verified] "
49
124
  : "";
50
125
  const title = `${m.title}(重要性 ${m.importance})`;
51
- let content = String(m.content ?? "");
52
- if (content.length > MAX_CONTENT) content = `${content.slice(0, MAX_CONTENT)}…`;
126
+ const content = injectMemory(m);
53
127
  const full = `- [${m.type}] ${verified}${title}:${content}`;
54
128
  if (budget - full.length >= 0) {
55
129
  lines.push(full);
@@ -108,7 +182,14 @@ export function createInjector(ctx, service, settings, config) {
108
182
  if (query) prefetchQueryVector(query);
109
183
  const queryVector = queryVectorCache.get(query);
110
184
  const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold });
111
- return render(candidates);
185
+ // Hot memory (v0.5.0 1.3) leads the single memory block: the agent
186
+ // sees the short-term rounds first, then the cross-session recall —
187
+ // the documented injection order 1→2. Folding it here (instead of a
188
+ // separate context) keeps the prompt assembly stable at two blocks.
189
+ const hotText = renderHotContext(ctx);
190
+ const body = render(candidates);
191
+ if (!hotText) return body;
192
+ return body ? `${hotText}\n\n${body}` : hotText;
112
193
  }
113
194
  }),
114
195
  ctx.systemPrompt.context({
@@ -22,7 +22,13 @@ function modelHash(model) {
22
22
 
23
23
  /** Lazy default loader: dynamic import keeps module load cheap. */
24
24
  async function defaultPipelineLoader(task, model, options) {
25
- const { pipeline } = await import("@huggingface/transformers");
25
+ const { env, pipeline } = await import("@huggingface/transformers");
26
+ // issue #13: transformers.js's get_tokenizer_files() drops the caller's
27
+ // cache_dir when it pre-checks tokenizer_config.json metadata, so the HEAD
28
+ // request falls back to env.cacheDir and hits the network even when the
29
+ // model is fully cached locally. Mirroring the cache_dir onto env.cacheDir
30
+ // makes that pre-check resolve locally too — fully offline loading.
31
+ if (options?.cache_dir) env.cacheDir = options.cache_dir;
26
32
  return pipeline(task, model, options);
27
33
  }
28
34
 
package/src/reranker.js CHANGED
@@ -21,7 +21,10 @@ function modelHash(model) {
21
21
 
22
22
  /** Lazy default pipeline factory: dynamic import keeps module load cheap. */
23
23
  async function defaultPipelineLoader(task, model, options) {
24
- const { pipeline } = await import("@huggingface/transformers");
24
+ const { env, pipeline } = await import("@huggingface/transformers");
25
+ // issue #13: mirror cache_dir onto env.cacheDir so the tokenizer pre-check
26
+ // resolves locally too (same fix as local-embedder.js).
27
+ if (options?.cache_dir) env.cacheDir = options.cache_dir;
25
28
  return pipeline(task, model, options);
26
29
  }
27
30