@modusensus/dsh-mneme 0.6.7 → 0.6.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/.github/workflows/test.yml +32 -0
  2. package/.release-notes-v0.6.9.md +13 -0
  3. package/CHANGELOG.md +89 -0
  4. package/LICENSE +21 -21
  5. package/README.md +219 -463
  6. package/SECURITY.md +544 -0
  7. package/docs/devlog/2026-08-14-dsh-mneme-dev-log.md +247 -0
  8. package/docs/devlog/2026-08-15-dsh-mneme-audit-stress-dev-log.md +145 -0
  9. package/docs/devlog/2026-08-15-dsh-mneme-pipeline-dev-log.md +56 -0
  10. package/docs/devlog/2026-08-15-dsh-mneme-reflection-dev-log.md +77 -0
  11. package/docs/devlog/2026-08-15-dsh-mneme-review-fixes-dev-log.md +64 -0
  12. package/docs/devlog/2026-08-15-dsh-mneme-semantic-dev-log.md +90 -0
  13. package/dsh-mneme/CHANGELOG.md +248 -0
  14. package/dsh-mneme/LICENSE +21 -0
  15. package/dsh-mneme/README.md +465 -0
  16. package/{cordis.patch.yml → dsh-mneme/cordis.patch.yml} +15 -15
  17. package/dsh-mneme/docs/AGENT_MEMORY_RESEARCH.md +183 -0
  18. package/dsh-mneme/docs/ENTITIES.md +245 -0
  19. package/dsh-mneme/docs/LOCAL_MODEL.md +141 -0
  20. package/dsh-mneme/docs/MIGRATION.md +127 -0
  21. package/dsh-mneme/docs/SEMANTIC.md +256 -0
  22. package/dsh-mneme/docs/SLEEP.md +163 -0
  23. package/{lib → dsh-mneme/lib}/api.js +783 -783
  24. package/{lib → dsh-mneme/lib}/client.js +1757 -1757
  25. package/{src → dsh-mneme/lib}/commands.js +64 -64
  26. package/{lib → dsh-mneme/lib}/config.js +298 -288
  27. package/{lib → dsh-mneme/lib}/dream/clustering.js +118 -118
  28. package/{src → dsh-mneme/lib}/dream/decisions.js +488 -439
  29. package/{lib → dsh-mneme/lib}/dream/sleep.js +561 -554
  30. package/{src → dsh-mneme/lib}/dream/tag-extractor.js +156 -156
  31. package/{lib → dsh-mneme/lib}/dream.js +958 -929
  32. package/{src → dsh-mneme/lib}/embedding.js +154 -154
  33. package/{src → dsh-mneme/lib}/entities/extractor.js +242 -242
  34. package/{src → dsh-mneme/lib}/hot-memory.js +53 -53
  35. package/{lib → dsh-mneme/lib}/index.js +361 -361
  36. package/{src → dsh-mneme/lib}/inject.js +208 -208
  37. package/{lib → dsh-mneme/lib}/local-embedder.js +282 -282
  38. package/{lib → dsh-mneme/lib}/mirror.js +170 -170
  39. package/{lib → dsh-mneme/lib}/parser/tag.js +59 -59
  40. package/{lib → dsh-mneme/lib}/parser/wiki-link.js +38 -38
  41. package/{src → dsh-mneme/lib}/quality-filter.js +123 -123
  42. package/{lib → dsh-mneme/lib}/reranker.js +218 -218
  43. package/{lib → dsh-mneme/lib}/search/adaptive.js +22 -22
  44. package/{lib → dsh-mneme/lib}/search/bm25.js +96 -96
  45. package/{src → dsh-mneme/lib}/search/tag-boost.js +61 -61
  46. package/{src → dsh-mneme/lib}/service.js +1726 -1726
  47. package/{lib → dsh-mneme/lib}/settings.js +172 -172
  48. package/{lib → dsh-mneme/lib}/store.js +2238 -2238
  49. package/{lib → dsh-mneme/lib}/summarize.js +236 -236
  50. package/{lib → dsh-mneme/lib}/tools.js +290 -290
  51. package/{lib → dsh-mneme/lib}/vector-index.js +116 -116
  52. package/dsh-mneme/package-lock.json +1936 -0
  53. package/dsh-mneme/package.json +80 -0
  54. package/{scripts → dsh-mneme/scripts}/benchmark-embed.js +201 -201
  55. package/{scripts → dsh-mneme/scripts}/benchmark-recall.js +133 -133
  56. package/{scripts → dsh-mneme/scripts}/benchmark-rerank.js +166 -166
  57. package/{scripts → dsh-mneme/scripts}/e2e-dsh.js +218 -218
  58. package/{scripts → dsh-mneme/scripts}/stress-dsh.js +255 -255
  59. package/{scripts → dsh-mneme/scripts}/sync-lib.js +52 -52
  60. package/{src → dsh-mneme/src}/api.js +783 -783
  61. package/{lib → dsh-mneme/src}/commands.js +64 -64
  62. package/{src → dsh-mneme/src}/config.js +298 -288
  63. package/{src → dsh-mneme/src}/dream/clustering.js +118 -118
  64. package/{lib → dsh-mneme/src}/dream/decisions.js +488 -439
  65. package/{src → dsh-mneme/src}/dream/sleep.js +561 -554
  66. package/{lib → dsh-mneme/src}/dream/tag-extractor.js +156 -156
  67. package/{src → dsh-mneme/src}/dream.js +958 -929
  68. package/{lib → dsh-mneme/src}/embedding.js +154 -154
  69. package/{lib → dsh-mneme/src}/entities/extractor.js +242 -242
  70. package/{lib → dsh-mneme/src}/hot-memory.js +53 -53
  71. package/{src → dsh-mneme/src}/index.js +361 -361
  72. package/{lib → dsh-mneme/src}/inject.js +208 -208
  73. package/{src → dsh-mneme/src}/local-embedder.js +282 -282
  74. package/{src → dsh-mneme/src}/mirror.js +170 -170
  75. package/{src → dsh-mneme/src}/parser/tag.js +59 -59
  76. package/{src → dsh-mneme/src}/parser/wiki-link.js +38 -38
  77. package/{lib → dsh-mneme/src}/quality-filter.js +123 -123
  78. package/{src → dsh-mneme/src}/reranker.js +218 -218
  79. package/{src → dsh-mneme/src}/search/adaptive.js +22 -22
  80. package/{src → dsh-mneme/src}/search/bm25.js +96 -96
  81. package/{lib → dsh-mneme/src}/search/tag-boost.js +61 -61
  82. package/{lib → dsh-mneme/src}/service.js +1726 -1726
  83. package/{src → dsh-mneme/src}/settings.js +172 -172
  84. package/{src → dsh-mneme/src}/store.js +2238 -2238
  85. package/{src → dsh-mneme/src}/summarize.js +236 -236
  86. package/{src → dsh-mneme/src}/tools.js +290 -290
  87. package/{src → dsh-mneme/src}/vector-index.js +116 -116
  88. package/{test → dsh-mneme/test}/api.test.js +594 -594
  89. package/{test → dsh-mneme/test}/audit.test.js +448 -448
  90. package/{test → dsh-mneme/test}/benchmark.test.js +35 -35
  91. package/{test → dsh-mneme/test}/boundary-v0625.test.js +82 -82
  92. package/{test → dsh-mneme/test}/client.test.js +368 -368
  93. package/{test → dsh-mneme/test}/clustering.test.js +100 -100
  94. package/{test → dsh-mneme/test}/commands.test.js +69 -69
  95. package/{test → dsh-mneme/test}/config.test.js +50 -50
  96. package/{test → dsh-mneme/test}/conflict-freeze.test.js +290 -290
  97. package/{test → dsh-mneme/test}/directory.test.js +134 -134
  98. package/{test → dsh-mneme/test}/dream.test.js +1060 -901
  99. package/{test → dsh-mneme/test}/entities.test.js +522 -522
  100. package/{test → dsh-mneme/test}/epistemic.test.js +298 -298
  101. package/{test → dsh-mneme/test}/fnew-0112.test.js +311 -311
  102. package/{test → dsh-mneme/test}/fnew-03.test.js +422 -422
  103. package/{test → dsh-mneme/test}/graph-api.test.js +175 -175
  104. package/{test → dsh-mneme/test}/helpers/dream-mock.js +82 -82
  105. package/{test → dsh-mneme/test}/hot-memory.test.js +174 -174
  106. package/{test → dsh-mneme/test}/inject.test.js +103 -103
  107. package/{test → dsh-mneme/test}/llm-audit.test.js +279 -279
  108. package/{test → dsh-mneme/test}/local-embedder.test.js +227 -227
  109. package/{test → dsh-mneme/test}/mirror-dirty.test.js +424 -424
  110. package/{test → dsh-mneme/test}/mirror-edit-digest.test.js +187 -187
  111. package/{test → dsh-mneme/test}/mirror-generation.test.js +499 -499
  112. package/{test → dsh-mneme/test}/mirror.test.js +249 -249
  113. package/{test → dsh-mneme/test}/normalize-decisions.test.js +120 -120
  114. package/{test → dsh-mneme/test}/peer-blockers.test.js +190 -190
  115. package/{test → dsh-mneme/test}/policy-epoch.test.js +259 -259
  116. package/{test → dsh-mneme/test}/provenance.test.js +103 -103
  117. package/{test → dsh-mneme/test}/quality-filter.test.js +118 -118
  118. package/{test → dsh-mneme/test}/reasoning-effort.test.js +199 -199
  119. package/{test → dsh-mneme/test}/recall-evals.test.js +235 -235
  120. package/{test → dsh-mneme/test}/recall-layer.test.js +315 -315
  121. package/{test → dsh-mneme/test}/receipt-chain.test.js +451 -451
  122. package/{test → dsh-mneme/test}/reflection.test.js +226 -226
  123. package/{test → dsh-mneme/test}/reranker.test.js +240 -240
  124. package/{test → dsh-mneme/test}/search-fusion.test.js +90 -90
  125. package/{test → dsh-mneme/test}/semantic.test.js +124 -124
  126. package/{test → dsh-mneme/test}/service-search.test.js +199 -199
  127. package/{test → dsh-mneme/test}/service.test.js +435 -435
  128. package/{test → dsh-mneme/test}/settings.test.js +118 -118
  129. package/{test → dsh-mneme/test}/sleep.test.js +365 -365
  130. package/{test → dsh-mneme/test}/store.test.js +436 -436
  131. package/{test → dsh-mneme/test}/stress.test.js +209 -209
  132. package/{test → dsh-mneme/test}/summarize.test.js +191 -191
  133. package/{test → dsh-mneme/test}/tag-boost.test.js +125 -125
  134. package/{test → dsh-mneme/test}/tag.test.js +312 -312
  135. package/{test → dsh-mneme/test}/tools.test.js +285 -285
  136. package/{test → dsh-mneme/test}/vector-index.test.js +221 -221
  137. package/{test → dsh-mneme/test}/wiki-link.test.js +332 -332
  138. package/package.json +18 -40
  139. package//346/250/252/345/271/205.png +0 -0
@@ -1,554 +1,561 @@
1
- // System-level sleep (v0.4.0): an idle-triggered, LLM-assisted deep pass over
2
- // the memory store. Four independent, fail-safe phases:
3
- // 1. conflict resolution — high-similarity same-type pairs are either parked
4
- // for review (freeze mode) or adjudicated by the LLM (winner kept / loser
5
- // archived), reusing the dream conflict machinery. Strictness-graded.
6
- // 2. archival demotion — memories unreferenced past sleepArchiveDays shrink
7
- // to a one-line summary with the full body moved to _full_content; past
8
- // sleepCompressDays they are archived outright.
9
- // 3. pattern discovery — the LLM scans the most recent memories and mints
10
- // type=pattern entries carrying evidence id references.
11
- // 4. relation completion — orphan entities (zero relations) get implied
12
- // relations completed from memory co-occurrence.
13
- // Each phase is wrapped so one failure never aborts the others, and a missing
14
- // LLM route / semantic embedder only skips the phases that need it. A run is
15
- // abortable via an AbortController signal (user activity) — phases check the
16
- // signal between batches so a running cycle yields promptly.
17
- import { randomUUID, createHash } from "node:crypto";
18
- import { validateDecisions, applyDecisions } from "./decisions.js";
19
- import { findPotentialConflicts } from "./clustering.js";
20
- import { buildReceipt } from "../dream.js";
21
-
22
- const SUMMARY_MAX = 120;
23
- // Conflict similarity threshold per strictness level (v0.4.0):
24
- // gentle only high-confidence pairs (0.92) — first-time users
25
- // normal standard dream-level (0.85) — default
26
- // aggressive low-confidence pairs too (0.75) — bloated stores
27
- const CONFLICT_THRESHOLDS = { gentle: 0.92, normal: 0.85, aggressive: 0.75 };
28
-
29
- const CONFLICT_PROMPT = `你是记忆库冲突仲裁助手。下面是检测到的高相似度记忆对,可能内容矛盾或重复。
30
- 对每一对输出一个 decision 对象:
31
- - 两条确实矛盾/重复 → { "action": "conflict", "winner": <保留的id>, "loser": <归档的id>, "reason": "理由" }
32
- - 两条只是主题相近、并无矛盾 → { "action": "keep", "ids": [<两个id>] }
33
- 规则:
34
- - winner 应为信息更完整、更新或更可信的一条
35
- - 只使用提供的 id,不要编造
36
- - 每对必须输出一个 decision
37
- - 只输出 JSON 数组,不要其他文字`;
38
-
39
- const PATTERN_PROMPT = `你是记忆库模式发现助手。下面是最近的记忆条目(id、类型、标题、内容)。
40
- 请发现跨条目的稳定模式:用户偏好的规律、反复出现的主题、可复用的工作流或项目规律。
41
- 对每个模式输出一个 create decision:
42
- { "action": "create", "type": "pattern", "title": "模式一句话标题", "content": "模式详细描述(2-4句)", "importance": 1-5, "evidence": ["支持该模式的记忆id"] }
43
- 规则:
44
- - 只输出有据可依的模式,宁缺毋滥
45
- - evidence 必须是列表中真实存在的 id
46
- - 最多输出 N 个模式
47
- - 只输出 JSON 数组,不要其他文字`;
48
-
49
- function parseJsonArray(text) {
50
- if (typeof text !== "string") return undefined;
51
- const start = text.indexOf("[");
52
- const end = text.lastIndexOf("]");
53
- if (start === -1 || end <= start) return undefined;
54
- try {
55
- const parsed = JSON.parse(text.slice(start, end + 1));
56
- return Array.isArray(parsed) ? parsed : undefined;
57
- } catch {
58
- return undefined;
59
- }
60
- }
61
-
62
- /** Same stream consumption contract as dream.js. */
63
- async function streamText(ctx, options) {
64
- if (!ctx?.llm?.stream) return undefined;
65
- let text = "";
66
- for await (const chunk of ctx.llm.stream(options)) {
67
- if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
68
- if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
69
- return undefined;
70
- }
71
- }
72
- return text;
73
- }
74
-
75
- /** LLM route: agent default model first, then sleepProvider/Model, then the
76
- * dream route as a shared fallback. Sleep can pin a cheaper model for its
77
- * bulk passes without disturbing the dream route. */
78
- function resolveSleepRoute(ctx, config, logger) {
79
- try {
80
- const sel = ctx?.agentDefaultModel?.currentSelection?.();
81
- if (sel?.provider && sel?.model) return { provider: sel.provider, model: sel.model };
82
- } catch { /* fall through to config route */ }
83
- if (config.sleepProvider && config.sleepModel) return { provider: config.sleepProvider, model: config.sleepModel };
84
- if (config.dreamProvider && config.dreamModel) return { provider: config.dreamProvider, model: config.dreamModel };
85
- logger?.warn?.("dsh-mneme sleep: no llm route available");
86
- return undefined;
87
- }
88
-
89
- function makeSummary(m) {
90
- const text = (m.content ?? "").trim();
91
- if (!text) return (m.title ?? "").trim();
92
- return text.length <= SUMMARY_MAX ? text : `${text.slice(0, SUMMARY_MAX)}…`;
93
- }
94
-
95
- // ---------------------------------------------------------------- phases
96
-
97
- /**
98
- * Phase 1 — conflict resolution. Needs a semantic embedder + vector index.
99
- * In freeze mode (conflictFreezeEnabled) conflicting pairs are parked in
100
- * conflict_pending for human review (no LLM). Otherwise the LLM adjudicates:
101
- * each pair → winner kept / loser archived. Returns a per-run summary.
102
- */
103
- async function phaseConflicts(ctx, service, config, logger, runId, semantic = null, signal = null) {
104
- const embedder = semantic?.embedder;
105
- const vectorIndex = semantic?.vectorIndex;
106
- if (!embedder || !vectorIndex || typeof embedder.embed !== "function") {
107
- return { status: "skipped", reason: "no semantic embedder" };
108
- }
109
- const strictness = config.sleepConflictStrictness ?? "normal";
110
- const threshold = CONFLICT_THRESHOLDS[strictness] ?? CONFLICT_THRESHOLDS.normal;
111
- const memories = service.all().filter((m) => !m.archived && !m.session_disposed_at && !m.forgotten && m.type !== "summary");
112
- if (memories.length < 2) return { status: "skipped", reason: "too few memories" };
113
- if (signal?.aborted) return { status: "aborted", reason: "user activity" };
114
-
115
- // Backfill + collect vectors for every eligible memory (best effort).
116
- const vectors = new Array(memories.length);
117
- const missing = [];
118
- for (let i = 0; i < memories.length; i++) {
119
- const cached = vectorIndex.getEmbedding?.(memories[i].id);
120
- if (cached) vectors[i] = cached;
121
- else missing.push(i);
122
- }
123
- if (missing.length) {
124
- try {
125
- const texts = missing.map((i) => [memories[i].title, memories[i].content].filter(Boolean).join("\n"));
126
- const rows = await embedder.embed(texts);
127
- missing.forEach((mi, j) => {
128
- if (rows[j]?.length) {
129
- vectors[mi] = rows[j];
130
- vectorIndex.saveEmbedding?.(memories[mi].id, rows[j]);
131
- }
132
- });
133
- } catch (error) {
134
- logger?.warn?.(`dsh-mneme sleep: conflict vector backfill failed: ${String(error)}`);
135
- }
136
- }
137
- const usable = [];
138
- for (let i = 0; i < memories.length; i++) {
139
- if (vectors[i]?.length) usable.push(i);
140
- }
141
- if (usable.length < 2) return { status: "skipped", reason: "no usable vectors" };
142
- const usableMemories = usable.map((i) => memories[i]);
143
- const usableVectors = usable.map((i) => vectors[i]);
144
-
145
- const pairs = findPotentialConflicts(usableMemories, usableVectors, threshold);
146
- if (pairs.length === 0) return { status: "skipped", reason: "no conflicts found" };
147
-
148
- // Dedupe: each memory participates in at most one pair, highest similarity
149
- // first overlapping pairs would violate validateDecisions' "one claim".
150
- pairs.sort((a, b) => b.similarity - a.similarity);
151
- const used = new Set();
152
- const selected = [];
153
- for (const p of pairs) {
154
- if (used.has(p.a.id) || used.has(p.b.id)) continue;
155
- used.add(p.a.id);
156
- used.add(p.b.id);
157
- selected.push(p);
158
- }
159
-
160
- // Freeze mode: park pairs for manual review, no LLM required.
161
- if (config.conflictFreezeEnabled === true) {
162
- let frozen = 0;
163
- for (const p of selected) {
164
- try {
165
- service.saveConflictPending({ run_id: runId, memory_a: p.a.id, memory_b: p.b.id, reason: `相似度 ${p.similarity.toFixed(2)}` });
166
- frozen++;
167
- } catch (error) {
168
- logger?.warn?.(`dsh-mneme sleep: failed to freeze conflict ${p.a.id}/${p.b.id}: ${String(error)}`);
169
- }
170
- }
171
- return { status: frozen > 0 ? "ok" : "noop", frozen, pairs: selected.length };
172
- }
173
-
174
- // LLM adjudication.
175
- const route = resolveSleepRoute(ctx, config, logger);
176
- if (!route) return { status: "skipped", reason: "no llm route" };
177
- const snapshot = new Map();
178
- for (const p of selected) {
179
- snapshot.set(p.a.id, p.a);
180
- snapshot.set(p.b.id, p.b);
181
- }
182
- const listText = selected.map((p) =>
183
- `候选冲突:\nid=${p.a.id} | type=${p.a.type} | title=${p.a.title}\n${p.a.content}\n---\nid=${p.b.id} | type=${p.b.type} | title=${p.b.title}\n${p.b.content}\n(相似度 ${p.similarity.toFixed(2)})`
184
- ).join("\n\n");
185
- const text = await streamText(ctx, {
186
- provider: route.provider,
187
- model: route.model,
188
- purpose: "sleep-conflict",
189
- maxTokens: 2048,
190
- ...(config.sleepReasoningEffort && config.sleepReasoningEffort !== "none"
191
- ? { reasoningEffort: config.sleepReasoningEffort }
192
- : {}),
193
- messages: [
194
- { role: "system", content: [{ type: "text", text: CONFLICT_PROMPT }] },
195
- { role: "user", content: [{ type: "text", text: listText }] }
196
- ]
197
- });
198
- if (text === undefined) return { status: "failed", error: "llm failed" };
199
- const decisions = parseJsonArray(text);
200
- if (!decisions) return { status: "failed", error: "invalid decisions json" };
201
- // validateDecisions 要求每个 snapshot id 恰好被 claim 一次。v0.4.4 起它本身
202
- // 就会为未覆盖的 id 自动补 keep(dreamImplicitKeep 默认开启),这里保留显式
203
- // 预填作为防御性双保险——漏判读作"未裁决冲突"而非"冲突阶段整体失败"。
204
- const covered = new Set();
205
- for (const d of decisions) {
206
- if (d?.action === "conflict") {
207
- if (typeof d?.winner === "string") covered.add(d.winner);
208
- if (typeof d?.loser === "string") covered.add(d.loser);
209
- } else if (Array.isArray(d?.ids)) {
210
- for (const id of d.ids) covered.add(id);
211
- }
212
- }
213
- for (const id of snapshot.keys()) {
214
- if (!covered.has(id)) decisions.push({ action: "keep", ids: [id] });
215
- }
216
- const { ok, errors } = validateDecisions(decisions, snapshot, {});
217
- if (!ok) return { status: "failed", error: `invalid decisions: ${errors.join("; ")}` };
218
- const { applied, failures, conflicts } = applyDecisions(decisions, service, logger, snapshot, config);
219
- return {
220
- status: applied > 0 ? "ok" : failures.length ? "failed" : "noop",
221
- pairs: selected.length,
222
- applied,
223
- failures,
224
- conflicts
225
- };
226
- }
227
-
228
- /**
229
- * Phase 2 — archival demotion. No LLM: tiering is time-based, summaries are
230
- * truncations, and the full body is preserved in _full_content so nothing is
231
- * lost. Deterministic and cheap, so it runs even with no LLM route.
232
- */
233
- function phaseDemotion(service, config, logger, runId, signal = null) {
234
- const archiveDays = config.sleepArchiveDays ?? 30;
235
- const compressDays = config.sleepCompressDays ?? 90;
236
- const archiveCut = Date.now() - archiveDays * 86400000;
237
- const compressCut = Date.now() - compressDays * 86400000;
238
- const demoted = [];
239
- const archived = [];
240
- for (const m of service.all()) {
241
- if (signal?.aborted) break;
242
- if (m.archived || m.forgotten || m.session_disposed_at) continue;
243
- const ref = m.last_accessed_at ?? m.updated_at ?? m.created_at;
244
- if (!ref) continue;
245
- const t = new Date(ref).getTime();
246
- if (Number.isNaN(t)) continue;
247
- if (t < compressCut) {
248
- service.setArchived(m.id, true);
249
- archived.push(m.id);
250
- } else if (t < archiveCut) {
251
- // minRefTimeMs re-checks freshness inside demoteToSummary's transaction:
252
- // a recall touch landing after this snapshot must not demote the memory.
253
- service.demoteToSummary(m.id, makeSummary(m), { minRefTimeMs: archiveCut });
254
- demoted.push(m.id);
255
- }
256
- }
257
- return {
258
- status: demoted.length || archived.length ? "ok" : "noop",
259
- demoted,
260
- archived
261
- };
262
- }
263
-
264
- /**
265
- * Phase 3 — pattern discovery. The LLM scans the most recent memories and
266
- * mints type=pattern entries (create actions) with evidence references.
267
- * The empty snapshot is intentional: create claims no existing id, so the
268
- * "every id claimed" invariant is trivially satisfied for pure-create lists.
269
- */
270
- async function phasePatterns(ctx, service, config, logger, runId, signal = null) {
271
- const route = resolveSleepRoute(ctx, config, logger);
272
- if (!route) return { status: "skipped", reason: "no llm route" };
273
- const limit = config.sleepPatternMinMemories ?? 100;
274
- const memories = service
275
- .list({ limit: 200, includeForgotten: false })
276
- .filter((m) => !m.archived && !m.session_disposed_at && m.type !== "summary" && m.type !== "pattern")
277
- .sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1))
278
- .slice(0, limit);
279
- if (memories.length === 0) return { status: "skipped", reason: "no memories to scan" };
280
- if (signal?.aborted) return { status: "aborted", reason: "user activity" };
281
- const listText = memories
282
- .map((m) => `id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}`)
283
- .join("\n");
284
- const maxPatterns = config.sleepMaxPatternPerRun ?? 3;
285
- const text = await streamText(ctx, {
286
- provider: route.provider,
287
- model: route.model,
288
- purpose: "sleep-pattern",
289
- maxTokens: 2048,
290
- ...(config.sleepReasoningEffort && config.sleepReasoningEffort !== "none"
291
- ? { reasoningEffort: config.sleepReasoningEffort }
292
- : {}),
293
- messages: [
294
- { role: "system", content: [{ type: "text", text: PATTERN_PROMPT.replace("N", String(maxPatterns)) }] },
295
- { role: "user", content: [{ type: "text", text: listText }] }
296
- ]
297
- });
298
- if (text === undefined) return { status: "failed", error: "llm failed" };
299
- const decisions = parseJsonArray(text);
300
- if (!decisions || decisions.length === 0) return { status: "skipped", reason: "no patterns found" };
301
- // Evidence ids are provenance refs; an LLM-fabricated id would mint a dead
302
- // ev:tag pointing nowhere. Intersect evidence with the scanned set so only
303
- // real memory references survive.
304
- const scannedIds = new Set(memories.map((m) => m.id));
305
- for (const d of decisions) {
306
- if (d?.action === "create" && Array.isArray(d.evidence)) {
307
- d.evidence = d.evidence.filter((id) => typeof id === "string" && scannedIds.has(id));
308
- }
309
- }
310
- const snapshot = new Map();
311
- const { ok, errors } = validateDecisions(decisions, snapshot, {
312
- maxCreatePerRun: maxPatterns
313
- });
314
- if (!ok) return { status: "failed", error: `invalid decisions: ${errors.join("; ")}` };
315
- const { applied, failures, conflicts } = applyDecisions(decisions, service, logger, snapshot, config);
316
- return {
317
- status: applied > 0 ? "ok" : "noop",
318
- scanned: memories.length,
319
- applied,
320
- failures,
321
- conflicts
322
- };
323
- }
324
-
325
- /**
326
- * Phase 4 — entity relation completion. Detects orphan entities (zero
327
- * relations) and completes implied relations from memory co-occurrence:
328
- * entities named in the same memory → related_to; container kinds
329
- * (project/module) → part_of; tech-ish pairs → depends_on. Deterministic,
330
- * no LLM — cheap, so it runs even without a route. saveRelation is
331
- * bookkeeping (no write hook), so it never re-triggers the scheduler.
332
- */
333
- function inferRelationType(a, b) {
334
- if ((a.type === "project" || a.type === "module") && a.type !== b.type) return "part_of";
335
- if ((b.type === "project" || b.type === "module") && b.type !== a.type) return "part_of";
336
- if (/npm|plugin|api|sdk|lib|framework|package|deps?|build/i.test(`${a.name} ${b.name}`)) return "depends_on";
337
- return "related_to";
338
- }
339
-
340
- function phaseRelations(service, config, logger, runId, signal = null) {
341
- const entities = service.listEntities({ limit: 1000 }) ?? [];
342
- if (entities.length < 2) return { status: "skipped", reason: "too few entities" };
343
- const orphans = entities.filter((e) => (service.getRelations(e.id) ?? []).length === 0);
344
- if (orphans.length === 0) return { status: "skipped", reason: "no orphan entities" };
345
- const memories = service.all().filter((m) => !m.archived && !m.session_disposed_at && !m.forgotten);
346
- const seen = new Set();
347
- const related = [];
348
- const MAX_RELATIONS_PER_ORPHAN = 3;
349
- for (const o of orphans) {
350
- if (signal?.aborted) break;
351
- let made = 0;
352
- for (const m of memories) {
353
- if (signal?.aborted || made >= MAX_RELATIONS_PER_ORPHAN) break;
354
- const text = `${m.title ?? ""} ${m.content ?? ""}`;
355
- if (!text.includes(o.name)) continue;
356
- for (const other of entities) {
357
- if (other.id === o.id || other.name === o.name) continue;
358
- const key = [o.id, other.id].sort().join("|");
359
- if (seen.has(key)) continue;
360
- if (!text.includes(other.name)) continue;
361
- const relationType = inferRelationType(o, other);
362
- try {
363
- service.saveRelation({ from_entity: o.id, to_entity: other.id, relation_type: relationType, memory_id: m.id, metadata: { source: "sleep_relation_completion" } });
364
- seen.add(key);
365
- related.push({ from: o.id, to: other.id, type: relationType });
366
- made++;
367
- } catch (error) {
368
- logger?.warn?.(`dsh-mneme sleep: relation ${o.id}/${other.id} failed: ${String(error)}`);
369
- }
370
- }
371
- }
372
- }
373
- return {
374
- status: related.length > 0 ? "ok" : "noop",
375
- orphanCount: orphans.length,
376
- related
377
- };
378
- }
379
-
380
- // ---------------------------------------------------------------- run
381
-
382
- function deriveStatus(phases) {
383
- const list = Object.values(phases);
384
- if (list.length === 0) return "noop";
385
- const anyError = list.some((p) => p.status === "failed" || p.status === "error");
386
- const anyWork = list.some((p) => p.status === "ok");
387
- if (anyWork && anyError) return "degraded";
388
- if (anyError) return "failed";
389
- if (anyWork) return "ok";
390
- return "noop";
391
- }
392
-
393
- /**
394
- * Run one full sleep cycle. Best-effort across all phases; writes a
395
- * run_type='sleep' audit row (same dream_runs table) so sleep activity is
396
- * observable alongside consolidation runs.
397
- */
398
- export async function runSleep(ctx, service, config, logger, semantic = null, signal = null) {
399
- const runId = randomUUID();
400
- const phases = {};
401
- const attempt = async (name, fn) => {
402
- if (signal?.aborted) return; // user resumed activity stop before next phase
403
- try {
404
- phases[name] = await fn();
405
- } catch (error) {
406
- phases[name] = { status: "failed", error: error?.message ?? String(error) };
407
- logger?.warn?.(`dsh-mneme sleep: ${name} phase failed: ${error?.message ?? error}`);
408
- }
409
- };
410
- await attempt("conflicts", () => phaseConflicts(ctx, service, config, logger, runId, semantic, signal));
411
- await attempt("demotion", () => phaseDemotion(service, config, logger, runId, signal));
412
- await attempt("patterns", () => phasePatterns(ctx, service, config, logger, runId, signal));
413
- await attempt("relations", () => phaseRelations(service, config, logger, runId, signal));
414
-
415
- const status = deriveStatus(phases);
416
- const route = resolveSleepRoute(ctx, config, logger);
417
- const totalApplied = Object.values(phases).reduce((n, p) => n + (Number.isInteger(p?.applied) ? p.applied : 0), 0);
418
- const snapshotHash = createHash("sha256").update(JSON.stringify(phases)).digest("hex");
419
- const receipt = buildReceipt({
420
- runId,
421
- status,
422
- snapshotHash,
423
- inputCount: 0,
424
- applied: totalApplied,
425
- summaryStored: false
426
- });
427
- try {
428
- service.saveDreamRun({
429
- id: runId,
430
- status,
431
- provider: route?.provider,
432
- model: route?.model,
433
- snapshot_hash: snapshotHash,
434
- input_count: 0,
435
- input: null,
436
- decisions: phases,
437
- outcome: phases,
438
- applied: totalApplied,
439
- summary_stored: false,
440
- receipt,
441
- policy_epoch: config.policyEpoch ?? 0,
442
- run_type: "sleep"
443
- });
444
- } catch (error) {
445
- logger?.warn?.(`dsh-mneme sleep: failed to record audit run: ${String(error)}`);
446
- }
447
- return { ok: status === "ok" || status === "degraded", status, runId, phases, receipt };
448
- }
449
-
450
- // ---------------------------------------------------------------- scheduler
451
-
452
- /**
453
- * Idle-triggered scheduler. DSH plugins have no resident cron, so a sleep run
454
- * fires when: sleep is enabled, the store has been quiet for sleepIdleMinutes,
455
- * and the previous run is older than sleepMinIntervalHours. noteWrite() is
456
- * called on every store write and (re)arms an idle timer that re-checks at the
457
- * exact moment the idle window elapses — no polling, no cron.
458
- *
459
- * A `now` clock can be injected for tests; it defaults to Date.now.
460
- */
461
- export function createSleepScheduler({
462
- service,
463
- config,
464
- logger,
465
- onRun = null,
466
- now = () => Date.now(),
467
- setTimeoutFn = setTimeout,
468
- clearTimeoutFn = clearTimeout
469
- }) {
470
- let lastWriteAt = now();
471
- let lastRunAt = 0;
472
- let running = false;
473
- let disposed = false;
474
- let idleTimer = null;
475
- let sleepAbort = null;
476
-
477
- function armIdleTimer() {
478
- if (disposed || idleTimer) return;
479
- if (config.sleepModeEnabled !== true) return;
480
- const idleMs = (config.sleepIdleMinutes ?? 5) * 60000;
481
- const delay = Math.max(0, idleMs - (now() - lastWriteAt)) + 1000;
482
- idleTimer = setTimeoutFn(async () => {
483
- idleTimer = null;
484
- await maybeSchedule();
485
- }, delay);
486
- idleTimer.unref?.();
487
- }
488
-
489
- function shouldRun(at = now()) {
490
- if (disposed || running) return false;
491
- if (config.sleepModeEnabled !== true) return false;
492
- if (at - lastWriteAt < (config.sleepIdleMinutes ?? 5) * 60000) return false;
493
- // lastRunAt === 0 means never ran — the min-interval check must not block
494
- // the very first cycle (a real run stamps a nonzero timestamp).
495
- if (lastRunAt > 0 && at - lastRunAt < (config.sleepMinIntervalHours ?? 8) * 3600000) return false;
496
- return true;
497
- }
498
-
499
- /** Called on writes: resets the idle clock and re-arms the fire timer. The
500
- * pending timer is cleared first a stale timer armed against the old idle
501
- * window would otherwise fire early, fail shouldRun, and leave nothing armed
502
- * for the next window (a missed trigger until the next write).
503
- *
504
- * While a sleep run is executing (running=true) the in-flight AbortController
505
- * is NOT aborted: the run's own writes (demoteToSummary / setArchived ride
506
- * the normal write-hook path) would otherwise self-abort the cycle. External
507
- * activity during the run still resets the idle clock here, so no new cycle
508
- * fires until the store is quiet again. */
509
- function noteWrite() {
510
- lastWriteAt = now();
511
- if (!running && sleepAbort) {
512
- sleepAbort.abort(); // user resumed activity interrupt an idle run
513
- }
514
- if (idleTimer) {
515
- clearTimeoutFn(idleTimer);
516
- idleTimer = null;
517
- }
518
- armIdleTimer();
519
- }
520
-
521
- async function maybeSchedule() {
522
- if (!shouldRun()) return false;
523
- running = true;
524
- const abort = new AbortController();
525
- sleepAbort = abort;
526
- try {
527
- lastRunAt = now();
528
- const result = await service.enqueue(() =>
529
- onRun ? onRun(abort.signal) : Promise.resolve({ ok: true, skipped: true })
530
- );
531
- return !!(result && result.ok);
532
- } catch (error) {
533
- logger?.warn?.(`dsh-mneme sleep: run failed: ${error?.message ?? error}`);
534
- return false;
535
- } finally {
536
- sleepAbort = null;
537
- running = false;
538
- }
539
- }
540
-
541
- async function dispose() {
542
- disposed = true;
543
- if (idleTimer) {
544
- clearTimeoutFn(idleTimer);
545
- idleTimer = null;
546
- }
547
- if (sleepAbort) {
548
- sleepAbort.abort();
549
- sleepAbort = null;
550
- }
551
- }
552
-
553
- return { noteWrite, maybeSchedule, shouldRun, dispose };
554
- }
1
+ // System-level sleep (v0.4.0): an idle-triggered, LLM-assisted deep pass over
2
+ // the memory store. Four independent, fail-safe phases:
3
+ // 1. conflict resolution — high-similarity same-type pairs are either parked
4
+ // for review (freeze mode) or adjudicated by the LLM (winner kept / loser
5
+ // archived), reusing the dream conflict machinery. Strictness-graded.
6
+ // 2. archival demotion — memories unreferenced past sleepArchiveDays shrink
7
+ // to a one-line summary with the full body moved to _full_content; past
8
+ // sleepCompressDays they are archived outright.
9
+ // 3. pattern discovery — the LLM scans the most recent memories and mints
10
+ // type=pattern entries carrying evidence id references.
11
+ // 4. relation completion — orphan entities (zero relations) get implied
12
+ // relations completed from memory co-occurrence.
13
+ // Each phase is wrapped so one failure never aborts the others, and a missing
14
+ // LLM route / semantic embedder only skips the phases that need it. A run is
15
+ // abortable via an AbortController signal (user activity) — phases check the
16
+ // signal between batches so a running cycle yields promptly.
17
+ import { randomUUID, createHash } from "node:crypto";
18
+ import { validateDecisions, applyDecisions } from "./decisions.js";
19
+ import { findPotentialConflicts } from "./clustering.js";
20
+ import { buildReceipt } from "../dream.js";
21
+
22
+ const SUMMARY_MAX = 120;
23
+ // Conflict similarity threshold per strictness level (v0.4.0):
24
+ // gentle only high-confidence pairs (0.92) — first-time users
25
+ // normal standard dream-level (0.85) — default
26
+ // aggressive low-confidence pairs too (0.75) — bloated stores
27
+ const CONFLICT_THRESHOLDS = { gentle: 0.92, normal: 0.85, aggressive: 0.75 };
28
+
29
+ const CONFLICT_PROMPT = `你是记忆库冲突仲裁助手。下面是检测到的高相似度记忆对,可能内容矛盾或重复。
30
+ 对每一对输出一个 decision 对象:
31
+ - 两条确实矛盾/重复 → { "action": "conflict", "winner": <保留的id>, "loser": <归档的id>, "reason": "理由" }
32
+ - 两条只是主题相近、并无矛盾 → { "action": "keep", "ids": [<两个id>] }
33
+ 规则:
34
+ - winner 应为信息更完整、更新或更可信的一条
35
+ - 只使用提供的 id,不要编造
36
+ - 每对必须输出一个 decision
37
+ - 只输出 JSON 数组,不要其他文字`;
38
+
39
+ const PATTERN_PROMPT = `你是记忆库模式发现助手。下面是最近的记忆条目(id、类型、标题、内容)。
40
+ 请发现跨条目的稳定模式:用户偏好的规律、反复出现的主题、可复用的工作流或项目规律。
41
+ 对每个模式输出一个 create decision:
42
+ { "action": "create", "type": "pattern", "title": "模式一句话标题", "content": "模式详细描述(2-4句)", "importance": 1-5, "evidence": ["支持该模式的记忆id"] }
43
+ 规则:
44
+ - 只输出有据可依的模式,宁缺毋滥
45
+ - evidence 必须是列表中真实存在的 id
46
+ - 最多输出 N 个模式
47
+ - 只输出 JSON 数组,不要其他文字`;
48
+
49
+ function parseJsonArray(text) {
50
+ if (typeof text !== "string") return undefined;
51
+ const start = text.indexOf("[");
52
+ const end = text.lastIndexOf("]");
53
+ if (start === -1 || end <= start) return undefined;
54
+ try {
55
+ const parsed = JSON.parse(text.slice(start, end + 1));
56
+ return Array.isArray(parsed) ? parsed : undefined;
57
+ } catch {
58
+ return undefined;
59
+ }
60
+ }
61
+
62
+ /** Same stream consumption contract as dream.js. */
63
+ async function streamText(ctx, options) {
64
+ if (!ctx?.llm?.stream) return undefined;
65
+ let text = "";
66
+ for await (const chunk of ctx.llm.stream(options)) {
67
+ if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
68
+ if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
69
+ return undefined;
70
+ }
71
+ }
72
+ return text;
73
+ }
74
+
75
+ // LLM route (Issue #25): explicit config wins sleepProvider/sleepModel first,
76
+ // then the dream route as a shared fallback (config is the user's declared
77
+ // override), and agent default model only last as the generic fallback. Sleep
78
+ // can pin a cheaper model for its bulk passes without disturbing the dream
79
+ // route. The agent-default-last ordering matters: in a standard DSH install
80
+ // agentDefaultModel always resolves, so putting it first would make every
81
+ // config route dead code.
82
+ function resolveSleepRoute(ctx, config, logger) {
83
+ if (config.sleepProvider && config.sleepModel) return { provider: config.sleepProvider, model: config.sleepModel };
84
+ if (config.dreamProvider && config.dreamModel) return { provider: config.dreamProvider, model: config.dreamModel };
85
+ try {
86
+ const sel = ctx?.agentDefaultModel?.currentSelection?.();
87
+ if (sel?.provider && sel?.model) {
88
+ logger?.info?.("dsh-mneme sleep: no sleepProvider/sleepModel config, falling back to agent default");
89
+ return { provider: sel.provider, model: sel.model };
90
+ }
91
+ } catch { /* fall through */ }
92
+ logger?.warn?.("dsh-mneme sleep: no llm route available");
93
+ return undefined;
94
+ }
95
+
96
+ function makeSummary(m) {
97
+ const text = (m.content ?? "").trim();
98
+ if (!text) return (m.title ?? "").trim();
99
+ return text.length <= SUMMARY_MAX ? text : `${text.slice(0, SUMMARY_MAX)}…`;
100
+ }
101
+
102
+ // ---------------------------------------------------------------- phases
103
+
104
+ /**
105
+ * Phase 1 — conflict resolution. Needs a semantic embedder + vector index.
106
+ * In freeze mode (conflictFreezeEnabled) conflicting pairs are parked in
107
+ * conflict_pending for human review (no LLM). Otherwise the LLM adjudicates:
108
+ * each pair → winner kept / loser archived. Returns a per-run summary.
109
+ */
110
+ async function phaseConflicts(ctx, service, config, logger, runId, semantic = null, signal = null) {
111
+ const embedder = semantic?.embedder;
112
+ const vectorIndex = semantic?.vectorIndex;
113
+ if (!embedder || !vectorIndex || typeof embedder.embed !== "function") {
114
+ return { status: "skipped", reason: "no semantic embedder" };
115
+ }
116
+ const strictness = config.sleepConflictStrictness ?? "normal";
117
+ const threshold = CONFLICT_THRESHOLDS[strictness] ?? CONFLICT_THRESHOLDS.normal;
118
+ const memories = service.all().filter((m) => !m.archived && !m.session_disposed_at && !m.forgotten && m.type !== "summary");
119
+ if (memories.length < 2) return { status: "skipped", reason: "too few memories" };
120
+ if (signal?.aborted) return { status: "aborted", reason: "user activity" };
121
+
122
+ // Backfill + collect vectors for every eligible memory (best effort).
123
+ const vectors = new Array(memories.length);
124
+ const missing = [];
125
+ for (let i = 0; i < memories.length; i++) {
126
+ const cached = vectorIndex.getEmbedding?.(memories[i].id);
127
+ if (cached) vectors[i] = cached;
128
+ else missing.push(i);
129
+ }
130
+ if (missing.length) {
131
+ try {
132
+ const texts = missing.map((i) => [memories[i].title, memories[i].content].filter(Boolean).join("\n"));
133
+ const rows = await embedder.embed(texts);
134
+ missing.forEach((mi, j) => {
135
+ if (rows[j]?.length) {
136
+ vectors[mi] = rows[j];
137
+ vectorIndex.saveEmbedding?.(memories[mi].id, rows[j]);
138
+ }
139
+ });
140
+ } catch (error) {
141
+ logger?.warn?.(`dsh-mneme sleep: conflict vector backfill failed: ${String(error)}`);
142
+ }
143
+ }
144
+ const usable = [];
145
+ for (let i = 0; i < memories.length; i++) {
146
+ if (vectors[i]?.length) usable.push(i);
147
+ }
148
+ if (usable.length < 2) return { status: "skipped", reason: "no usable vectors" };
149
+ const usableMemories = usable.map((i) => memories[i]);
150
+ const usableVectors = usable.map((i) => vectors[i]);
151
+
152
+ const pairs = findPotentialConflicts(usableMemories, usableVectors, threshold);
153
+ if (pairs.length === 0) return { status: "skipped", reason: "no conflicts found" };
154
+
155
+ // Dedupe: each memory participates in at most one pair, highest similarity
156
+ // first — overlapping pairs would violate validateDecisions' "one claim".
157
+ pairs.sort((a, b) => b.similarity - a.similarity);
158
+ const used = new Set();
159
+ const selected = [];
160
+ for (const p of pairs) {
161
+ if (used.has(p.a.id) || used.has(p.b.id)) continue;
162
+ used.add(p.a.id);
163
+ used.add(p.b.id);
164
+ selected.push(p);
165
+ }
166
+
167
+ // Freeze mode: park pairs for manual review, no LLM required.
168
+ if (config.conflictFreezeEnabled === true) {
169
+ let frozen = 0;
170
+ for (const p of selected) {
171
+ try {
172
+ service.saveConflictPending({ run_id: runId, memory_a: p.a.id, memory_b: p.b.id, reason: `相似度 ${p.similarity.toFixed(2)}` });
173
+ frozen++;
174
+ } catch (error) {
175
+ logger?.warn?.(`dsh-mneme sleep: failed to freeze conflict ${p.a.id}/${p.b.id}: ${String(error)}`);
176
+ }
177
+ }
178
+ return { status: frozen > 0 ? "ok" : "noop", frozen, pairs: selected.length };
179
+ }
180
+
181
+ // LLM adjudication.
182
+ const route = resolveSleepRoute(ctx, config, logger);
183
+ if (!route) return { status: "skipped", reason: "no llm route" };
184
+ const snapshot = new Map();
185
+ for (const p of selected) {
186
+ snapshot.set(p.a.id, p.a);
187
+ snapshot.set(p.b.id, p.b);
188
+ }
189
+ const listText = selected.map((p) =>
190
+ `候选冲突:\nid=${p.a.id} | type=${p.a.type} | title=${p.a.title}\n${p.a.content}\n---\nid=${p.b.id} | type=${p.b.type} | title=${p.b.title}\n${p.b.content}\n(相似度 ${p.similarity.toFixed(2)})`
191
+ ).join("\n\n");
192
+ const text = await streamText(ctx, {
193
+ provider: route.provider,
194
+ model: route.model,
195
+ purpose: "sleep-conflict",
196
+ maxTokens: 2048,
197
+ ...(config.sleepReasoningEffort && config.sleepReasoningEffort !== "none"
198
+ ? { reasoningEffort: config.sleepReasoningEffort }
199
+ : {}),
200
+ messages: [
201
+ { role: "system", content: [{ type: "text", text: CONFLICT_PROMPT }] },
202
+ { role: "user", content: [{ type: "text", text: listText }] }
203
+ ]
204
+ });
205
+ if (text === undefined) return { status: "failed", error: "llm failed" };
206
+ const decisions = parseJsonArray(text);
207
+ if (!decisions) return { status: "failed", error: "invalid decisions json" };
208
+ // validateDecisions 要求每个 snapshot id 恰好被 claim 一次。v0.4.4 起它本身
209
+ // 就会为未覆盖的 id 自动补 keep(dreamImplicitKeep 默认开启),这里保留显式
210
+ // 预填作为防御性双保险——漏判读作"未裁决冲突"而非"冲突阶段整体失败"。
211
+ const covered = new Set();
212
+ for (const d of decisions) {
213
+ if (d?.action === "conflict") {
214
+ if (typeof d?.winner === "string") covered.add(d.winner);
215
+ if (typeof d?.loser === "string") covered.add(d.loser);
216
+ } else if (Array.isArray(d?.ids)) {
217
+ for (const id of d.ids) covered.add(id);
218
+ }
219
+ }
220
+ for (const id of snapshot.keys()) {
221
+ if (!covered.has(id)) decisions.push({ action: "keep", ids: [id] });
222
+ }
223
+ const { ok, errors } = validateDecisions(decisions, snapshot, {});
224
+ if (!ok) return { status: "failed", error: `invalid decisions: ${errors.join("; ")}` };
225
+ const { applied, failures, conflicts } = applyDecisions(decisions, service, logger, snapshot, config);
226
+ return {
227
+ status: applied > 0 ? "ok" : failures.length ? "failed" : "noop",
228
+ pairs: selected.length,
229
+ applied,
230
+ failures,
231
+ conflicts
232
+ };
233
+ }
234
+
235
+ /**
236
+ * Phase 2 — archival demotion. No LLM: tiering is time-based, summaries are
237
+ * truncations, and the full body is preserved in _full_content so nothing is
238
+ * lost. Deterministic and cheap, so it runs even with no LLM route.
239
+ */
240
+ function phaseDemotion(service, config, logger, runId, signal = null) {
241
+ const archiveDays = config.sleepArchiveDays ?? 30;
242
+ const compressDays = config.sleepCompressDays ?? 90;
243
+ const archiveCut = Date.now() - archiveDays * 86400000;
244
+ const compressCut = Date.now() - compressDays * 86400000;
245
+ const demoted = [];
246
+ const archived = [];
247
+ for (const m of service.all()) {
248
+ if (signal?.aborted) break;
249
+ if (m.archived || m.forgotten || m.session_disposed_at) continue;
250
+ const ref = m.last_accessed_at ?? m.updated_at ?? m.created_at;
251
+ if (!ref) continue;
252
+ const t = new Date(ref).getTime();
253
+ if (Number.isNaN(t)) continue;
254
+ if (t < compressCut) {
255
+ service.setArchived(m.id, true);
256
+ archived.push(m.id);
257
+ } else if (t < archiveCut) {
258
+ // minRefTimeMs re-checks freshness inside demoteToSummary's transaction:
259
+ // a recall touch landing after this snapshot must not demote the memory.
260
+ service.demoteToSummary(m.id, makeSummary(m), { minRefTimeMs: archiveCut });
261
+ demoted.push(m.id);
262
+ }
263
+ }
264
+ return {
265
+ status: demoted.length || archived.length ? "ok" : "noop",
266
+ demoted,
267
+ archived
268
+ };
269
+ }
270
+
271
+ /**
272
+ * Phase 3 pattern discovery. The LLM scans the most recent memories and
273
+ * mints type=pattern entries (create actions) with evidence references.
274
+ * The empty snapshot is intentional: create claims no existing id, so the
275
+ * "every id claimed" invariant is trivially satisfied for pure-create lists.
276
+ */
277
+ async function phasePatterns(ctx, service, config, logger, runId, signal = null) {
278
+ const route = resolveSleepRoute(ctx, config, logger);
279
+ if (!route) return { status: "skipped", reason: "no llm route" };
280
+ const limit = config.sleepPatternMinMemories ?? 100;
281
+ const memories = service
282
+ .list({ limit: 200, includeForgotten: false })
283
+ .filter((m) => !m.archived && !m.session_disposed_at && m.type !== "summary" && m.type !== "pattern")
284
+ .sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1))
285
+ .slice(0, limit);
286
+ if (memories.length === 0) return { status: "skipped", reason: "no memories to scan" };
287
+ if (signal?.aborted) return { status: "aborted", reason: "user activity" };
288
+ const listText = memories
289
+ .map((m) => `id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}`)
290
+ .join("\n");
291
+ const maxPatterns = config.sleepMaxPatternPerRun ?? 3;
292
+ const text = await streamText(ctx, {
293
+ provider: route.provider,
294
+ model: route.model,
295
+ purpose: "sleep-pattern",
296
+ maxTokens: 2048,
297
+ ...(config.sleepReasoningEffort && config.sleepReasoningEffort !== "none"
298
+ ? { reasoningEffort: config.sleepReasoningEffort }
299
+ : {}),
300
+ messages: [
301
+ { role: "system", content: [{ type: "text", text: PATTERN_PROMPT.replace("N", String(maxPatterns)) }] },
302
+ { role: "user", content: [{ type: "text", text: listText }] }
303
+ ]
304
+ });
305
+ if (text === undefined) return { status: "failed", error: "llm failed" };
306
+ const decisions = parseJsonArray(text);
307
+ if (!decisions || decisions.length === 0) return { status: "skipped", reason: "no patterns found" };
308
+ // Evidence ids are provenance refs; an LLM-fabricated id would mint a dead
309
+ // ev:tag pointing nowhere. Intersect evidence with the scanned set so only
310
+ // real memory references survive.
311
+ const scannedIds = new Set(memories.map((m) => m.id));
312
+ for (const d of decisions) {
313
+ if (d?.action === "create" && Array.isArray(d.evidence)) {
314
+ d.evidence = d.evidence.filter((id) => typeof id === "string" && scannedIds.has(id));
315
+ }
316
+ }
317
+ const snapshot = new Map();
318
+ const { ok, errors } = validateDecisions(decisions, snapshot, {
319
+ maxCreatePerRun: maxPatterns
320
+ });
321
+ if (!ok) return { status: "failed", error: `invalid decisions: ${errors.join("; ")}` };
322
+ const { applied, failures, conflicts } = applyDecisions(decisions, service, logger, snapshot, config);
323
+ return {
324
+ status: applied > 0 ? "ok" : "noop",
325
+ scanned: memories.length,
326
+ applied,
327
+ failures,
328
+ conflicts
329
+ };
330
+ }
331
+
332
+ /**
333
+ * Phase 4 — entity relation completion. Detects orphan entities (zero
334
+ * relations) and completes implied relations from memory co-occurrence:
335
+ * entities named in the same memory related_to; container kinds
336
+ * (project/module) part_of; tech-ish pairs → depends_on. Deterministic,
337
+ * no LLM — cheap, so it runs even without a route. saveRelation is
338
+ * bookkeeping (no write hook), so it never re-triggers the scheduler.
339
+ */
340
+ function inferRelationType(a, b) {
341
+ if ((a.type === "project" || a.type === "module") && a.type !== b.type) return "part_of";
342
+ if ((b.type === "project" || b.type === "module") && b.type !== a.type) return "part_of";
343
+ if (/npm|plugin|api|sdk|lib|framework|package|deps?|build/i.test(`${a.name} ${b.name}`)) return "depends_on";
344
+ return "related_to";
345
+ }
346
+
347
+ function phaseRelations(service, config, logger, runId, signal = null) {
348
+ const entities = service.listEntities({ limit: 1000 }) ?? [];
349
+ if (entities.length < 2) return { status: "skipped", reason: "too few entities" };
350
+ const orphans = entities.filter((e) => (service.getRelations(e.id) ?? []).length === 0);
351
+ if (orphans.length === 0) return { status: "skipped", reason: "no orphan entities" };
352
+ const memories = service.all().filter((m) => !m.archived && !m.session_disposed_at && !m.forgotten);
353
+ const seen = new Set();
354
+ const related = [];
355
+ const MAX_RELATIONS_PER_ORPHAN = 3;
356
+ for (const o of orphans) {
357
+ if (signal?.aborted) break;
358
+ let made = 0;
359
+ for (const m of memories) {
360
+ if (signal?.aborted || made >= MAX_RELATIONS_PER_ORPHAN) break;
361
+ const text = `${m.title ?? ""} ${m.content ?? ""}`;
362
+ if (!text.includes(o.name)) continue;
363
+ for (const other of entities) {
364
+ if (other.id === o.id || other.name === o.name) continue;
365
+ const key = [o.id, other.id].sort().join("|");
366
+ if (seen.has(key)) continue;
367
+ if (!text.includes(other.name)) continue;
368
+ const relationType = inferRelationType(o, other);
369
+ try {
370
+ service.saveRelation({ from_entity: o.id, to_entity: other.id, relation_type: relationType, memory_id: m.id, metadata: { source: "sleep_relation_completion" } });
371
+ seen.add(key);
372
+ related.push({ from: o.id, to: other.id, type: relationType });
373
+ made++;
374
+ } catch (error) {
375
+ logger?.warn?.(`dsh-mneme sleep: relation ${o.id}/${other.id} failed: ${String(error)}`);
376
+ }
377
+ }
378
+ }
379
+ }
380
+ return {
381
+ status: related.length > 0 ? "ok" : "noop",
382
+ orphanCount: orphans.length,
383
+ related
384
+ };
385
+ }
386
+
387
+ // ---------------------------------------------------------------- run
388
+
389
+ function deriveStatus(phases) {
390
+ const list = Object.values(phases);
391
+ if (list.length === 0) return "noop";
392
+ const anyError = list.some((p) => p.status === "failed" || p.status === "error");
393
+ const anyWork = list.some((p) => p.status === "ok");
394
+ if (anyWork && anyError) return "degraded";
395
+ if (anyError) return "failed";
396
+ if (anyWork) return "ok";
397
+ return "noop";
398
+ }
399
+
400
+ /**
401
+ * Run one full sleep cycle. Best-effort across all phases; writes a
402
+ * run_type='sleep' audit row (same dream_runs table) so sleep activity is
403
+ * observable alongside consolidation runs.
404
+ */
405
+ export async function runSleep(ctx, service, config, logger, semantic = null, signal = null) {
406
+ const runId = randomUUID();
407
+ const phases = {};
408
+ const attempt = async (name, fn) => {
409
+ if (signal?.aborted) return; // user resumed activity — stop before next phase
410
+ try {
411
+ phases[name] = await fn();
412
+ } catch (error) {
413
+ phases[name] = { status: "failed", error: error?.message ?? String(error) };
414
+ logger?.warn?.(`dsh-mneme sleep: ${name} phase failed: ${error?.message ?? error}`);
415
+ }
416
+ };
417
+ await attempt("conflicts", () => phaseConflicts(ctx, service, config, logger, runId, semantic, signal));
418
+ await attempt("demotion", () => phaseDemotion(service, config, logger, runId, signal));
419
+ await attempt("patterns", () => phasePatterns(ctx, service, config, logger, runId, signal));
420
+ await attempt("relations", () => phaseRelations(service, config, logger, runId, signal));
421
+
422
+ const status = deriveStatus(phases);
423
+ const route = resolveSleepRoute(ctx, config, logger);
424
+ const totalApplied = Object.values(phases).reduce((n, p) => n + (Number.isInteger(p?.applied) ? p.applied : 0), 0);
425
+ const snapshotHash = createHash("sha256").update(JSON.stringify(phases)).digest("hex");
426
+ const receipt = buildReceipt({
427
+ runId,
428
+ status,
429
+ snapshotHash,
430
+ inputCount: 0,
431
+ applied: totalApplied,
432
+ summaryStored: false
433
+ });
434
+ try {
435
+ service.saveDreamRun({
436
+ id: runId,
437
+ status,
438
+ provider: route?.provider,
439
+ model: route?.model,
440
+ snapshot_hash: snapshotHash,
441
+ input_count: 0,
442
+ input: null,
443
+ decisions: phases,
444
+ outcome: phases,
445
+ applied: totalApplied,
446
+ summary_stored: false,
447
+ receipt,
448
+ policy_epoch: config.policyEpoch ?? 0,
449
+ run_type: "sleep"
450
+ });
451
+ } catch (error) {
452
+ logger?.warn?.(`dsh-mneme sleep: failed to record audit run: ${String(error)}`);
453
+ }
454
+ return { ok: status === "ok" || status === "degraded", status, runId, phases, receipt };
455
+ }
456
+
457
+ // ---------------------------------------------------------------- scheduler
458
+
459
+ /**
460
+ * Idle-triggered scheduler. DSH plugins have no resident cron, so a sleep run
461
+ * fires when: sleep is enabled, the store has been quiet for sleepIdleMinutes,
462
+ * and the previous run is older than sleepMinIntervalHours. noteWrite() is
463
+ * called on every store write and (re)arms an idle timer that re-checks at the
464
+ * exact moment the idle window elapses — no polling, no cron.
465
+ *
466
+ * A `now` clock can be injected for tests; it defaults to Date.now.
467
+ */
468
+ export function createSleepScheduler({
469
+ service,
470
+ config,
471
+ logger,
472
+ onRun = null,
473
+ now = () => Date.now(),
474
+ setTimeoutFn = setTimeout,
475
+ clearTimeoutFn = clearTimeout
476
+ }) {
477
+ let lastWriteAt = now();
478
+ let lastRunAt = 0;
479
+ let running = false;
480
+ let disposed = false;
481
+ let idleTimer = null;
482
+ let sleepAbort = null;
483
+
484
+ function armIdleTimer() {
485
+ if (disposed || idleTimer) return;
486
+ if (config.sleepModeEnabled !== true) return;
487
+ const idleMs = (config.sleepIdleMinutes ?? 5) * 60000;
488
+ const delay = Math.max(0, idleMs - (now() - lastWriteAt)) + 1000;
489
+ idleTimer = setTimeoutFn(async () => {
490
+ idleTimer = null;
491
+ await maybeSchedule();
492
+ }, delay);
493
+ idleTimer.unref?.();
494
+ }
495
+
496
+ function shouldRun(at = now()) {
497
+ if (disposed || running) return false;
498
+ if (config.sleepModeEnabled !== true) return false;
499
+ if (at - lastWriteAt < (config.sleepIdleMinutes ?? 5) * 60000) return false;
500
+ // lastRunAt === 0 means never ran the min-interval check must not block
501
+ // the very first cycle (a real run stamps a nonzero timestamp).
502
+ if (lastRunAt > 0 && at - lastRunAt < (config.sleepMinIntervalHours ?? 8) * 3600000) return false;
503
+ return true;
504
+ }
505
+
506
+ /** Called on writes: resets the idle clock and re-arms the fire timer. The
507
+ * pending timer is cleared first a stale timer armed against the old idle
508
+ * window would otherwise fire early, fail shouldRun, and leave nothing armed
509
+ * for the next window (a missed trigger until the next write).
510
+ *
511
+ * While a sleep run is executing (running=true) the in-flight AbortController
512
+ * is NOT aborted: the run's own writes (demoteToSummary / setArchived ride
513
+ * the normal write-hook path) would otherwise self-abort the cycle. External
514
+ * activity during the run still resets the idle clock here, so no new cycle
515
+ * fires until the store is quiet again. */
516
+ function noteWrite() {
517
+ lastWriteAt = now();
518
+ if (!running && sleepAbort) {
519
+ sleepAbort.abort(); // user resumed activity — interrupt an idle run
520
+ }
521
+ if (idleTimer) {
522
+ clearTimeoutFn(idleTimer);
523
+ idleTimer = null;
524
+ }
525
+ armIdleTimer();
526
+ }
527
+
528
+ async function maybeSchedule() {
529
+ if (!shouldRun()) return false;
530
+ running = true;
531
+ const abort = new AbortController();
532
+ sleepAbort = abort;
533
+ try {
534
+ lastRunAt = now();
535
+ const result = await service.enqueue(() =>
536
+ onRun ? onRun(abort.signal) : Promise.resolve({ ok: true, skipped: true })
537
+ );
538
+ return !!(result && result.ok);
539
+ } catch (error) {
540
+ logger?.warn?.(`dsh-mneme sleep: run failed: ${error?.message ?? error}`);
541
+ return false;
542
+ } finally {
543
+ sleepAbort = null;
544
+ running = false;
545
+ }
546
+ }
547
+
548
+ async function dispose() {
549
+ disposed = true;
550
+ if (idleTimer) {
551
+ clearTimeoutFn(idleTimer);
552
+ idleTimer = null;
553
+ }
554
+ if (sleepAbort) {
555
+ sleepAbort.abort();
556
+ sleepAbort = null;
557
+ }
558
+ }
559
+
560
+ return { noteWrite, maybeSchedule, shouldRun, dispose };
561
+ }