@modusensus/dsh-mneme 0.3.8 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -3
- package/lib/api.js +1 -1
- package/lib/config.js +38 -0
- package/lib/dream/decisions.js +49 -1
- package/lib/dream/sleep.js +550 -0
- package/lib/index.js +18 -0
- package/lib/mirror.js +24 -12
- package/lib/service.js +141 -23
- package/lib/store.js +168 -34
- package/package.json +2 -2
- package/scripts/e2e-dsh.js +4 -2
- package/src/api.js +1 -1
- package/src/config.js +38 -0
- package/src/dream/decisions.js +49 -1
- package/src/dream/sleep.js +550 -0
- package/src/index.js +18 -0
- package/src/mirror.js +24 -12
- package/src/service.js +141 -23
- package/src/store.js +168 -34
- package/test/mirror-generation.test.js +34 -1
- package/test/peer-blockers.test.js +42 -0
- package/test/sleep.test.js +365 -0
package/src/config.js
CHANGED
|
@@ -91,4 +91,42 @@ export const Config = z.object({
|
|
|
91
91
|
entityExtractionMaxAttrs: z.natural().min(1).max(50).default(20),
|
|
92
92
|
// Prefix/semantic search over entity names (used by recall).
|
|
93
93
|
entitySearchEnabled: z.boolean().default(true),
|
|
94
|
+
|
|
95
|
+
// --- sleep mode: idle-triggered deep maintenance (v0.4.0) ---------------
|
|
96
|
+
// Opt-in, off by default. Unlike autoDream (threshold-triggered, lightweight)
|
|
97
|
+
// sleep fires when the store has been quiet for sleepIdleMinutes and deep-
|
|
98
|
+
// maintains the whole library: conflict resolution, archival demotion,
|
|
99
|
+
// pattern discovery and entity relation completion. Abortable on user
|
|
100
|
+
// activity, audited into dream_runs (run_type='sleep'), and serialized with
|
|
101
|
+
// autoDream so the two never overlap.
|
|
102
|
+
sleepModeEnabled: z.boolean().default(false),
|
|
103
|
+
// Quiet window before a cycle fires (minutes).
|
|
104
|
+
sleepIdleMinutes: z.natural().min(1).max(60).default(5),
|
|
105
|
+
// Minimum gap between two sleep runs (hours) — a second idle window within
|
|
106
|
+
// this interval does not retrigger.
|
|
107
|
+
sleepMinIntervalHours: z.natural().min(1).max(168).default(8),
|
|
108
|
+
// Conflict adjudication strictness:
|
|
109
|
+
// gentle only high-confidence conflicts (threshold 0.92) are resolved
|
|
110
|
+
// normal standard dream-level (threshold 0.85)
|
|
111
|
+
// aggressive low-confidence pairs are also adjudicated (threshold 0.75)
|
|
112
|
+
sleepConflictStrictness: z.union([
|
|
113
|
+
z.const("gentle"),
|
|
114
|
+
z.const("normal"),
|
|
115
|
+
z.const("aggressive")
|
|
116
|
+
]).default("normal"),
|
|
117
|
+
// Archival demotion tiering (days since last access):
|
|
118
|
+
// >= sleepArchiveDays → shrink to summary, full body kept in _full_content
|
|
119
|
+
// >= sleepCompressDays → archived outright (entity relations preserved)
|
|
120
|
+
sleepArchiveDays: z.natural().min(7).max(365).default(30),
|
|
121
|
+
sleepCompressDays: z.natural().min(7).max(365).default(90),
|
|
122
|
+
// Pattern discovery scan window (most recent memories to scan).
|
|
123
|
+
sleepPatternMinMemories: z.natural().min(10).max(1000).default(100),
|
|
124
|
+
// How far back pattern discovery considers entity attr changes (days).
|
|
125
|
+
sleepPatternLookbackDays: z.natural().min(1).max(90).default(30),
|
|
126
|
+
// Max pattern memories minted per run (0 = disabled).
|
|
127
|
+
sleepMaxPatternPerRun: z.natural().min(0).max(10).default(3),
|
|
128
|
+
// Optional LLM route override for sleep's bulk passes (empty = use dream
|
|
129
|
+
// route / agent default model).
|
|
130
|
+
sleepProvider: z.string().default(""),
|
|
131
|
+
sleepModel: z.string().default(""),
|
|
94
132
|
});
|
package/src/dream/decisions.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const ACTIONS = new Set(["keep", "merge", "archive", "conflict", "update"]);
|
|
1
|
+
const ACTIONS = new Set(["keep", "merge", "archive", "conflict", "update", "create"]);
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Validate a dream decision list against a snapshot of eligible memories.
|
|
@@ -26,6 +26,25 @@ export function validateDecisions(decisions, snapshot, options = {}) {
|
|
|
26
26
|
errors.push(`${at}: conflict needs distinct winner and loser`);
|
|
27
27
|
continue;
|
|
28
28
|
}
|
|
29
|
+
} else if (d.action === "create") {
|
|
30
|
+
// Mint a fresh memory (sleep pattern discovery). Claims no existing id,
|
|
31
|
+
// so it skips the claiming loop below; evidence is optional provenance
|
|
32
|
+
// (already filtered to real ids by the caller) and is stored in content.
|
|
33
|
+
if (typeof d.title !== "string" || !d.title.trim()) {
|
|
34
|
+
errors.push(`${at}: create needs non-empty title`);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
if (typeof d.content !== "string" || !d.content.trim()) {
|
|
38
|
+
errors.push(`${at}: create needs non-empty content`);
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (d.importance !== undefined && (!Number.isInteger(d.importance) || d.importance < 1 || d.importance > 5)) {
|
|
42
|
+
errors.push(`${at}: create importance must be an integer 1-5 when provided`);
|
|
43
|
+
}
|
|
44
|
+
if (typeof d.type !== "string" || !d.type.trim()) {
|
|
45
|
+
errors.push(`${at}: create needs non-empty type`);
|
|
46
|
+
}
|
|
47
|
+
continue;
|
|
29
48
|
} else if (!Array.isArray(d.ids) || d.ids.length === 0) {
|
|
30
49
|
errors.push(`${at}: ${d.action} needs non-empty ids`);
|
|
31
50
|
continue;
|
|
@@ -95,6 +114,12 @@ export function validateDecisions(decisions, snapshot, options = {}) {
|
|
|
95
114
|
if (updateCount > maxUpdatePerRun) {
|
|
96
115
|
errors.push(`too many update decisions: ${updateCount} > ${maxUpdatePerRun}`);
|
|
97
116
|
}
|
|
117
|
+
// Cap pattern minting per run (sleepMaxPatternPerRun passes through here).
|
|
118
|
+
const createCount = decisions.filter((d) => d.action === "create").length;
|
|
119
|
+
const maxCreatePerRun = options.maxCreatePerRun ?? 5;
|
|
120
|
+
if (createCount > maxCreatePerRun) {
|
|
121
|
+
errors.push(`too many create decisions: ${createCount} > ${maxCreatePerRun}`);
|
|
122
|
+
}
|
|
98
123
|
// Every snapshot id must appear in at least one decision
|
|
99
124
|
for (const id of snapshot.keys()) {
|
|
100
125
|
if (!claimed.has(id)) errors.push(`memory ${JSON.stringify(id)} missing from decisions`);
|
|
@@ -202,10 +227,33 @@ function applyOne(d, service, snapshot, config = {}) {
|
|
|
202
227
|
case "archive": return applyArchive(d, service, snapshot);
|
|
203
228
|
case "merge": return applyMerge(d, service, snapshot, config);
|
|
204
229
|
case "conflict": return applyConflict(d, service, snapshot);
|
|
230
|
+
case "create": return applyCreate(d, service, config);
|
|
205
231
|
default: return applyUpdate(d, service, snapshot, config);
|
|
206
232
|
}
|
|
207
233
|
}
|
|
208
234
|
|
|
235
|
+
/**
|
|
236
|
+
* Mint a fresh memory (pattern discovery). No existing target, so no CAS guard.
|
|
237
|
+
* Evidence ids ride in the content so a pattern stays traceable to its source
|
|
238
|
+
* memories. saveWithDedupe dedupes identical mints (idempotent replay-safe).
|
|
239
|
+
*/
|
|
240
|
+
function applyCreate(d, service, config = {}) {
|
|
241
|
+
const title = String(d.title ?? "").trim();
|
|
242
|
+
const content = String(d.content ?? "").trim();
|
|
243
|
+
const importance = Number.isInteger(d.importance) ? d.importance : 3;
|
|
244
|
+
const type = typeof d.type === "string" ? d.type : "pattern";
|
|
245
|
+
const evidence = Array.isArray(d.evidence)
|
|
246
|
+
? d.evidence.filter((id) => typeof id === "string")
|
|
247
|
+
: [];
|
|
248
|
+
const body = evidence.length > 0
|
|
249
|
+
? `${content}\n\n[证据: ${evidence.join(", ")}]`
|
|
250
|
+
: content;
|
|
251
|
+
const created = service.saveWithDedupe({ type, title, content: body, importance });
|
|
252
|
+
const memory = created?.memory;
|
|
253
|
+
if (!memory) return "skipped"; // deduped/subsumed: nothing minted, clean no-op
|
|
254
|
+
return { applied: 1, committed: { action: "create", id: memory.id, type } };
|
|
255
|
+
}
|
|
256
|
+
|
|
209
257
|
function applyArchive(d, service, snapshot) {
|
|
210
258
|
const targets = d.ids.filter((id) => {
|
|
211
259
|
const mem = service.getById(id);
|
|
@@ -0,0 +1,550 @@
|
|
|
1
|
+
// System-level sleep (v0.4.0): an idle-triggered, LLM-assisted deep pass over
|
|
2
|
+
// the memory store. Four independent, fail-safe phases:
|
|
3
|
+
// 1. conflict resolution — high-similarity same-type pairs are either parked
|
|
4
|
+
// for review (freeze mode) or adjudicated by the LLM (winner kept / loser
|
|
5
|
+
// archived), reusing the dream conflict machinery. Strictness-graded.
|
|
6
|
+
// 2. archival demotion — memories unreferenced past sleepArchiveDays shrink
|
|
7
|
+
// to a one-line summary with the full body moved to _full_content; past
|
|
8
|
+
// sleepCompressDays they are archived outright.
|
|
9
|
+
// 3. pattern discovery — the LLM scans the most recent memories and mints
|
|
10
|
+
// type=pattern entries carrying evidence id references.
|
|
11
|
+
// 4. relation completion — orphan entities (zero relations) get implied
|
|
12
|
+
// relations completed from memory co-occurrence.
|
|
13
|
+
// Each phase is wrapped so one failure never aborts the others, and a missing
|
|
14
|
+
// LLM route / semantic embedder only skips the phases that need it. A run is
|
|
15
|
+
// abortable via an AbortController signal (user activity) — phases check the
|
|
16
|
+
// signal between batches so a running cycle yields promptly.
|
|
17
|
+
import { randomUUID, createHash } from "node:crypto";
|
|
18
|
+
import { validateDecisions, applyDecisions } from "./decisions.js";
|
|
19
|
+
import { findPotentialConflicts } from "./clustering.js";
|
|
20
|
+
import { buildReceipt } from "../dream.js";
|
|
21
|
+
|
|
22
|
+
const SUMMARY_MAX = 120;
|
|
23
|
+
// Conflict similarity threshold per strictness level (v0.4.0):
|
|
24
|
+
// gentle only high-confidence pairs (0.92) — first-time users
|
|
25
|
+
// normal standard dream-level (0.85) — default
|
|
26
|
+
// aggressive low-confidence pairs too (0.75) — bloated stores
|
|
27
|
+
const CONFLICT_THRESHOLDS = { gentle: 0.92, normal: 0.85, aggressive: 0.75 };
|
|
28
|
+
|
|
29
|
+
const CONFLICT_PROMPT = `你是记忆库冲突仲裁助手。下面是检测到的高相似度记忆对,可能内容矛盾或重复。
|
|
30
|
+
对每一对输出一个 decision 对象:
|
|
31
|
+
- 两条确实矛盾/重复 → { "action": "conflict", "winner": <保留的id>, "loser": <归档的id>, "reason": "理由" }
|
|
32
|
+
- 两条只是主题相近、并无矛盾 → { "action": "keep", "ids": [<两个id>] }
|
|
33
|
+
规则:
|
|
34
|
+
- winner 应为信息更完整、更新或更可信的一条
|
|
35
|
+
- 只使用提供的 id,不要编造
|
|
36
|
+
- 每对必须输出一个 decision
|
|
37
|
+
- 只输出 JSON 数组,不要其他文字`;
|
|
38
|
+
|
|
39
|
+
const PATTERN_PROMPT = `你是记忆库模式发现助手。下面是最近的记忆条目(id、类型、标题、内容)。
|
|
40
|
+
请发现跨条目的稳定模式:用户偏好的规律、反复出现的主题、可复用的工作流或项目规律。
|
|
41
|
+
对每个模式输出一个 create decision:
|
|
42
|
+
{ "action": "create", "type": "pattern", "title": "模式一句话标题", "content": "模式详细描述(2-4句)", "importance": 1-5, "evidence": ["支持该模式的记忆id"] }
|
|
43
|
+
规则:
|
|
44
|
+
- 只输出有据可依的模式,宁缺毋滥
|
|
45
|
+
- evidence 必须是列表中真实存在的 id
|
|
46
|
+
- 最多输出 N 个模式
|
|
47
|
+
- 只输出 JSON 数组,不要其他文字`;
|
|
48
|
+
|
|
49
|
+
function parseJsonArray(text) {
|
|
50
|
+
if (typeof text !== "string") return undefined;
|
|
51
|
+
const start = text.indexOf("[");
|
|
52
|
+
const end = text.lastIndexOf("]");
|
|
53
|
+
if (start === -1 || end <= start) return undefined;
|
|
54
|
+
try {
|
|
55
|
+
const parsed = JSON.parse(text.slice(start, end + 1));
|
|
56
|
+
return Array.isArray(parsed) ? parsed : undefined;
|
|
57
|
+
} catch {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Same stream consumption contract as dream.js. */
|
|
63
|
+
async function streamText(ctx, options) {
|
|
64
|
+
if (!ctx?.llm?.stream) return undefined;
|
|
65
|
+
let text = "";
|
|
66
|
+
for await (const chunk of ctx.llm.stream(options)) {
|
|
67
|
+
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
68
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return text;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** LLM route: agent default model first, then sleepProvider/Model, then the
|
|
76
|
+
* dream route as a shared fallback. Sleep can pin a cheaper model for its
|
|
77
|
+
* bulk passes without disturbing the dream route. */
|
|
78
|
+
function resolveSleepRoute(ctx, config, logger) {
|
|
79
|
+
try {
|
|
80
|
+
const sel = ctx?.agentDefaultModel?.currentSelection?.();
|
|
81
|
+
if (sel?.provider && sel?.model) return { provider: sel.provider, model: sel.model };
|
|
82
|
+
} catch { /* fall through to config route */ }
|
|
83
|
+
if (config.sleepProvider && config.sleepModel) return { provider: config.sleepProvider, model: config.sleepModel };
|
|
84
|
+
if (config.dreamProvider && config.dreamModel) return { provider: config.dreamProvider, model: config.dreamModel };
|
|
85
|
+
logger?.warn?.("dsh-mneme sleep: no llm route available");
|
|
86
|
+
return undefined;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function makeSummary(m) {
|
|
90
|
+
const text = (m.content ?? "").trim();
|
|
91
|
+
if (!text) return (m.title ?? "").trim();
|
|
92
|
+
return text.length <= SUMMARY_MAX ? text : `${text.slice(0, SUMMARY_MAX)}…`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ---------------------------------------------------------------- phases
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Phase 1 — conflict resolution. Needs a semantic embedder + vector index.
|
|
99
|
+
* In freeze mode (conflictFreezeEnabled) conflicting pairs are parked in
|
|
100
|
+
* conflict_pending for human review (no LLM). Otherwise the LLM adjudicates:
|
|
101
|
+
* each pair → winner kept / loser archived. Returns a per-run summary.
|
|
102
|
+
*/
|
|
103
|
+
async function phaseConflicts(ctx, service, config, logger, runId, semantic = null, signal = null) {
|
|
104
|
+
const embedder = semantic?.embedder;
|
|
105
|
+
const vectorIndex = semantic?.vectorIndex;
|
|
106
|
+
if (!embedder || !vectorIndex || typeof embedder.embed !== "function") {
|
|
107
|
+
return { status: "skipped", reason: "no semantic embedder" };
|
|
108
|
+
}
|
|
109
|
+
const strictness = config.sleepConflictStrictness ?? "normal";
|
|
110
|
+
const threshold = CONFLICT_THRESHOLDS[strictness] ?? CONFLICT_THRESHOLDS.normal;
|
|
111
|
+
const memories = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary");
|
|
112
|
+
if (memories.length < 2) return { status: "skipped", reason: "too few memories" };
|
|
113
|
+
if (signal?.aborted) return { status: "aborted", reason: "user activity" };
|
|
114
|
+
|
|
115
|
+
// Backfill + collect vectors for every eligible memory (best effort).
|
|
116
|
+
const vectors = new Array(memories.length);
|
|
117
|
+
const missing = [];
|
|
118
|
+
for (let i = 0; i < memories.length; i++) {
|
|
119
|
+
const cached = vectorIndex.getEmbedding?.(memories[i].id);
|
|
120
|
+
if (cached) vectors[i] = cached;
|
|
121
|
+
else missing.push(i);
|
|
122
|
+
}
|
|
123
|
+
if (missing.length) {
|
|
124
|
+
try {
|
|
125
|
+
const texts = missing.map((i) => [memories[i].title, memories[i].content].filter(Boolean).join("\n"));
|
|
126
|
+
const rows = await embedder.embed(texts);
|
|
127
|
+
missing.forEach((mi, j) => {
|
|
128
|
+
if (rows[j]?.length) {
|
|
129
|
+
vectors[mi] = rows[j];
|
|
130
|
+
vectorIndex.saveEmbedding?.(memories[mi].id, rows[j]);
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
} catch (error) {
|
|
134
|
+
logger?.warn?.(`dsh-mneme sleep: conflict vector backfill failed: ${String(error)}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const usable = [];
|
|
138
|
+
for (let i = 0; i < memories.length; i++) {
|
|
139
|
+
if (vectors[i]?.length) usable.push(i);
|
|
140
|
+
}
|
|
141
|
+
if (usable.length < 2) return { status: "skipped", reason: "no usable vectors" };
|
|
142
|
+
const usableMemories = usable.map((i) => memories[i]);
|
|
143
|
+
const usableVectors = usable.map((i) => vectors[i]);
|
|
144
|
+
|
|
145
|
+
const pairs = findPotentialConflicts(usableMemories, usableVectors, threshold);
|
|
146
|
+
if (pairs.length === 0) return { status: "skipped", reason: "no conflicts found" };
|
|
147
|
+
|
|
148
|
+
// Dedupe: each memory participates in at most one pair, highest similarity
|
|
149
|
+
// first — overlapping pairs would violate validateDecisions' "one claim".
|
|
150
|
+
pairs.sort((a, b) => b.similarity - a.similarity);
|
|
151
|
+
const used = new Set();
|
|
152
|
+
const selected = [];
|
|
153
|
+
for (const p of pairs) {
|
|
154
|
+
if (used.has(p.a.id) || used.has(p.b.id)) continue;
|
|
155
|
+
used.add(p.a.id);
|
|
156
|
+
used.add(p.b.id);
|
|
157
|
+
selected.push(p);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Freeze mode: park pairs for manual review, no LLM required.
|
|
161
|
+
if (config.conflictFreezeEnabled === true) {
|
|
162
|
+
let frozen = 0;
|
|
163
|
+
for (const p of selected) {
|
|
164
|
+
try {
|
|
165
|
+
service.saveConflictPending({ run_id: runId, memory_a: p.a.id, memory_b: p.b.id, reason: `相似度 ${p.similarity.toFixed(2)}` });
|
|
166
|
+
frozen++;
|
|
167
|
+
} catch (error) {
|
|
168
|
+
logger?.warn?.(`dsh-mneme sleep: failed to freeze conflict ${p.a.id}/${p.b.id}: ${String(error)}`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return { status: frozen > 0 ? "ok" : "noop", frozen, pairs: selected.length };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// LLM adjudication.
|
|
175
|
+
const route = resolveSleepRoute(ctx, config, logger);
|
|
176
|
+
if (!route) return { status: "skipped", reason: "no llm route" };
|
|
177
|
+
const snapshot = new Map();
|
|
178
|
+
for (const p of selected) {
|
|
179
|
+
snapshot.set(p.a.id, p.a);
|
|
180
|
+
snapshot.set(p.b.id, p.b);
|
|
181
|
+
}
|
|
182
|
+
const listText = selected.map((p) =>
|
|
183
|
+
`候选冲突:\nid=${p.a.id} | type=${p.a.type} | title=${p.a.title}\n${p.a.content}\n---\nid=${p.b.id} | type=${p.b.type} | title=${p.b.title}\n${p.b.content}\n(相似度 ${p.similarity.toFixed(2)})`
|
|
184
|
+
).join("\n\n");
|
|
185
|
+
const text = await streamText(ctx, {
|
|
186
|
+
provider: route.provider,
|
|
187
|
+
model: route.model,
|
|
188
|
+
purpose: "sleep-conflict",
|
|
189
|
+
maxTokens: 2048,
|
|
190
|
+
messages: [
|
|
191
|
+
{ role: "system", content: [{ type: "text", text: CONFLICT_PROMPT }] },
|
|
192
|
+
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
193
|
+
]
|
|
194
|
+
});
|
|
195
|
+
if (text === undefined) return { status: "failed", error: "llm failed" };
|
|
196
|
+
const decisions = parseJsonArray(text);
|
|
197
|
+
if (!decisions) return { status: "failed", error: "invalid decisions json" };
|
|
198
|
+
// validateDecisions requires every snapshot id to be claimed exactly once.
|
|
199
|
+
// An LLM that omits a pair would otherwise fail the whole phase, so any
|
|
200
|
+
// snapshot id the output leaves uncovered is defaulted to `keep` — the
|
|
201
|
+
// fail-safe reading is "no conflict decided" rather than "conflict phase
|
|
202
|
+
// aborted". validateDecisions stays strict for the dream consolidation path.
|
|
203
|
+
const covered = new Set();
|
|
204
|
+
for (const d of decisions) {
|
|
205
|
+
if (d?.action === "conflict") {
|
|
206
|
+
if (typeof d?.winner === "string") covered.add(d.winner);
|
|
207
|
+
if (typeof d?.loser === "string") covered.add(d.loser);
|
|
208
|
+
} else if (Array.isArray(d?.ids)) {
|
|
209
|
+
for (const id of d.ids) covered.add(id);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
for (const id of snapshot.keys()) {
|
|
213
|
+
if (!covered.has(id)) decisions.push({ action: "keep", ids: [id] });
|
|
214
|
+
}
|
|
215
|
+
const { ok, errors } = validateDecisions(decisions, snapshot, {});
|
|
216
|
+
if (!ok) return { status: "failed", error: `invalid decisions: ${errors.join("; ")}` };
|
|
217
|
+
const { applied, failures, conflicts } = applyDecisions(decisions, service, logger, snapshot, config);
|
|
218
|
+
return {
|
|
219
|
+
status: applied > 0 ? "ok" : failures.length ? "failed" : "noop",
|
|
220
|
+
pairs: selected.length,
|
|
221
|
+
applied,
|
|
222
|
+
failures,
|
|
223
|
+
conflicts
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Phase 2 — archival demotion. No LLM: tiering is time-based, summaries are
|
|
229
|
+
* truncations, and the full body is preserved in _full_content so nothing is
|
|
230
|
+
* lost. Deterministic and cheap, so it runs even with no LLM route.
|
|
231
|
+
*/
|
|
232
|
+
function phaseDemotion(service, config, logger, runId, signal = null) {
|
|
233
|
+
const archiveDays = config.sleepArchiveDays ?? 30;
|
|
234
|
+
const compressDays = config.sleepCompressDays ?? 90;
|
|
235
|
+
const archiveCut = Date.now() - archiveDays * 86400000;
|
|
236
|
+
const compressCut = Date.now() - compressDays * 86400000;
|
|
237
|
+
const demoted = [];
|
|
238
|
+
const archived = [];
|
|
239
|
+
for (const m of service.all()) {
|
|
240
|
+
if (signal?.aborted) break;
|
|
241
|
+
if (m.archived || m.forgotten) continue;
|
|
242
|
+
const ref = m.last_accessed_at ?? m.updated_at ?? m.created_at;
|
|
243
|
+
if (!ref) continue;
|
|
244
|
+
const t = new Date(ref).getTime();
|
|
245
|
+
if (Number.isNaN(t)) continue;
|
|
246
|
+
if (t < compressCut) {
|
|
247
|
+
service.setArchived(m.id, true);
|
|
248
|
+
archived.push(m.id);
|
|
249
|
+
} else if (t < archiveCut) {
|
|
250
|
+
// minRefTimeMs re-checks freshness inside demoteToSummary's transaction:
|
|
251
|
+
// a recall touch landing after this snapshot must not demote the memory.
|
|
252
|
+
service.demoteToSummary(m.id, makeSummary(m), { minRefTimeMs: archiveCut });
|
|
253
|
+
demoted.push(m.id);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return {
|
|
257
|
+
status: demoted.length || archived.length ? "ok" : "noop",
|
|
258
|
+
demoted,
|
|
259
|
+
archived
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Phase 3 — pattern discovery. The LLM scans the most recent memories and
|
|
265
|
+
* mints type=pattern entries (create actions) with evidence references.
|
|
266
|
+
* The empty snapshot is intentional: create claims no existing id, so the
|
|
267
|
+
* "every id claimed" invariant is trivially satisfied for pure-create lists.
|
|
268
|
+
*/
|
|
269
|
+
async function phasePatterns(ctx, service, config, logger, runId, signal = null) {
|
|
270
|
+
const route = resolveSleepRoute(ctx, config, logger);
|
|
271
|
+
if (!route) return { status: "skipped", reason: "no llm route" };
|
|
272
|
+
const limit = config.sleepPatternMinMemories ?? 100;
|
|
273
|
+
const memories = service
|
|
274
|
+
.list({ limit: 200, includeForgotten: false })
|
|
275
|
+
.filter((m) => !m.archived && m.type !== "summary" && m.type !== "pattern")
|
|
276
|
+
.sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1))
|
|
277
|
+
.slice(0, limit);
|
|
278
|
+
if (memories.length === 0) return { status: "skipped", reason: "no memories to scan" };
|
|
279
|
+
if (signal?.aborted) return { status: "aborted", reason: "user activity" };
|
|
280
|
+
const listText = memories
|
|
281
|
+
.map((m) => `id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}`)
|
|
282
|
+
.join("\n");
|
|
283
|
+
const maxPatterns = config.sleepMaxPatternPerRun ?? 3;
|
|
284
|
+
const text = await streamText(ctx, {
|
|
285
|
+
provider: route.provider,
|
|
286
|
+
model: route.model,
|
|
287
|
+
purpose: "sleep-pattern",
|
|
288
|
+
maxTokens: 2048,
|
|
289
|
+
messages: [
|
|
290
|
+
{ role: "system", content: [{ type: "text", text: PATTERN_PROMPT.replace("N", String(maxPatterns)) }] },
|
|
291
|
+
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
292
|
+
]
|
|
293
|
+
});
|
|
294
|
+
if (text === undefined) return { status: "failed", error: "llm failed" };
|
|
295
|
+
const decisions = parseJsonArray(text);
|
|
296
|
+
if (!decisions || decisions.length === 0) return { status: "skipped", reason: "no patterns found" };
|
|
297
|
+
// Evidence ids are provenance refs; an LLM-fabricated id would mint a dead
|
|
298
|
+
// ev:tag pointing nowhere. Intersect evidence with the scanned set so only
|
|
299
|
+
// real memory references survive.
|
|
300
|
+
const scannedIds = new Set(memories.map((m) => m.id));
|
|
301
|
+
for (const d of decisions) {
|
|
302
|
+
if (d?.action === "create" && Array.isArray(d.evidence)) {
|
|
303
|
+
d.evidence = d.evidence.filter((id) => typeof id === "string" && scannedIds.has(id));
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const snapshot = new Map();
|
|
307
|
+
const { ok, errors } = validateDecisions(decisions, snapshot, {
|
|
308
|
+
maxCreatePerRun: maxPatterns
|
|
309
|
+
});
|
|
310
|
+
if (!ok) return { status: "failed", error: `invalid decisions: ${errors.join("; ")}` };
|
|
311
|
+
const { applied, failures, conflicts } = applyDecisions(decisions, service, logger, snapshot, config);
|
|
312
|
+
return {
|
|
313
|
+
status: applied > 0 ? "ok" : "noop",
|
|
314
|
+
scanned: memories.length,
|
|
315
|
+
applied,
|
|
316
|
+
failures,
|
|
317
|
+
conflicts
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* Phase 4 — entity relation completion. Detects orphan entities (zero
|
|
323
|
+
* relations) and completes implied relations from memory co-occurrence:
|
|
324
|
+
* entities named in the same memory → related_to; container kinds
|
|
325
|
+
* (project/module) → part_of; tech-ish pairs → depends_on. Deterministic,
|
|
326
|
+
* no LLM — cheap, so it runs even without a route. saveRelation is
|
|
327
|
+
* bookkeeping (no write hook), so it never re-triggers the scheduler.
|
|
328
|
+
*/
|
|
329
|
+
function inferRelationType(a, b) {
|
|
330
|
+
if ((a.type === "project" || a.type === "module") && a.type !== b.type) return "part_of";
|
|
331
|
+
if ((b.type === "project" || b.type === "module") && b.type !== a.type) return "part_of";
|
|
332
|
+
if (/npm|plugin|api|sdk|lib|framework|package|deps?|build/i.test(`${a.name} ${b.name}`)) return "depends_on";
|
|
333
|
+
return "related_to";
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function phaseRelations(service, config, logger, runId, signal = null) {
|
|
337
|
+
const entities = service.listEntities({ limit: 1000 }) ?? [];
|
|
338
|
+
if (entities.length < 2) return { status: "skipped", reason: "too few entities" };
|
|
339
|
+
const orphans = entities.filter((e) => (service.getRelations(e.id) ?? []).length === 0);
|
|
340
|
+
if (orphans.length === 0) return { status: "skipped", reason: "no orphan entities" };
|
|
341
|
+
const memories = service.all().filter((m) => !m.archived && !m.forgotten);
|
|
342
|
+
const seen = new Set();
|
|
343
|
+
const related = [];
|
|
344
|
+
const MAX_RELATIONS_PER_ORPHAN = 3;
|
|
345
|
+
for (const o of orphans) {
|
|
346
|
+
if (signal?.aborted) break;
|
|
347
|
+
let made = 0;
|
|
348
|
+
for (const m of memories) {
|
|
349
|
+
if (signal?.aborted || made >= MAX_RELATIONS_PER_ORPHAN) break;
|
|
350
|
+
const text = `${m.title ?? ""} ${m.content ?? ""}`;
|
|
351
|
+
if (!text.includes(o.name)) continue;
|
|
352
|
+
for (const other of entities) {
|
|
353
|
+
if (other.id === o.id || other.name === o.name) continue;
|
|
354
|
+
const key = [o.id, other.id].sort().join("|");
|
|
355
|
+
if (seen.has(key)) continue;
|
|
356
|
+
if (!text.includes(other.name)) continue;
|
|
357
|
+
const relationType = inferRelationType(o, other);
|
|
358
|
+
try {
|
|
359
|
+
service.saveRelation({ from_entity: o.id, to_entity: other.id, relation_type: relationType, memory_id: m.id, metadata: { source: "sleep_relation_completion" } });
|
|
360
|
+
seen.add(key);
|
|
361
|
+
related.push({ from: o.id, to: other.id, type: relationType });
|
|
362
|
+
made++;
|
|
363
|
+
} catch (error) {
|
|
364
|
+
logger?.warn?.(`dsh-mneme sleep: relation ${o.id}/${other.id} failed: ${String(error)}`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
return {
|
|
370
|
+
status: related.length > 0 ? "ok" : "noop",
|
|
371
|
+
orphanCount: orphans.length,
|
|
372
|
+
related
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// ---------------------------------------------------------------- run
|
|
377
|
+
|
|
378
|
+
function deriveStatus(phases) {
|
|
379
|
+
const list = Object.values(phases);
|
|
380
|
+
if (list.length === 0) return "noop";
|
|
381
|
+
const anyError = list.some((p) => p.status === "failed" || p.status === "error");
|
|
382
|
+
const anyWork = list.some((p) => p.status === "ok");
|
|
383
|
+
if (anyWork && anyError) return "degraded";
|
|
384
|
+
if (anyError) return "failed";
|
|
385
|
+
if (anyWork) return "ok";
|
|
386
|
+
return "noop";
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* Run one full sleep cycle. Best-effort across all phases; writes a
|
|
391
|
+
* run_type='sleep' audit row (same dream_runs table) so sleep activity is
|
|
392
|
+
* observable alongside consolidation runs.
|
|
393
|
+
*/
|
|
394
|
+
export async function runSleep(ctx, service, config, logger, semantic = null, signal = null) {
|
|
395
|
+
const runId = randomUUID();
|
|
396
|
+
const phases = {};
|
|
397
|
+
const attempt = async (name, fn) => {
|
|
398
|
+
if (signal?.aborted) return; // user resumed activity — stop before next phase
|
|
399
|
+
try {
|
|
400
|
+
phases[name] = await fn();
|
|
401
|
+
} catch (error) {
|
|
402
|
+
phases[name] = { status: "failed", error: error?.message ?? String(error) };
|
|
403
|
+
logger?.warn?.(`dsh-mneme sleep: ${name} phase failed: ${error?.message ?? error}`);
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
await attempt("conflicts", () => phaseConflicts(ctx, service, config, logger, runId, semantic, signal));
|
|
407
|
+
await attempt("demotion", () => phaseDemotion(service, config, logger, runId, signal));
|
|
408
|
+
await attempt("patterns", () => phasePatterns(ctx, service, config, logger, runId, signal));
|
|
409
|
+
await attempt("relations", () => phaseRelations(service, config, logger, runId, signal));
|
|
410
|
+
|
|
411
|
+
const status = deriveStatus(phases);
|
|
412
|
+
const route = resolveSleepRoute(ctx, config, logger);
|
|
413
|
+
const totalApplied = Object.values(phases).reduce((n, p) => n + (Number.isInteger(p?.applied) ? p.applied : 0), 0);
|
|
414
|
+
const snapshotHash = createHash("sha256").update(JSON.stringify(phases)).digest("hex");
|
|
415
|
+
const receipt = buildReceipt({
|
|
416
|
+
runId,
|
|
417
|
+
status,
|
|
418
|
+
snapshotHash,
|
|
419
|
+
inputCount: 0,
|
|
420
|
+
applied: totalApplied,
|
|
421
|
+
summaryStored: false
|
|
422
|
+
});
|
|
423
|
+
try {
|
|
424
|
+
service.saveDreamRun({
|
|
425
|
+
id: runId,
|
|
426
|
+
status,
|
|
427
|
+
provider: route?.provider,
|
|
428
|
+
model: route?.model,
|
|
429
|
+
snapshot_hash: snapshotHash,
|
|
430
|
+
input_count: 0,
|
|
431
|
+
input: null,
|
|
432
|
+
decisions: phases,
|
|
433
|
+
outcome: phases,
|
|
434
|
+
applied: totalApplied,
|
|
435
|
+
summary_stored: false,
|
|
436
|
+
receipt,
|
|
437
|
+
policy_epoch: config.policyEpoch ?? 0,
|
|
438
|
+
run_type: "sleep"
|
|
439
|
+
});
|
|
440
|
+
} catch (error) {
|
|
441
|
+
logger?.warn?.(`dsh-mneme sleep: failed to record audit run: ${String(error)}`);
|
|
442
|
+
}
|
|
443
|
+
return { ok: status === "ok" || status === "degraded", status, runId, phases, receipt };
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// ---------------------------------------------------------------- scheduler
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* Idle-triggered scheduler. DSH plugins have no resident cron, so a sleep run
|
|
450
|
+
* fires when: sleep is enabled, the store has been quiet for sleepIdleMinutes,
|
|
451
|
+
* and the previous run is older than sleepMinIntervalHours. noteWrite() is
|
|
452
|
+
* called on every store write and (re)arms an idle timer that re-checks at the
|
|
453
|
+
* exact moment the idle window elapses — no polling, no cron.
|
|
454
|
+
*
|
|
455
|
+
* A `now` clock can be injected for tests; it defaults to Date.now.
|
|
456
|
+
*/
|
|
457
|
+
export function createSleepScheduler({
|
|
458
|
+
service,
|
|
459
|
+
config,
|
|
460
|
+
logger,
|
|
461
|
+
onRun = null,
|
|
462
|
+
now = () => Date.now(),
|
|
463
|
+
setTimeoutFn = setTimeout,
|
|
464
|
+
clearTimeoutFn = clearTimeout
|
|
465
|
+
}) {
|
|
466
|
+
let lastWriteAt = now();
|
|
467
|
+
let lastRunAt = 0;
|
|
468
|
+
let running = false;
|
|
469
|
+
let disposed = false;
|
|
470
|
+
let idleTimer = null;
|
|
471
|
+
let sleepAbort = null;
|
|
472
|
+
|
|
473
|
+
function armIdleTimer() {
|
|
474
|
+
if (disposed || idleTimer) return;
|
|
475
|
+
if (config.sleepModeEnabled !== true) return;
|
|
476
|
+
const idleMs = (config.sleepIdleMinutes ?? 5) * 60000;
|
|
477
|
+
const delay = Math.max(0, idleMs - (now() - lastWriteAt)) + 1000;
|
|
478
|
+
idleTimer = setTimeoutFn(async () => {
|
|
479
|
+
idleTimer = null;
|
|
480
|
+
await maybeSchedule();
|
|
481
|
+
}, delay);
|
|
482
|
+
idleTimer.unref?.();
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function shouldRun(at = now()) {
|
|
486
|
+
if (disposed || running) return false;
|
|
487
|
+
if (config.sleepModeEnabled !== true) return false;
|
|
488
|
+
if (at - lastWriteAt < (config.sleepIdleMinutes ?? 5) * 60000) return false;
|
|
489
|
+
// lastRunAt === 0 means never ran — the min-interval check must not block
|
|
490
|
+
// the very first cycle (a real run stamps a nonzero timestamp).
|
|
491
|
+
if (lastRunAt > 0 && at - lastRunAt < (config.sleepMinIntervalHours ?? 8) * 3600000) return false;
|
|
492
|
+
return true;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** Called on writes: resets the idle clock and re-arms the fire timer. The
|
|
496
|
+
* pending timer is cleared first — a stale timer armed against the old idle
|
|
497
|
+
* window would otherwise fire early, fail shouldRun, and leave nothing armed
|
|
498
|
+
* for the next window (a missed trigger until the next write).
|
|
499
|
+
*
|
|
500
|
+
* While a sleep run is executing (running=true) the in-flight AbortController
|
|
501
|
+
* is NOT aborted: the run's own writes (demoteToSummary / setArchived ride
|
|
502
|
+
* the normal write-hook path) would otherwise self-abort the cycle. External
|
|
503
|
+
* activity during the run still resets the idle clock here, so no new cycle
|
|
504
|
+
* fires until the store is quiet again. */
|
|
505
|
+
function noteWrite() {
|
|
506
|
+
lastWriteAt = now();
|
|
507
|
+
if (!running && sleepAbort) {
|
|
508
|
+
sleepAbort.abort(); // user resumed activity — interrupt an idle run
|
|
509
|
+
}
|
|
510
|
+
if (idleTimer) {
|
|
511
|
+
clearTimeoutFn(idleTimer);
|
|
512
|
+
idleTimer = null;
|
|
513
|
+
}
|
|
514
|
+
armIdleTimer();
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
async function maybeSchedule() {
|
|
518
|
+
if (!shouldRun()) return false;
|
|
519
|
+
running = true;
|
|
520
|
+
const abort = new AbortController();
|
|
521
|
+
sleepAbort = abort;
|
|
522
|
+
try {
|
|
523
|
+
lastRunAt = now();
|
|
524
|
+
const result = await service.enqueue(() =>
|
|
525
|
+
onRun ? onRun(abort.signal) : Promise.resolve({ ok: true, skipped: true })
|
|
526
|
+
);
|
|
527
|
+
return !!(result && result.ok);
|
|
528
|
+
} catch (error) {
|
|
529
|
+
logger?.warn?.(`dsh-mneme sleep: run failed: ${error?.message ?? error}`);
|
|
530
|
+
return false;
|
|
531
|
+
} finally {
|
|
532
|
+
sleepAbort = null;
|
|
533
|
+
running = false;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
async function dispose() {
|
|
538
|
+
disposed = true;
|
|
539
|
+
if (idleTimer) {
|
|
540
|
+
clearTimeoutFn(idleTimer);
|
|
541
|
+
idleTimer = null;
|
|
542
|
+
}
|
|
543
|
+
if (sleepAbort) {
|
|
544
|
+
sleepAbort.abort();
|
|
545
|
+
sleepAbort = null;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
return { noteWrite, maybeSchedule, shouldRun, dispose };
|
|
550
|
+
}
|