@modusensus/dsh-mneme 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +419 -397
- package/lib/api.js +516 -516
- package/lib/commands.js +64 -64
- package/lib/config.js +252 -252
- package/lib/dream.js +817 -817
- package/lib/embedding.js +154 -154
- package/lib/hot-memory.js +7 -0
- package/lib/index.js +341 -341
- package/lib/inject.js +208 -208
- package/lib/local-embedder.js +282 -282
- package/lib/reranker.js +218 -215
- package/lib/service.js +1489 -1484
- package/lib/store.js +6 -2
- package/lib/summarize.js +5 -1
- package/lib/tools.js +7 -2
- package/package.json +1 -1
- package/scripts/sync-lib.js +7 -2
- package/src/hot-memory.js +53 -46
- package/src/reranker.js +218 -215
- package/src/service.js +1489 -1484
- package/src/store.js +6 -2
- package/src/summarize.js +5 -1
- package/src/tools.js +7 -2
- package/test/hot-memory.test.js +174 -145
- package/test/provenance.test.js +103 -0
- package/test/reranker.test.js +240 -197
- package/test/service-search.test.js +199 -174
package/lib/dream.js
CHANGED
|
@@ -1,817 +1,817 @@
|
|
|
1
|
-
import { validateDecisions, applyDecisions } from "./dream/decisions.js";
|
|
2
|
-
import { clusterMemories, findPotentialConflicts } from "./dream/clustering.js";
|
|
3
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
-
export { validateDecisions, applyDecisions };
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
// Extract the first JSON array from LLM output, tolerating markdown fences,
|
|
8
|
-
// leading/trailing prose, and common wrapper noise. Returns an array or null.
|
|
9
|
-
function extractJsonArray(text) {
|
|
10
|
-
if (typeof text !== "string" || text.trim().length === 0) return null;
|
|
11
|
-
|
|
12
|
-
// 1. Strip markdown code fences (```json ... ``` or ``` ... ```).
|
|
13
|
-
let cleaned = text.replace(/```(?:json)?\s*([\s\S]*?)```/gi, "$1");
|
|
14
|
-
cleaned = cleaned.trim();
|
|
15
|
-
|
|
16
|
-
// 2. Find the first '[' and the matching last ']' that yields valid JSON.
|
|
17
|
-
const start = cleaned.indexOf("[");
|
|
18
|
-
if (start === -1) return null;
|
|
19
|
-
for (let end = cleaned.lastIndexOf("]"); end > start; end = cleaned.lastIndexOf("]", end - 1)) {
|
|
20
|
-
const candidate = cleaned.slice(start, end + 1);
|
|
21
|
-
try {
|
|
22
|
-
return JSON.parse(candidate);
|
|
23
|
-
} catch {
|
|
24
|
-
// Light repair: remove trailing commas before ] or }.
|
|
25
|
-
try {
|
|
26
|
-
const repaired = candidate.replace(/,(\s*[}\]])/g, "$1");
|
|
27
|
-
return JSON.parse(repaired);
|
|
28
|
-
} catch {
|
|
29
|
-
// keep searching backwards
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
// 3. Fallback: a broader regex extraction.
|
|
35
|
-
try {
|
|
36
|
-
const match = cleaned.match(/\[[\s\S]*\]/);
|
|
37
|
-
if (match) return JSON.parse(match[0]);
|
|
38
|
-
} catch {
|
|
39
|
-
// fall through
|
|
40
|
-
}
|
|
41
|
-
return null;
|
|
42
|
-
}
|
|
43
|
-
const SUMMARY_PROMPT = `你是记忆库摘要助手。根据整理后的记忆,生成一段 150-200 字的记忆库总览,覆盖:用户偏好、活跃项目、关键决策。之后作为会话上下文注入。只输出摘要文本,不要其他内容。`;
|
|
44
|
-
|
|
45
|
-
const CONSOLIDATION_PROMPT = `你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。
|
|
46
|
-
请执行记忆巩固(consolidation),输出一个决策 JSON 数组。
|
|
47
|
-
|
|
48
|
-
【决策格式(必须严格遵守)】
|
|
49
|
-
每个决策必须是对象,字段固定:
|
|
50
|
-
- "action":必填。取值只能是 "keep" / "merge" / "archive" / "update" / "conflict" 之一(字段名必须是 action,严禁写成 type)
|
|
51
|
-
- "ids":必填,数组,本决策涉及的记忆 id 列表
|
|
52
|
-
- "reason":可选,字符串,决策理由
|
|
53
|
-
- "importance":可选,整数 1-5
|
|
54
|
-
- merge 额外字段:"keepSource"(单个 id 字符串,必须是 ids 之一)+ 合并后的 "title"、"content"
|
|
55
|
-
- conflict 额外字段:"winner" 与 "loser",都是【单个 id 字符串,不是数组】
|
|
56
|
-
- update 额外字段:修正后的 "title" 和/或 "content";"ids" 只能包含一个 id
|
|
57
|
-
|
|
58
|
-
【决策 JSON 示例】
|
|
59
|
-
[
|
|
60
|
-
{ "action": "merge", "ids": ["m1", "m2"], "keepSource": "m1", "title": "合并标题", "content": "合并后的摘要内容", "importance": 4, "reason": "主题相近" },
|
|
61
|
-
{ "action": "conflict", "winner": "m3", "loser": "m4", "reason": "内容矛盾,保留更新的信息" },
|
|
62
|
-
{ "action": "update", "ids": ["m5"], "content": "修正后的内容", "reason": "信息过时" },
|
|
63
|
-
{ "action": "archive", "ids": ["m6"], "reason": "重复或过时" }
|
|
64
|
-
]
|
|
65
|
-
|
|
66
|
-
【任务】
|
|
67
|
-
1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)
|
|
68
|
-
2. 识别重复/过时信息 → archive
|
|
69
|
-
3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)
|
|
70
|
-
4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)
|
|
71
|
-
- update 的 ids 只能包含一个 id
|
|
72
|
-
- 必须提供修正后的 title 和/或 content
|
|
73
|
-
- 仅当内容确实需要修正时才使用,不要滥用
|
|
74
|
-
- 每次整理最多输出 2 个 update
|
|
75
|
-
- 24 小时内新建的记忆不可 update
|
|
76
|
-
5. 无问题的条目无需输出(未提及的条目将自动保留 keep)
|
|
77
|
-
|
|
78
|
-
【硬性规则】
|
|
79
|
-
- 字段名必须精确为 "action",严禁写成 "type";字段名统一用双引号
|
|
80
|
-
- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】
|
|
81
|
-
- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)
|
|
82
|
-
- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep
|
|
83
|
-
- merge 的 keepSource 必须是 ids 之一
|
|
84
|
-
- 仅合并同类型条目(type 相同)
|
|
85
|
-
- 不要编造 ids;只使用提供的 id
|
|
86
|
-
- 重要性 1-5,合并后取最高
|
|
87
|
-
- 只输出 JSON 数组,不要其他文字`;
|
|
88
|
-
|
|
89
|
-
function totalChars(memories) {
|
|
90
|
-
return memories.reduce((sum, m) => sum + (m.title?.length ?? 0) + (m.content?.length ?? 0), 0);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// ---------------------------------------------------------------- audit
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* Canonical digest of the consolidation input snapshot. Built from stable
|
|
97
|
-
* fields sorted by id, so identical inputs always yield the same hash — the
|
|
98
|
-
* basis for replaying/verifying a recorded decision (receipt check).
|
|
99
|
-
*/
|
|
100
|
-
export function hashSnapshot(memories) {
|
|
101
|
-
const canon = memories
|
|
102
|
-
.map((m) => [m.id, m.type, m.title, m.content, m.importance, m.updated_at])
|
|
103
|
-
.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
|
|
104
|
-
.map((parts) => parts.map((p) => String(p ?? "")).join("\u0001"))
|
|
105
|
-
.join("\u0002");
|
|
106
|
-
return createHash("sha256").update(canon).digest("hex");
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* Compact machine-verifiable receipt for one autoDream run. Format:
|
|
111
|
-
* dsh-mneme:run:<runId>:<status>:<snapshotHash(12)>:<inputCount>:<applied>:<summaryFlag>
|
|
112
|
-
* Enough to correlate a run with its persisted audit row and to spot silent
|
|
113
|
-
* drift (same snapshot hash + same decisions must reproduce the same outcome).
|
|
114
|
-
*/
|
|
115
|
-
export function buildReceipt({ runId, status, snapshotHash, inputCount, applied, summaryStored }) {
|
|
116
|
-
return `dsh-mneme:run:${runId}:${status}:${snapshotHash.slice(0, 12)}:${inputCount}:${applied}:${summaryStored ? 1 : 0}`;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
/**
|
|
120
|
-
* Parse a receipt back into fields; returns undefined for malformed input.
|
|
121
|
-
*/
|
|
122
|
-
export function parseReceipt(receipt) {
|
|
123
|
-
if (typeof receipt !== "string") return undefined;
|
|
124
|
-
const parts = receipt.split(":");
|
|
125
|
-
if (parts.length !== 8 || parts[0] !== "dsh-mneme" || parts[1] !== "run") return undefined;
|
|
126
|
-
const [, , runId, status, snapshotHash, inputCount, applied, summaryStored] = parts;
|
|
127
|
-
// reconcile = decisions validated but one or more did not commit (CAS
|
|
128
|
-
// conflict / transaction rollback) — the store diverges from the decision
|
|
129
|
-
// list and the run must be reconciled, never reported as a fake ok.
|
|
130
|
-
if (!runId || !/^(ok|noop|degraded|reconcile|failed)$/.test(status)) return undefined;
|
|
131
|
-
const count = Number(inputCount);
|
|
132
|
-
const appliedN = Number(applied);
|
|
133
|
-
if (!Number.isInteger(count) || !Number.isInteger(appliedN)) return undefined;
|
|
134
|
-
return { runId, status, snapshotHash, inputCount: count, applied: appliedN, summaryStored: summaryStored === "1" };
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Derive the per-id disposition (keep / merge-keep / merge-archived /
|
|
139
|
-
* archived / conflict-winner / conflict-archived) from a validated decision
|
|
140
|
-
* list. Stored in the audit row so a run can be replayed without re-running
|
|
141
|
-
* the LLM.
|
|
142
|
-
*/
|
|
143
|
-
export function buildOutcome(decisions) {
|
|
144
|
-
const byId = {};
|
|
145
|
-
for (const d of decisions ?? []) {
|
|
146
|
-
if (d.action === "keep") {
|
|
147
|
-
for (const id of d.ids) byId[id] = "keep";
|
|
148
|
-
} else if (d.action === "archive") {
|
|
149
|
-
for (const id of d.ids) byId[id] = "archived";
|
|
150
|
-
} else if (d.action === "merge") {
|
|
151
|
-
for (const id of d.ids) byId[id] = id === d.keepSource ? "merge-keep" : "merge-archived";
|
|
152
|
-
} else if (d.action === "conflict") {
|
|
153
|
-
byId[d.winner] = "conflict-winner";
|
|
154
|
-
byId[d.loser] = "conflict-archived";
|
|
155
|
-
} else if (d.action === "update") {
|
|
156
|
-
for (const id of d.ids) byId[id] = "updated";
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
return { byId };
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/**
|
|
163
|
-
* Content-addressed digest of the memories a verdict was decided against
|
|
164
|
-
* (id + title + content + importance), sorted by id so identical inputs always
|
|
165
|
-
* hash the same. This is the per-record "判定依据" fingerprint: a receipt whose
|
|
166
|
-
* digest cannot be reproduced from the involved memories is a bare claim, and a
|
|
167
|
-
* digest match with a divergent outcome pinpoints drift to the exact record.
|
|
168
|
-
*/
|
|
169
|
-
export function hashDecisionInput(memories) {
|
|
170
|
-
const canon = (memories ?? [])
|
|
171
|
-
.map((m) => [m.id, m.title, m.content, m.importance])
|
|
172
|
-
.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
|
|
173
|
-
.map((p) => p.map((x) => String(x ?? "")).join(""))
|
|
174
|
-
.join("");
|
|
175
|
-
return createHash("sha256").update(canon).digest("hex");
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/**
|
|
179
|
-
* Build the per-record receipts for a run's actually-committed mutable verdicts
|
|
180
|
-
* (merge/conflict/update) — one row per verdict in the receipt_chain. Inputs
|
|
181
|
-
* are drawn from the run snapshot (what the LLM actually arbitrated against),
|
|
182
|
-
* and the idempotency counters count_before → count_after come from the
|
|
183
|
-
* committed sub-step, so replaying the same decision must reproduce the same
|
|
184
|
-
* numbers. verdict starts "live"; a later policy_epoch upgrade will batch-mark
|
|
185
|
-
* older verdicts "historical" (a receipt_chain rewrite driven by the store's
|
|
186
|
-
* getLatestPolicyEpoch — out of scope for this pass), while "revoked" is
|
|
187
|
-
* reserved for verdicts later overturned by an explicit human decision.
|
|
188
|
-
*/
|
|
189
|
-
function buildRecordReceipts({ runId, committed, snapshot, policyEpoch }) {
|
|
190
|
-
const at = (id) => snapshot?.get?.(id);
|
|
191
|
-
const receipts = [];
|
|
192
|
-
for (const c of committed ?? []) {
|
|
193
|
-
const base = {
|
|
194
|
-
run_id: runId,
|
|
195
|
-
verdict: "live",
|
|
196
|
-
count_before: c.count_before,
|
|
197
|
-
count_after: c.count_after,
|
|
198
|
-
policy_epoch: policyEpoch,
|
|
199
|
-
created_at: new Date().toISOString()
|
|
200
|
-
};
|
|
201
|
-
if (c.action === "merge") {
|
|
202
|
-
receipts.push({
|
|
203
|
-
...base,
|
|
204
|
-
receipt_id: randomUUID(),
|
|
205
|
-
record_id: c.keepSource,
|
|
206
|
-
kind: "merge",
|
|
207
|
-
input_digest: hashDecisionInput((c.ids ?? []).map(at).filter(Boolean)),
|
|
208
|
-
keep_source: c.keepSource,
|
|
209
|
-
sources: c.ids
|
|
210
|
-
});
|
|
211
|
-
} else if (c.action === "conflict") {
|
|
212
|
-
receipts.push({
|
|
213
|
-
...base,
|
|
214
|
-
receipt_id: randomUUID(),
|
|
215
|
-
record_id: c.winner,
|
|
216
|
-
kind: "conflict",
|
|
217
|
-
input_digest: hashDecisionInput([at(c.winner), at(c.loser)].filter(Boolean)),
|
|
218
|
-
winner_id: c.winner,
|
|
219
|
-
loser_id: c.loser
|
|
220
|
-
});
|
|
221
|
-
} else if (c.action === "update") {
|
|
222
|
-
receipts.push({
|
|
223
|
-
...base,
|
|
224
|
-
receipt_id: randomUUID(),
|
|
225
|
-
record_id: c.ids[0],
|
|
226
|
-
kind: "update",
|
|
227
|
-
input_digest: hashDecisionInput([at(c.ids[0])].filter(Boolean))
|
|
228
|
-
});
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
return receipts;
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/**
|
|
235
|
-
* Consume an LLM stream and return the accumulated text. Direct text-delta
|
|
236
|
-
* accumulation covers both the real protocol ({type:"text-delta", index, text})
|
|
237
|
-
* and looser test doubles ({type:"text-delta", text}); a terminal error/abort
|
|
238
|
-
* surfaces as undefined. The caller decides how to treat an empty result.
|
|
239
|
-
* `onUsage` (optional, Bug8) receives any usage chunk for token accounting.
|
|
240
|
-
*/
|
|
241
|
-
async function streamText(ctx, options, onUsage) {
|
|
242
|
-
let text = "";
|
|
243
|
-
for await (const chunk of ctx.llm.stream(options)) {
|
|
244
|
-
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
245
|
-
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk);
|
|
246
|
-
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
247
|
-
return undefined;
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return text;
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
/**
|
|
254
|
-
* Bug8: wrap a background LLM call so its token/time/status are recorded in the
|
|
255
|
-
* llm_audit_logs table. Best-effort bookkeeping: a failure to WRITE the audit
|
|
256
|
-
* row is swallowed (never blocks the LLM call), while a failure of the call
|
|
257
|
-
* itself is captured as status='error' and re-thrown so the caller keeps its
|
|
258
|
-
* existing error path. `spec` carries the static metadata (trigger_source,
|
|
259
|
-
* operation_type, model_id, related_memory_ids); `body(reportUsage)` performs
|
|
260
|
-
* the actual stream consumption and is handed a usage reporter for the chunks.
|
|
261
|
-
*/
|
|
262
|
-
async function runAuditedLlm(ctx, service, config, spec, body) {
|
|
263
|
-
const audit = config?.llmAudit;
|
|
264
|
-
if (audit?.enabled === false || typeof service?.saveLlmAudit !== "function") return body(() => {});
|
|
265
|
-
const startedAt = Date.now();
|
|
266
|
-
const timestamp = new Date(startedAt).toISOString();
|
|
267
|
-
let inputTokens = 0;
|
|
268
|
-
let outputTokens = 0;
|
|
269
|
-
let status = "success";
|
|
270
|
-
let errorMessage = null;
|
|
271
|
-
let result;
|
|
272
|
-
try {
|
|
273
|
-
result = await body((usage) => {
|
|
274
|
-
if (!usage) return;
|
|
275
|
-
const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
|
|
276
|
-
const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
|
|
277
|
-
if (Number.isFinite(i)) inputTokens = i;
|
|
278
|
-
if (Number.isFinite(o)) outputTokens = o;
|
|
279
|
-
});
|
|
280
|
-
if (result === undefined) {
|
|
281
|
-
// stream aborted/errored: the caller treats undefined as a failed run;
|
|
282
|
-
// record it as error here so the audit shows the truth.
|
|
283
|
-
status = "error";
|
|
284
|
-
errorMessage = errorMessage ?? "llm stream aborted or errored";
|
|
285
|
-
}
|
|
286
|
-
return result;
|
|
287
|
-
} catch (error) {
|
|
288
|
-
status = "error";
|
|
289
|
-
errorMessage = String(error?.message ?? error);
|
|
290
|
-
throw error;
|
|
291
|
-
} finally {
|
|
292
|
-
try {
|
|
293
|
-
service.saveLlmAudit({
|
|
294
|
-
timestamp,
|
|
295
|
-
trigger_source: spec.triggerSource,
|
|
296
|
-
operation_type: spec.operationType,
|
|
297
|
-
model_id: spec.modelId,
|
|
298
|
-
input_tokens: inputTokens,
|
|
299
|
-
output_tokens: outputTokens,
|
|
300
|
-
total_tokens: inputTokens + outputTokens,
|
|
301
|
-
cost_usd: 0,
|
|
302
|
-
duration_ms: Date.now() - startedAt,
|
|
303
|
-
status,
|
|
304
|
-
error_message: errorMessage,
|
|
305
|
-
related_memory_ids: spec.relatedMemoryIds ?? []
|
|
306
|
-
});
|
|
307
|
-
} catch (auditError) {
|
|
308
|
-
ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
/**
|
|
314
|
-
* Resolve the LLM route: agent default model (deployment) first, plugin config
|
|
315
|
-
* (dreamProvider/dreamModel) as fallback. Falls through to undefined when no
|
|
316
|
-
* route exists — runDream then fails safe. Fallback is logged so a silent
|
|
317
|
-
* route switch is observable.
|
|
318
|
-
*/
|
|
319
|
-
function resolveRoute(ctx, config, logger) {
|
|
320
|
-
try {
|
|
321
|
-
const sel = ctx.agentDefaultModel?.currentSelection?.();
|
|
322
|
-
if (sel?.provider && sel?.model) return { provider: sel.provider, model: sel.model };
|
|
323
|
-
logger?.warn?.("dsh-mneme dream: agentDefaultModel unavailable, falling back to config route");
|
|
324
|
-
} catch (error) {
|
|
325
|
-
logger?.warn?.(`dsh-mneme dream: agentDefaultModel lookup failed, falling back to config route: ${String(error)}`);
|
|
326
|
-
}
|
|
327
|
-
if (config.dreamProvider && config.dreamModel) return { provider: config.dreamProvider, model: config.dreamModel };
|
|
328
|
-
return undefined;
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
// ------------------------------------------------------- semantic enhancement
|
|
332
|
-
// Best-effort: any failure here degrades to plain consolidation. The dream
|
|
333
|
-
// path must never be broken by an unavailable embedder/index.
|
|
334
|
-
|
|
335
|
-
/** Backfill + return vectors for every memory; null when impossible. */
|
|
336
|
-
async function collectVectors(memories, semantic) {
|
|
337
|
-
const { embedder, vectorIndex } = semantic;
|
|
338
|
-
if (!embedder || !vectorIndex || typeof embedder.embedSingle !== "function") return null;
|
|
339
|
-
const vectors = new Array(memories.length);
|
|
340
|
-
const missing = [];
|
|
341
|
-
for (let i = 0; i < memories.length; i++) {
|
|
342
|
-
const cached = vectorIndex.getEmbedding?.(memories[i].id);
|
|
343
|
-
if (cached) vectors[i] = cached;
|
|
344
|
-
else missing.push(i);
|
|
345
|
-
}
|
|
346
|
-
if (missing.length) {
|
|
347
|
-
const texts = missing.map((i) => [memories[i].title, memories[i].content].filter(Boolean).join("\n"));
|
|
348
|
-
const rows = await embedder.embed(texts);
|
|
349
|
-
missing.forEach((mi, j) => {
|
|
350
|
-
if (rows[j]?.length) {
|
|
351
|
-
vectors[mi] = rows[j];
|
|
352
|
-
vectorIndex.saveEmbedding(memories[mi].id, rows[j]);
|
|
353
|
-
}
|
|
354
|
-
});
|
|
355
|
-
}
|
|
356
|
-
return vectors.some((v) => !v) ? null : vectors;
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
/**
|
|
360
|
-
* Rebuild the vector index after dream decisions so the store and the index
|
|
361
|
-
* stay in sync: merged-away/archived/conflict-loser rows lose their vectors,
|
|
362
|
-
* the merge keeper gets a fresh one.
|
|
363
|
-
*/
|
|
364
|
-
async function maintainIndexAfterDream(decisions, service, semantic) {
|
|
365
|
-
const { embedder, vectorIndex } = semantic;
|
|
366
|
-
if (!embedder || !vectorIndex || typeof embedder.embedSingle !== "function") return;
|
|
367
|
-
const rebuild = new Map();
|
|
368
|
-
for (const d of decisions ?? []) {
|
|
369
|
-
if (d.action === "merge") {
|
|
370
|
-
for (const id of d.ids ?? []) {
|
|
371
|
-
if (id !== d.keepSource) vectorIndex.deleteEmbedding(id);
|
|
372
|
-
}
|
|
373
|
-
if (d.keepSource) {
|
|
374
|
-
const keeper = service.getById(d.keepSource);
|
|
375
|
-
if (keeper) rebuild.set(keeper.id, [keeper.title, keeper.content].filter(Boolean).join("\n"));
|
|
376
|
-
}
|
|
377
|
-
} else if (d.action === "archive" || d.action === "conflict") {
|
|
378
|
-
for (const id of d.ids ?? [d.loser]) vectorIndex.deleteEmbedding(id);
|
|
379
|
-
} else if (d.action === "update") {
|
|
380
|
-
const id = d.ids[0];
|
|
381
|
-
const mem = service.getById(id);
|
|
382
|
-
if (mem) {
|
|
383
|
-
vectorIndex.deleteEmbedding(id);
|
|
384
|
-
try {
|
|
385
|
-
const text = [mem.title, mem.content].filter(Boolean).join("\n");
|
|
386
|
-
const v = await embedder.embedSingle(text);
|
|
387
|
-
if (v?.length) vectorIndex.saveEmbedding(id, v);
|
|
388
|
-
} catch { /* best-effort */ }
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
for (const [id, text] of rebuild) {
|
|
393
|
-
try {
|
|
394
|
-
const v = await embedder.embedSingle(text);
|
|
395
|
-
if (v?.length) vectorIndex.saveEmbedding(id, v);
|
|
396
|
-
} catch { /* best-effort */ }
|
|
397
|
-
}
|
|
398
|
-
if (embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChars = 5000, delayMs = 2000, logger, semantic = null }) {
|
|
402
|
-
let pendingTimer = null;
|
|
403
|
-
let running = false;
|
|
404
|
-
let disposed = false;
|
|
405
|
-
let baseline = { count: 0, chars: 0 };
|
|
406
|
-
let inFlight = null;
|
|
407
|
-
|
|
408
|
-
function shouldTrigger(service) {
|
|
409
|
-
const memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
410
|
-
const count = memories.length;
|
|
411
|
-
const chars = totalChars(memories);
|
|
412
|
-
const overBase = count >= baseline.count + thresholdCount || chars >= baseline.chars + thresholdChars;
|
|
413
|
-
const overAbs = count >= thresholdCount || chars >= thresholdChars;
|
|
414
|
-
return { trigger: overAbs && overBase, count, chars };
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
function maybeSchedule(service) {
|
|
418
|
-
if (disposed || running || pendingTimer) return false;
|
|
419
|
-
const { trigger, count, chars } = shouldTrigger(service);
|
|
420
|
-
if (!trigger) return false;
|
|
421
|
-
pendingTimer = setTimeout(() => {
|
|
422
|
-
pendingTimer = null;
|
|
423
|
-
running = true;
|
|
424
|
-
// Defer the onRun invocation so a synchronous throw cannot escape the
|
|
425
|
-
// timer callback (which would crash the process) and skip the teardown.
|
|
426
|
-
// Errors are logged, never swallowed silently. inFlight lets dispose()
|
|
427
|
-
// await the running consolidation before the caller closes the store.
|
|
428
|
-
inFlight = Promise.resolve()
|
|
429
|
-
.then(() => (onRun ? onRun() : Promise.resolve({ ok: true, skipped: true })))
|
|
430
|
-
.then((result) => {
|
|
431
|
-
// Refresh the baseline only for a successful run (design §5.3: an
|
|
432
|
-
// LLM failure must not move the baseline, so the next write can
|
|
433
|
-
// immediately re-trigger a retry). A `{ok:false}` result or a throw
|
|
434
|
-
// keeps the old baseline. A run that reports nothing is treated as
|
|
435
|
-
// completed without failure (no-op hooks / minimal test doubles).
|
|
436
|
-
if (result && result.ok) {
|
|
437
|
-
try {
|
|
438
|
-
baseline = shouldTrigger(service);
|
|
439
|
-
} catch (error) {
|
|
440
|
-
// Store closed mid-flight: keep the last known baseline.
|
|
441
|
-
logger?.warn?.(`dsh-mneme dream: baseline refresh failed: ${String(error)}`);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
})
|
|
445
|
-
.catch((error) => {
|
|
446
|
-
logger?.warn?.(`dsh-mneme dream: run failed: ${error?.message ?? error}`);
|
|
447
|
-
// Failed runs do not refresh the baseline.
|
|
448
|
-
})
|
|
449
|
-
.finally(() => {
|
|
450
|
-
running = false;
|
|
451
|
-
inFlight = null;
|
|
452
|
-
});
|
|
453
|
-
}, delayMs);
|
|
454
|
-
return true;
|
|
455
|
-
}
|
|
456
|
-
|
|
457
|
-
async function dispose() {
|
|
458
|
-
disposed = true;
|
|
459
|
-
if (pendingTimer) { clearTimeout(pendingTimer); pendingTimer = null; }
|
|
460
|
-
// An in-flight run is left to complete naturally (its LLM calls are
|
|
461
|
-
// already paid for and aborting would discard the work). Await it so the
|
|
462
|
-
// caller can close the store only after every write has landed.
|
|
463
|
-
if (inFlight) await inFlight.catch(() => {});
|
|
464
|
-
}
|
|
465
|
-
|
|
466
|
-
async function runDream(ctx, service, config) {
|
|
467
|
-
const logger = ctx.logger;
|
|
468
|
-
let memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
469
|
-
if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
|
|
470
|
-
// v0.4.4 滑动窗口:只 consolidation 最近 dreamMaxSnapshotSize 条记忆,
|
|
471
|
-
// 窗口外的旧记忆不进 snapshot(大记忆量下全量快照会撑爆 LLM 输入,配合
|
|
472
|
-
// 隐式 keep 让 run 始终可收敛)。按 updated_at 倒序取前 maxSize 条。
|
|
473
|
-
const maxSize = Number.isInteger(config.dreamMaxSnapshotSize) ? config.dreamMaxSnapshotSize : 200;
|
|
474
|
-
memories = [...memories]
|
|
475
|
-
.sort((a, b) => {
|
|
476
|
-
const ta = String(a.updated_at ?? "");
|
|
477
|
-
const tb = String(b.updated_at ?? "");
|
|
478
|
-
if (ta < tb) return 1;
|
|
479
|
-
if (ta > tb) return -1;
|
|
480
|
-
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
481
|
-
})
|
|
482
|
-
.slice(0, Math.max(1, maxSize));
|
|
483
|
-
const snapshot = new Map(memories.map((m) => [m.id, m]));
|
|
484
|
-
const route = resolveRoute(ctx, config, logger);
|
|
485
|
-
const runId = randomUUID();
|
|
486
|
-
const snapshotHash = hashSnapshot([...snapshot.values()]);
|
|
487
|
-
// Conflict freeze (opt-in): when enabled, conflict decisions are parked for
|
|
488
|
-
// manual review instead of auto-adjudicated. Read once up front so the
|
|
489
|
-
// prompt hint and the apply-split agree on the same gate.
|
|
490
|
-
const freezeEnabled = config.conflictFreezeEnabled === true;
|
|
491
|
-
// Every exit (success or failure) funnels through `finish`, which writes
|
|
492
|
-
// the audit row + receipt. A record failure is logged, never thrown —
|
|
493
|
-
// auditing must not break the consolidation path. Failed runs still
|
|
494
|
-
// capture their decisions/outcome when the LLM produced a validated list
|
|
495
|
-
// (e.g. summary step failed after consolidation), so the partial write is
|
|
496
|
-
// replayable too.
|
|
497
|
-
const finish = (result) => {
|
|
498
|
-
// status is derived from what actually committed: ok only when the full
|
|
499
|
-
// decision list landed (or a summary was refreshed); noop when nothing
|
|
500
|
-
// changed; degraded when real changes landed without a summary;
|
|
501
|
-
// reconcile when decisions were validated but some did not commit (CAS
|
|
502
|
-
// conflict / rollback); failed on any LLM/validation error. No fake "ok"
|
|
503
|
-
// for an empty or partial run.
|
|
504
|
-
const status = result.status ?? (result.ok ? "ok" : "failed");
|
|
505
|
-
const applied = result.applied ?? 0;
|
|
506
|
-
const summaryStored = result.summary ?? false;
|
|
507
|
-
const receipt = buildReceipt({ runId, status, snapshotHash, inputCount: snapshot.size, applied, summaryStored });
|
|
508
|
-
try {
|
|
509
|
-
service.saveDreamRun({
|
|
510
|
-
id: runId,
|
|
511
|
-
status,
|
|
512
|
-
error: result.error,
|
|
513
|
-
provider: route?.provider,
|
|
514
|
-
model: route?.model,
|
|
515
|
-
snapshot_hash: snapshotHash,
|
|
516
|
-
input_count: snapshot.size,
|
|
517
|
-
// 裁决规则版本号:config.policyEpoch(默认 0)。规则升级后该行保留
|
|
518
|
-
// 当时的 epoch,旧裁决据此降级为历史证据(store 层 getLatestPolicyEpoch
|
|
519
|
-
// 只负责读取当前生效版本,写入由这里完成)。
|
|
520
|
-
policy_epoch: config.policyEpoch ?? 0,
|
|
521
|
-
// Full input snapshot (canonical fields) so the exact arbitration
|
|
522
|
-
// input can be rebuilt offline from the audit row alone — the
|
|
523
|
-
// digest + decisions + outcome triple makes silent errors locatable
|
|
524
|
-
// even after the store has moved on.
|
|
525
|
-
input: [...snapshot.values()].map((m) => ({
|
|
526
|
-
id: m.id,
|
|
527
|
-
type: m.type,
|
|
528
|
-
title: m.title,
|
|
529
|
-
content: m.content,
|
|
530
|
-
importance: m.importance,
|
|
531
|
-
updated_at: m.updated_at
|
|
532
|
-
})),
|
|
533
|
-
decisions: result.decisions,
|
|
534
|
-
outcome: result.outcome,
|
|
535
|
-
applied,
|
|
536
|
-
summary_stored: summaryStored,
|
|
537
|
-
receipt
|
|
538
|
-
});
|
|
539
|
-
} catch (error) {
|
|
540
|
-
logger?.warn?.(`dsh-mneme dream: failed to record audit run: ${String(error)}`);
|
|
541
|
-
}
|
|
542
|
-
return { ...result, runId, receipt, snapshotHash };
|
|
543
|
-
};
|
|
544
|
-
if (!route) {
|
|
545
|
-
logger?.warn?.("dsh-mneme dream: no llm route available");
|
|
546
|
-
return finish({ ok: false, error: "no llm route", summary: false });
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
let listText;
|
|
550
|
-
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
551
|
-
try {
|
|
552
|
-
const vectors = await collectVectors(memories, semantic);
|
|
553
|
-
if (vectors) {
|
|
554
|
-
const k = Math.min(10, Math.max(1, Math.floor(Math.sqrt(memories.length / 2))));
|
|
555
|
-
const clusters = clusterMemories(memories, vectors, k);
|
|
556
|
-
const conflicts = findPotentialConflicts(memories, vectors, 0.85);
|
|
557
|
-
const conflictIds = new Set(conflicts.flatMap((c) => [c.a.id, c.b.id]));
|
|
558
|
-
const parts = [];
|
|
559
|
-
clusters.forEach((cluster, ci) => {
|
|
560
|
-
parts.push(`# 聚类 ${ci + 1}`);
|
|
561
|
-
for (const m of cluster) {
|
|
562
|
-
parts.push(
|
|
563
|
-
`id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}` +
|
|
564
|
-
(conflictIds.has(m.id) ? " | [潜在冲突]" : "")
|
|
565
|
-
);
|
|
566
|
-
}
|
|
567
|
-
});
|
|
568
|
-
listText = parts.join("\n");
|
|
569
|
-
logger?.info?.(`[dsh-mneme] dream semantic pre-group: ${clusters.length} clusters, ${conflicts.length} conflict pairs`);
|
|
570
|
-
}
|
|
571
|
-
} catch (error) {
|
|
572
|
-
logger?.warn?.(`[dsh-mneme] dream semantic pre-group failed: ${String(error)}`);
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
if (!listText) {
|
|
576
|
-
listText = [...snapshot.values()].map((m) =>
|
|
577
|
-
`id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}`
|
|
578
|
-
).join("\n");
|
|
579
|
-
}
|
|
580
|
-
|
|
581
|
-
// Freeze-aware prompt: in freeze mode the conflict branch still outputs
|
|
582
|
-
// winner/loser (validation requires them) but they are treated as tentative
|
|
583
|
-
// candidates — the human makes the final call, not the model.
|
|
584
|
-
const consolidationPrompt = freezeEnabled
|
|
585
|
-
? CONSOLIDATION_PROMPT + `\n\n当前为「冲突冻结」模式:检测到内容矛盾的条目时,仍请输出 conflict,并以 winner/loser 作为候选、reason 说明理由;冲突不会被自动裁决,而会冻结待人工确认。`
|
|
586
|
-
: CONSOLIDATION_PROMPT;
|
|
587
|
-
let decisionText;
|
|
588
|
-
try {
|
|
589
|
-
// Bug8: the consolidation call is audited (tokens/time/status). A throw
|
|
590
|
-
// re-propagates to the catch below; an aborted stream returns undefined
|
|
591
|
-
// and is treated as a failed run after the check below.
|
|
592
|
-
decisionText = await runAuditedLlm(ctx, service, config, {
|
|
593
|
-
triggerSource: "autoDream",
|
|
594
|
-
operationType: "dream_consolidate",
|
|
595
|
-
modelId: `${route.provider}:${route.model}`,
|
|
596
|
-
relatedMemoryIds: [...snapshot.keys()]
|
|
597
|
-
}, (reportUsage) => streamText(ctx, {
|
|
598
|
-
provider: route.provider,
|
|
599
|
-
model: route.model,
|
|
600
|
-
purpose: "compaction",
|
|
601
|
-
maxTokens: config.dreamMaxTokens ?? 4096,
|
|
602
|
-
...(config.dreamReasoningEffort && config.dreamReasoningEffort !== "none"
|
|
603
|
-
? { reasoningEffort: config.dreamReasoningEffort }
|
|
604
|
-
: {}),
|
|
605
|
-
messages: [
|
|
606
|
-
{ role: "system", content: [{ type: "text", text: consolidationPrompt }] },
|
|
607
|
-
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
608
|
-
]
|
|
609
|
-
}, reportUsage));
|
|
610
|
-
} catch (error) {
|
|
611
|
-
logger?.warn?.(`dsh-mneme dream: consolidation llm call failed: ${String(error)}`);
|
|
612
|
-
return finish({ ok: false, error: "llm failed", summary: false });
|
|
613
|
-
}
|
|
614
|
-
if (decisionText === undefined) {
|
|
615
|
-
logger?.warn?.("dsh-mneme dream: consolidation llm stream aborted or errored");
|
|
616
|
-
return finish({ ok: false, error: "llm failed", summary: false });
|
|
617
|
-
}
|
|
618
|
-
|
|
619
|
-
const decisions = extractJsonArray(decisionText);
|
|
620
|
-
if (!Array.isArray(decisions)) {
|
|
621
|
-
logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0})`);
|
|
622
|
-
return finish({ ok: false, error: "no json array in llm output", summary: false });
|
|
623
|
-
}
|
|
624
|
-
const { ok, errors } = validateDecisions(decisions, snapshot, {
|
|
625
|
-
maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
|
|
626
|
-
minAgeHours: config.reflectionUpdateMinAgeHours,
|
|
627
|
-
// v0.4.4 fix:显式透传,用户配 dreamImplicitKeep:false 时严格模式必须
|
|
628
|
-
// 真正生效,dreamMinExplicitCoverage 决定隐式 keep 下的覆盖率下限。
|
|
629
|
-
dreamImplicitKeep: config.dreamImplicitKeep,
|
|
630
|
-
dreamMinExplicitCoverage: config.dreamMinExplicitCoverage
|
|
631
|
-
});
|
|
632
|
-
if (!ok) {
|
|
633
|
-
logger?.warn?.(`dsh-mneme dream: invalid decisions: ${errors.join("; ")}`);
|
|
634
|
-
return finish({ ok: false, error: `invalid decisions: ${errors.length} errors`, summary: false });
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
// Capture pre-update snapshots so the audit records what each update changed.
|
|
638
|
-
const updateSnapshots = {};
|
|
639
|
-
for (const d of decisions) {
|
|
640
|
-
if (d.action === "update") {
|
|
641
|
-
const mem = snapshot.get(d.ids[0]);
|
|
642
|
-
if (mem) updateSnapshots[d.ids[0]] = { title: mem.title, content: mem.content, importance: mem.importance };
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
// Conflict freeze (opt-in): when enabled, conflict decisions are not
|
|
647
|
-
// auto-adjudicated — no winner kept, no loser archived. The pair is parked
|
|
648
|
-
// in conflict_pending for human review instead. Best-effort: a store
|
|
649
|
-
// failure here must never block the run (fail-safe — the memories are left
|
|
650
|
-
// untouched and nothing is arbitrated). The cap (conflictFreezeMaxPending)
|
|
651
|
-
// bounds the review queue; overflow is skipped with a warning.
|
|
652
|
-
let frozenCount = 0;
|
|
653
|
-
const frozenIds = [];
|
|
654
|
-
const applyList = freezeEnabled ? decisions.filter((d) => d.action !== "conflict") : decisions;
|
|
655
|
-
if (freezeEnabled) {
|
|
656
|
-
const conflictsToFreeze = decisions.filter((d) => d.action === "conflict");
|
|
657
|
-
if (conflictsToFreeze.length > 0) {
|
|
658
|
-
try {
|
|
659
|
-
const maxPending = Number.isInteger(config.conflictFreezeMaxPending) ? config.conflictFreezeMaxPending : 100;
|
|
660
|
-
const pendingNow = service.countConflictPending();
|
|
661
|
-
const budget = Math.max(0, maxPending - pendingNow);
|
|
662
|
-
const toFreeze = conflictsToFreeze.slice(0, budget);
|
|
663
|
-
if (conflictsToFreeze.length > budget) {
|
|
664
|
-
logger?.warn?.(`dsh-mneme dream: conflict freeze queue full (${pendingNow}/${maxPending}), skipped ${conflictsToFreeze.length - budget} conflict(s)`);
|
|
665
|
-
}
|
|
666
|
-
for (const d of toFreeze) {
|
|
667
|
-
try {
|
|
668
|
-
service.saveConflictPending({ run_id: runId, memory_a: d.winner, memory_b: d.loser, reason: d.reason });
|
|
669
|
-
frozenCount++;
|
|
670
|
-
frozenIds.push(d.winner, d.loser);
|
|
671
|
-
} catch (error) {
|
|
672
|
-
logger?.warn?.(`dsh-mneme dream: failed to freeze conflict ${d.winner}/${d.loser}: ${String(error)}`);
|
|
673
|
-
}
|
|
674
|
-
}
|
|
675
|
-
} catch (error) {
|
|
676
|
-
logger?.warn?.(`dsh-mneme dream: conflict freeze lookup failed: ${String(error)}`);
|
|
677
|
-
}
|
|
678
|
-
}
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
// CAS-guarded, per-decision-transactional apply against the run snapshot:
|
|
682
|
-
// a target changed during the LLM call is skipped and reported as a
|
|
683
|
-
// conflict instead of being overwritten (item ①). Frozen conflicts are
|
|
684
|
-
// excluded from this list (they are parked, not applied).
|
|
685
|
-
const { applied, conflicts, failures, committed } = applyDecisions(applyList, service, logger, snapshot, config);
|
|
686
|
-
// Per-record receipt chain: one row per actually-committed merge/conflict/
|
|
687
|
-
// update verdict, stamped with the decision-basis digest + idempotency
|
|
688
|
-
// counters (count_before → count_after). Written here, before the run audit
|
|
689
|
-
// row, so the verdict trail always precedes the run trail it belongs to.
|
|
690
|
-
// Bookkeeping: a write failure is logged and swallowed — it must never
|
|
691
|
-
// block the consolidation flow.
|
|
692
|
-
try {
|
|
693
|
-
for (const r of buildRecordReceipts({ runId, committed, snapshot, policyEpoch: config.policyEpoch ?? 0 })) {
|
|
694
|
-
service.saveReceipt(r);
|
|
695
|
-
}
|
|
696
|
-
} catch (error) {
|
|
697
|
-
logger?.warn?.(`dsh-mneme dream: failed to write per-record receipt: ${String(error)}`);
|
|
698
|
-
}
|
|
699
|
-
// Attach the pre-update snapshot to the audit copy of each update decision
|
|
700
|
-
// so the recorded row shows the before/after delta, not just the target.
|
|
701
|
-
const auditDecisions = decisions.map((d) =>
|
|
702
|
-
d.action === "update" && updateSnapshots[d.ids[0]]
|
|
703
|
-
? { ...d, _before: updateSnapshots[d.ids[0]] }
|
|
704
|
-
: d
|
|
705
|
-
);
|
|
706
|
-
// Outcome is derived from the ACTUALLY committed sub-steps, never from the
|
|
707
|
-
// raw LLM decision list — a merge whose archive step rolled back must not
|
|
708
|
-
// claim "merge-archived" (item ②). Conflicts/failures ride along so the
|
|
709
|
-
// audit row records why the run diverged.
|
|
710
|
-
const outcome = { ...buildOutcome(committed), conflicts, failures };
|
|
711
|
-
// Frozen conflicts were not adjudicated: mark both sides pending in the
|
|
712
|
-
// per-id outcome so the audit row shows they were parked, not skipped.
|
|
713
|
-
if (frozenIds.length) {
|
|
714
|
-
for (const id of frozenIds) outcome.byId[id] = "conflict-pending";
|
|
715
|
-
}
|
|
716
|
-
// Decisions validated but not fully committed → reconcile (not ok).
|
|
717
|
-
const partial = conflicts.length > 0 || failures.length > 0;
|
|
718
|
-
// No decision landed (all-keep, or every decision skipped as an idempotent
|
|
719
|
-
// replay) → nothing substantive changed. Distinct from a success: such a
|
|
720
|
-
// run must never be reported as ok, or the audit claims work that never
|
|
721
|
-
// happened and the scheduler refreshes the baseline on a false positive.
|
|
722
|
-
// Frozen conflicts are substantive output (parked for review), so a run
|
|
723
|
-
// that only froze conflicts is not a noop.
|
|
724
|
-
const noChange = frozenCount === 0 && applied === 0 && committed.every((c) => c.action === "keep");
|
|
725
|
-
|
|
726
|
-
// Keep the vector index consistent with the post-dream store state.
|
|
727
|
-
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
728
|
-
try {
|
|
729
|
-
await maintainIndexAfterDream(applyList, service, semantic);
|
|
730
|
-
} catch (error) {
|
|
731
|
-
logger?.warn?.(`[dsh-mneme] dream index maintenance failed: ${String(error)}`);
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
|
|
735
|
-
// Summary generation (second LLM call). A throwing stream is reported as
|
|
736
|
-
// a failed run; summary:false marks a run that produced no summary.
|
|
737
|
-
let summaryText;
|
|
738
|
-
try {
|
|
739
|
-
// Bug8: the summary call is audited too (operation dream_summarize).
|
|
740
|
-
summaryText = await runAuditedLlm(ctx, service, config, {
|
|
741
|
-
triggerSource: "autoDream",
|
|
742
|
-
operationType: "dream_summarize",
|
|
743
|
-
modelId: `${route.provider}:${route.model}`,
|
|
744
|
-
relatedMemoryIds: []
|
|
745
|
-
}, (reportUsage) => streamText(ctx, {
|
|
746
|
-
provider: route.provider,
|
|
747
|
-
model: route.model,
|
|
748
|
-
purpose: "compaction",
|
|
749
|
-
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
750
|
-
...(config.dreamReasoningEffort && config.dreamReasoningEffort !== "none"
|
|
751
|
-
? { reasoningEffort: config.dreamReasoningEffort }
|
|
752
|
-
: {}),
|
|
753
|
-
messages: [
|
|
754
|
-
{ role: "system", content: [{ type: "text", text: SUMMARY_PROMPT }] },
|
|
755
|
-
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
|
|
756
|
-
]
|
|
757
|
-
}, reportUsage));
|
|
758
|
-
} catch (error) {
|
|
759
|
-
logger?.warn?.(`dsh-mneme dream: summary llm call failed: ${String(error)}`);
|
|
760
|
-
return finish({ ok: false, error: "llm failed", applied, decisions: auditDecisions, outcome, frozen: frozenCount, summary: false });
|
|
761
|
-
}
|
|
762
|
-
let summaryStored = false;
|
|
763
|
-
if (summaryText !== undefined && summaryText.trim()) {
|
|
764
|
-
// Bug5 carve-out: the library overview is regenerated every run, so it
|
|
765
|
-
// must REPLACE the previous overview (not append — that would grow the
|
|
766
|
-
// summary unboundedly). `_overwrite` still archives the old overview into
|
|
767
|
-
// content_history before replacing it.
|
|
768
|
-
service.saveWithDedupe({ type: "summary", title: "记忆库总览", content: summaryText.trim(), importance: 5, source: "dream", _overwrite: true });
|
|
769
|
-
summaryStored = true;
|
|
770
|
-
// Re-embed the fresh summary so the index stays in sync with the store.
|
|
771
|
-
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
772
|
-
try {
|
|
773
|
-
const summary = service.all().find((m) => m.type === "summary");
|
|
774
|
-
if (summary) {
|
|
775
|
-
const v = await semantic.embedder.embedSingle([summary.title, summary.content].filter(Boolean).join("\n"));
|
|
776
|
-
if (v?.length) semantic.vectorIndex.saveEmbedding(summary.id, v);
|
|
777
|
-
if (semantic.embedder.modelHash) semantic.vectorIndex.markModel?.(semantic.embedder.modelHash, semantic.embedder.dimension);
|
|
778
|
-
}
|
|
779
|
-
} catch { /* best-effort */ }
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
// Honest status assignment (never a fake ok):
|
|
783
|
-
// reconcile — some decisions validated but did not commit (CAS/rollback).
|
|
784
|
-
// noop — nothing changed and no summary persisted: truly an empty
|
|
785
|
-
// run. ok:false keeps the scheduler from moving the baseline.
|
|
786
|
-
// ok — either real changes landed, or a fresh summary was stored
|
|
787
|
-
// (all-keep + summary is a substantive summary refresh).
|
|
788
|
-
// degraded — real consolidation landed but the summary came back empty/
|
|
789
|
-
// missing: the store was absorbed (ok for the baseline) but
|
|
790
|
-
// the run did not produce its full output (marked, not faked).
|
|
791
|
-
let status;
|
|
792
|
-
let okResult;
|
|
793
|
-
if (partial) {
|
|
794
|
-
status = "reconcile";
|
|
795
|
-
okResult = false;
|
|
796
|
-
} else if (noChange) {
|
|
797
|
-
status = summaryStored ? "ok" : "noop";
|
|
798
|
-
okResult = summaryStored;
|
|
799
|
-
} else {
|
|
800
|
-
status = summaryStored ? "ok" : "degraded";
|
|
801
|
-
okResult = true;
|
|
802
|
-
}
|
|
803
|
-
return finish({
|
|
804
|
-
ok: okResult,
|
|
805
|
-
status,
|
|
806
|
-
applied,
|
|
807
|
-
decisions: auditDecisions,
|
|
808
|
-
outcome,
|
|
809
|
-
conflicts,
|
|
810
|
-
failures,
|
|
811
|
-
frozen: frozenCount,
|
|
812
|
-
summary: summaryStored
|
|
813
|
-
});
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
return { maybeSchedule, runDream, dispose };
|
|
817
|
-
}
|
|
1
|
+
import { validateDecisions, applyDecisions } from "./dream/decisions.js";
|
|
2
|
+
import { clusterMemories, findPotentialConflicts } from "./dream/clustering.js";
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
export { validateDecisions, applyDecisions };
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
// Extract the first JSON array from LLM output, tolerating markdown fences,
|
|
8
|
+
// leading/trailing prose, and common wrapper noise. Returns an array or null.
|
|
9
|
+
function extractJsonArray(text) {
|
|
10
|
+
if (typeof text !== "string" || text.trim().length === 0) return null;
|
|
11
|
+
|
|
12
|
+
// 1. Strip markdown code fences (```json ... ``` or ``` ... ```).
|
|
13
|
+
let cleaned = text.replace(/```(?:json)?\s*([\s\S]*?)```/gi, "$1");
|
|
14
|
+
cleaned = cleaned.trim();
|
|
15
|
+
|
|
16
|
+
// 2. Find the first '[' and the matching last ']' that yields valid JSON.
|
|
17
|
+
const start = cleaned.indexOf("[");
|
|
18
|
+
if (start === -1) return null;
|
|
19
|
+
for (let end = cleaned.lastIndexOf("]"); end > start; end = cleaned.lastIndexOf("]", end - 1)) {
|
|
20
|
+
const candidate = cleaned.slice(start, end + 1);
|
|
21
|
+
try {
|
|
22
|
+
return JSON.parse(candidate);
|
|
23
|
+
} catch {
|
|
24
|
+
// Light repair: remove trailing commas before ] or }.
|
|
25
|
+
try {
|
|
26
|
+
const repaired = candidate.replace(/,(\s*[}\]])/g, "$1");
|
|
27
|
+
return JSON.parse(repaired);
|
|
28
|
+
} catch {
|
|
29
|
+
// keep searching backwards
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// 3. Fallback: a broader regex extraction.
|
|
35
|
+
try {
|
|
36
|
+
const match = cleaned.match(/\[[\s\S]*\]/);
|
|
37
|
+
if (match) return JSON.parse(match[0]);
|
|
38
|
+
} catch {
|
|
39
|
+
// fall through
|
|
40
|
+
}
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
const SUMMARY_PROMPT = `你是记忆库摘要助手。根据整理后的记忆,生成一段 150-200 字的记忆库总览,覆盖:用户偏好、活跃项目、关键决策。之后作为会话上下文注入。只输出摘要文本,不要其他内容。`;
|
|
44
|
+
|
|
45
|
+
const CONSOLIDATION_PROMPT = `你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。
|
|
46
|
+
请执行记忆巩固(consolidation),输出一个决策 JSON 数组。
|
|
47
|
+
|
|
48
|
+
【决策格式(必须严格遵守)】
|
|
49
|
+
每个决策必须是对象,字段固定:
|
|
50
|
+
- "action":必填。取值只能是 "keep" / "merge" / "archive" / "update" / "conflict" 之一(字段名必须是 action,严禁写成 type)
|
|
51
|
+
- "ids":必填,数组,本决策涉及的记忆 id 列表
|
|
52
|
+
- "reason":可选,字符串,决策理由
|
|
53
|
+
- "importance":可选,整数 1-5
|
|
54
|
+
- merge 额外字段:"keepSource"(单个 id 字符串,必须是 ids 之一)+ 合并后的 "title"、"content"
|
|
55
|
+
- conflict 额外字段:"winner" 与 "loser",都是【单个 id 字符串,不是数组】
|
|
56
|
+
- update 额外字段:修正后的 "title" 和/或 "content";"ids" 只能包含一个 id
|
|
57
|
+
|
|
58
|
+
【决策 JSON 示例】
|
|
59
|
+
[
|
|
60
|
+
{ "action": "merge", "ids": ["m1", "m2"], "keepSource": "m1", "title": "合并标题", "content": "合并后的摘要内容", "importance": 4, "reason": "主题相近" },
|
|
61
|
+
{ "action": "conflict", "winner": "m3", "loser": "m4", "reason": "内容矛盾,保留更新的信息" },
|
|
62
|
+
{ "action": "update", "ids": ["m5"], "content": "修正后的内容", "reason": "信息过时" },
|
|
63
|
+
{ "action": "archive", "ids": ["m6"], "reason": "重复或过时" }
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
【任务】
|
|
67
|
+
1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)
|
|
68
|
+
2. 识别重复/过时信息 → archive
|
|
69
|
+
3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)
|
|
70
|
+
4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)
|
|
71
|
+
- update 的 ids 只能包含一个 id
|
|
72
|
+
- 必须提供修正后的 title 和/或 content
|
|
73
|
+
- 仅当内容确实需要修正时才使用,不要滥用
|
|
74
|
+
- 每次整理最多输出 2 个 update
|
|
75
|
+
- 24 小时内新建的记忆不可 update
|
|
76
|
+
5. 无问题的条目无需输出(未提及的条目将自动保留 keep)
|
|
77
|
+
|
|
78
|
+
【硬性规则】
|
|
79
|
+
- 字段名必须精确为 "action",严禁写成 "type";字段名统一用双引号
|
|
80
|
+
- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】
|
|
81
|
+
- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)
|
|
82
|
+
- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep
|
|
83
|
+
- merge 的 keepSource 必须是 ids 之一
|
|
84
|
+
- 仅合并同类型条目(type 相同)
|
|
85
|
+
- 不要编造 ids;只使用提供的 id
|
|
86
|
+
- 重要性 1-5,合并后取最高
|
|
87
|
+
- 只输出 JSON 数组,不要其他文字`;
|
|
88
|
+
|
|
89
|
+
function totalChars(memories) {
|
|
90
|
+
return memories.reduce((sum, m) => sum + (m.title?.length ?? 0) + (m.content?.length ?? 0), 0);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ---------------------------------------------------------------- audit
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Canonical digest of the consolidation input snapshot. Built from stable
|
|
97
|
+
* fields sorted by id, so identical inputs always yield the same hash — the
|
|
98
|
+
* basis for replaying/verifying a recorded decision (receipt check).
|
|
99
|
+
*/
|
|
100
|
+
export function hashSnapshot(memories) {
|
|
101
|
+
const canon = memories
|
|
102
|
+
.map((m) => [m.id, m.type, m.title, m.content, m.importance, m.updated_at])
|
|
103
|
+
.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
|
|
104
|
+
.map((parts) => parts.map((p) => String(p ?? "")).join("\u0001"))
|
|
105
|
+
.join("\u0002");
|
|
106
|
+
return createHash("sha256").update(canon).digest("hex");
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Compact machine-verifiable receipt for one autoDream run. Format:
|
|
111
|
+
* dsh-mneme:run:<runId>:<status>:<snapshotHash(12)>:<inputCount>:<applied>:<summaryFlag>
|
|
112
|
+
* Enough to correlate a run with its persisted audit row and to spot silent
|
|
113
|
+
* drift (same snapshot hash + same decisions must reproduce the same outcome).
|
|
114
|
+
*/
|
|
115
|
+
export function buildReceipt({ runId, status, snapshotHash, inputCount, applied, summaryStored }) {
|
|
116
|
+
return `dsh-mneme:run:${runId}:${status}:${snapshotHash.slice(0, 12)}:${inputCount}:${applied}:${summaryStored ? 1 : 0}`;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Parse a receipt back into fields; returns undefined for malformed input.
|
|
121
|
+
*/
|
|
122
|
+
export function parseReceipt(receipt) {
|
|
123
|
+
if (typeof receipt !== "string") return undefined;
|
|
124
|
+
const parts = receipt.split(":");
|
|
125
|
+
if (parts.length !== 8 || parts[0] !== "dsh-mneme" || parts[1] !== "run") return undefined;
|
|
126
|
+
const [, , runId, status, snapshotHash, inputCount, applied, summaryStored] = parts;
|
|
127
|
+
// reconcile = decisions validated but one or more did not commit (CAS
|
|
128
|
+
// conflict / transaction rollback) — the store diverges from the decision
|
|
129
|
+
// list and the run must be reconciled, never reported as a fake ok.
|
|
130
|
+
if (!runId || !/^(ok|noop|degraded|reconcile|failed)$/.test(status)) return undefined;
|
|
131
|
+
const count = Number(inputCount);
|
|
132
|
+
const appliedN = Number(applied);
|
|
133
|
+
if (!Number.isInteger(count) || !Number.isInteger(appliedN)) return undefined;
|
|
134
|
+
return { runId, status, snapshotHash, inputCount: count, applied: appliedN, summaryStored: summaryStored === "1" };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Derive the per-id disposition (keep / merge-keep / merge-archived /
|
|
139
|
+
* archived / conflict-winner / conflict-archived) from a validated decision
|
|
140
|
+
* list. Stored in the audit row so a run can be replayed without re-running
|
|
141
|
+
* the LLM.
|
|
142
|
+
*/
|
|
143
|
+
export function buildOutcome(decisions) {
|
|
144
|
+
const byId = {};
|
|
145
|
+
for (const d of decisions ?? []) {
|
|
146
|
+
if (d.action === "keep") {
|
|
147
|
+
for (const id of d.ids) byId[id] = "keep";
|
|
148
|
+
} else if (d.action === "archive") {
|
|
149
|
+
for (const id of d.ids) byId[id] = "archived";
|
|
150
|
+
} else if (d.action === "merge") {
|
|
151
|
+
for (const id of d.ids) byId[id] = id === d.keepSource ? "merge-keep" : "merge-archived";
|
|
152
|
+
} else if (d.action === "conflict") {
|
|
153
|
+
byId[d.winner] = "conflict-winner";
|
|
154
|
+
byId[d.loser] = "conflict-archived";
|
|
155
|
+
} else if (d.action === "update") {
|
|
156
|
+
for (const id of d.ids) byId[id] = "updated";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return { byId };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Content-addressed digest of the memories a verdict was decided against
|
|
164
|
+
* (id + title + content + importance), sorted by id so identical inputs always
|
|
165
|
+
* hash the same. This is the per-record "判定依据" fingerprint: a receipt whose
|
|
166
|
+
* digest cannot be reproduced from the involved memories is a bare claim, and a
|
|
167
|
+
* digest match with a divergent outcome pinpoints drift to the exact record.
|
|
168
|
+
*/
|
|
169
|
+
export function hashDecisionInput(memories) {
|
|
170
|
+
const canon = (memories ?? [])
|
|
171
|
+
.map((m) => [m.id, m.title, m.content, m.importance])
|
|
172
|
+
.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
|
|
173
|
+
.map((p) => p.map((x) => String(x ?? "")).join(""))
|
|
174
|
+
.join("");
|
|
175
|
+
return createHash("sha256").update(canon).digest("hex");
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Build the per-record receipts for a run's actually-committed mutable verdicts
|
|
180
|
+
* (merge/conflict/update) — one row per verdict in the receipt_chain. Inputs
|
|
181
|
+
* are drawn from the run snapshot (what the LLM actually arbitrated against),
|
|
182
|
+
* and the idempotency counters count_before → count_after come from the
|
|
183
|
+
* committed sub-step, so replaying the same decision must reproduce the same
|
|
184
|
+
* numbers. verdict starts "live"; a later policy_epoch upgrade will batch-mark
|
|
185
|
+
* older verdicts "historical" (a receipt_chain rewrite driven by the store's
|
|
186
|
+
* getLatestPolicyEpoch — out of scope for this pass), while "revoked" is
|
|
187
|
+
* reserved for verdicts later overturned by an explicit human decision.
|
|
188
|
+
*/
|
|
189
|
+
function buildRecordReceipts({ runId, committed, snapshot, policyEpoch }) {
|
|
190
|
+
const at = (id) => snapshot?.get?.(id);
|
|
191
|
+
const receipts = [];
|
|
192
|
+
for (const c of committed ?? []) {
|
|
193
|
+
const base = {
|
|
194
|
+
run_id: runId,
|
|
195
|
+
verdict: "live",
|
|
196
|
+
count_before: c.count_before,
|
|
197
|
+
count_after: c.count_after,
|
|
198
|
+
policy_epoch: policyEpoch,
|
|
199
|
+
created_at: new Date().toISOString()
|
|
200
|
+
};
|
|
201
|
+
if (c.action === "merge") {
|
|
202
|
+
receipts.push({
|
|
203
|
+
...base,
|
|
204
|
+
receipt_id: randomUUID(),
|
|
205
|
+
record_id: c.keepSource,
|
|
206
|
+
kind: "merge",
|
|
207
|
+
input_digest: hashDecisionInput((c.ids ?? []).map(at).filter(Boolean)),
|
|
208
|
+
keep_source: c.keepSource,
|
|
209
|
+
sources: c.ids
|
|
210
|
+
});
|
|
211
|
+
} else if (c.action === "conflict") {
|
|
212
|
+
receipts.push({
|
|
213
|
+
...base,
|
|
214
|
+
receipt_id: randomUUID(),
|
|
215
|
+
record_id: c.winner,
|
|
216
|
+
kind: "conflict",
|
|
217
|
+
input_digest: hashDecisionInput([at(c.winner), at(c.loser)].filter(Boolean)),
|
|
218
|
+
winner_id: c.winner,
|
|
219
|
+
loser_id: c.loser
|
|
220
|
+
});
|
|
221
|
+
} else if (c.action === "update") {
|
|
222
|
+
receipts.push({
|
|
223
|
+
...base,
|
|
224
|
+
receipt_id: randomUUID(),
|
|
225
|
+
record_id: c.ids[0],
|
|
226
|
+
kind: "update",
|
|
227
|
+
input_digest: hashDecisionInput([at(c.ids[0])].filter(Boolean))
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return receipts;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Consume an LLM stream and return the accumulated text. Direct text-delta
|
|
236
|
+
* accumulation covers both the real protocol ({type:"text-delta", index, text})
|
|
237
|
+
* and looser test doubles ({type:"text-delta", text}); a terminal error/abort
|
|
238
|
+
* surfaces as undefined. The caller decides how to treat an empty result.
|
|
239
|
+
* `onUsage` (optional, Bug8) receives any usage chunk for token accounting.
|
|
240
|
+
*/
|
|
241
|
+
async function streamText(ctx, options, onUsage) {
|
|
242
|
+
let text = "";
|
|
243
|
+
for await (const chunk of ctx.llm.stream(options)) {
|
|
244
|
+
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
245
|
+
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk);
|
|
246
|
+
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
return text;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Bug8: wrap a background LLM call so its token/time/status are recorded in the
|
|
255
|
+
* llm_audit_logs table. Best-effort bookkeeping: a failure to WRITE the audit
|
|
256
|
+
* row is swallowed (never blocks the LLM call), while a failure of the call
|
|
257
|
+
* itself is captured as status='error' and re-thrown so the caller keeps its
|
|
258
|
+
* existing error path. `spec` carries the static metadata (trigger_source,
|
|
259
|
+
* operation_type, model_id, related_memory_ids); `body(reportUsage)` performs
|
|
260
|
+
* the actual stream consumption and is handed a usage reporter for the chunks.
|
|
261
|
+
*/
|
|
262
|
+
async function runAuditedLlm(ctx, service, config, spec, body) {
|
|
263
|
+
const audit = config?.llmAudit;
|
|
264
|
+
if (audit?.enabled === false || typeof service?.saveLlmAudit !== "function") return body(() => {});
|
|
265
|
+
const startedAt = Date.now();
|
|
266
|
+
const timestamp = new Date(startedAt).toISOString();
|
|
267
|
+
let inputTokens = 0;
|
|
268
|
+
let outputTokens = 0;
|
|
269
|
+
let status = "success";
|
|
270
|
+
let errorMessage = null;
|
|
271
|
+
let result;
|
|
272
|
+
try {
|
|
273
|
+
result = await body((usage) => {
|
|
274
|
+
if (!usage) return;
|
|
275
|
+
const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
|
|
276
|
+
const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
|
|
277
|
+
if (Number.isFinite(i)) inputTokens = i;
|
|
278
|
+
if (Number.isFinite(o)) outputTokens = o;
|
|
279
|
+
});
|
|
280
|
+
if (result === undefined) {
|
|
281
|
+
// stream aborted/errored: the caller treats undefined as a failed run;
|
|
282
|
+
// record it as error here so the audit shows the truth.
|
|
283
|
+
status = "error";
|
|
284
|
+
errorMessage = errorMessage ?? "llm stream aborted or errored";
|
|
285
|
+
}
|
|
286
|
+
return result;
|
|
287
|
+
} catch (error) {
|
|
288
|
+
status = "error";
|
|
289
|
+
errorMessage = String(error?.message ?? error);
|
|
290
|
+
throw error;
|
|
291
|
+
} finally {
|
|
292
|
+
try {
|
|
293
|
+
service.saveLlmAudit({
|
|
294
|
+
timestamp,
|
|
295
|
+
trigger_source: spec.triggerSource,
|
|
296
|
+
operation_type: spec.operationType,
|
|
297
|
+
model_id: spec.modelId,
|
|
298
|
+
input_tokens: inputTokens,
|
|
299
|
+
output_tokens: outputTokens,
|
|
300
|
+
total_tokens: inputTokens + outputTokens,
|
|
301
|
+
cost_usd: 0,
|
|
302
|
+
duration_ms: Date.now() - startedAt,
|
|
303
|
+
status,
|
|
304
|
+
error_message: errorMessage,
|
|
305
|
+
related_memory_ids: spec.relatedMemoryIds ?? []
|
|
306
|
+
});
|
|
307
|
+
} catch (auditError) {
|
|
308
|
+
ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* Resolve the LLM route: agent default model (deployment) first, plugin config
|
|
315
|
+
* (dreamProvider/dreamModel) as fallback. Falls through to undefined when no
|
|
316
|
+
* route exists — runDream then fails safe. Fallback is logged so a silent
|
|
317
|
+
* route switch is observable.
|
|
318
|
+
*/
|
|
319
|
+
function resolveRoute(ctx, config, logger) {
|
|
320
|
+
try {
|
|
321
|
+
const sel = ctx.agentDefaultModel?.currentSelection?.();
|
|
322
|
+
if (sel?.provider && sel?.model) return { provider: sel.provider, model: sel.model };
|
|
323
|
+
logger?.warn?.("dsh-mneme dream: agentDefaultModel unavailable, falling back to config route");
|
|
324
|
+
} catch (error) {
|
|
325
|
+
logger?.warn?.(`dsh-mneme dream: agentDefaultModel lookup failed, falling back to config route: ${String(error)}`);
|
|
326
|
+
}
|
|
327
|
+
if (config.dreamProvider && config.dreamModel) return { provider: config.dreamProvider, model: config.dreamModel };
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// ------------------------------------------------------- semantic enhancement
|
|
332
|
+
// Best-effort: any failure here degrades to plain consolidation. The dream
|
|
333
|
+
// path must never be broken by an unavailable embedder/index.
|
|
334
|
+
|
|
335
|
+
/** Backfill + return vectors for every memory; null when impossible. */
|
|
336
|
+
async function collectVectors(memories, semantic) {
|
|
337
|
+
const { embedder, vectorIndex } = semantic;
|
|
338
|
+
if (!embedder || !vectorIndex || typeof embedder.embedSingle !== "function") return null;
|
|
339
|
+
const vectors = new Array(memories.length);
|
|
340
|
+
const missing = [];
|
|
341
|
+
for (let i = 0; i < memories.length; i++) {
|
|
342
|
+
const cached = vectorIndex.getEmbedding?.(memories[i].id);
|
|
343
|
+
if (cached) vectors[i] = cached;
|
|
344
|
+
else missing.push(i);
|
|
345
|
+
}
|
|
346
|
+
if (missing.length) {
|
|
347
|
+
const texts = missing.map((i) => [memories[i].title, memories[i].content].filter(Boolean).join("\n"));
|
|
348
|
+
const rows = await embedder.embed(texts);
|
|
349
|
+
missing.forEach((mi, j) => {
|
|
350
|
+
if (rows[j]?.length) {
|
|
351
|
+
vectors[mi] = rows[j];
|
|
352
|
+
vectorIndex.saveEmbedding(memories[mi].id, rows[j]);
|
|
353
|
+
}
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
return vectors.some((v) => !v) ? null : vectors;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Rebuild the vector index after dream decisions so the store and the index
|
|
361
|
+
* stay in sync: merged-away/archived/conflict-loser rows lose their vectors,
|
|
362
|
+
* the merge keeper gets a fresh one.
|
|
363
|
+
*/
|
|
364
|
+
async function maintainIndexAfterDream(decisions, service, semantic) {
|
|
365
|
+
const { embedder, vectorIndex } = semantic;
|
|
366
|
+
if (!embedder || !vectorIndex || typeof embedder.embedSingle !== "function") return;
|
|
367
|
+
const rebuild = new Map();
|
|
368
|
+
for (const d of decisions ?? []) {
|
|
369
|
+
if (d.action === "merge") {
|
|
370
|
+
for (const id of d.ids ?? []) {
|
|
371
|
+
if (id !== d.keepSource) vectorIndex.deleteEmbedding(id);
|
|
372
|
+
}
|
|
373
|
+
if (d.keepSource) {
|
|
374
|
+
const keeper = service.getById(d.keepSource);
|
|
375
|
+
if (keeper) rebuild.set(keeper.id, [keeper.title, keeper.content].filter(Boolean).join("\n"));
|
|
376
|
+
}
|
|
377
|
+
} else if (d.action === "archive" || d.action === "conflict") {
|
|
378
|
+
for (const id of d.ids ?? [d.loser]) vectorIndex.deleteEmbedding(id);
|
|
379
|
+
} else if (d.action === "update") {
|
|
380
|
+
const id = d.ids[0];
|
|
381
|
+
const mem = service.getById(id);
|
|
382
|
+
if (mem) {
|
|
383
|
+
vectorIndex.deleteEmbedding(id);
|
|
384
|
+
try {
|
|
385
|
+
const text = [mem.title, mem.content].filter(Boolean).join("\n");
|
|
386
|
+
const v = await embedder.embedSingle(text);
|
|
387
|
+
if (v?.length) vectorIndex.saveEmbedding(id, v);
|
|
388
|
+
} catch { /* best-effort */ }
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
for (const [id, text] of rebuild) {
|
|
393
|
+
try {
|
|
394
|
+
const v = await embedder.embedSingle(text);
|
|
395
|
+
if (v?.length) vectorIndex.saveEmbedding(id, v);
|
|
396
|
+
} catch { /* best-effort */ }
|
|
397
|
+
}
|
|
398
|
+
if (embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChars = 5000, delayMs = 2000, logger, semantic = null }) {
|
|
402
|
+
let pendingTimer = null;
|
|
403
|
+
let running = false;
|
|
404
|
+
let disposed = false;
|
|
405
|
+
let baseline = { count: 0, chars: 0 };
|
|
406
|
+
let inFlight = null;
|
|
407
|
+
|
|
408
|
+
function shouldTrigger(service) {
|
|
409
|
+
const memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
410
|
+
const count = memories.length;
|
|
411
|
+
const chars = totalChars(memories);
|
|
412
|
+
const overBase = count >= baseline.count + thresholdCount || chars >= baseline.chars + thresholdChars;
|
|
413
|
+
const overAbs = count >= thresholdCount || chars >= thresholdChars;
|
|
414
|
+
return { trigger: overAbs && overBase, count, chars };
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
function maybeSchedule(service) {
|
|
418
|
+
if (disposed || running || pendingTimer) return false;
|
|
419
|
+
const { trigger, count, chars } = shouldTrigger(service);
|
|
420
|
+
if (!trigger) return false;
|
|
421
|
+
pendingTimer = setTimeout(() => {
|
|
422
|
+
pendingTimer = null;
|
|
423
|
+
running = true;
|
|
424
|
+
// Defer the onRun invocation so a synchronous throw cannot escape the
|
|
425
|
+
// timer callback (which would crash the process) and skip the teardown.
|
|
426
|
+
// Errors are logged, never swallowed silently. inFlight lets dispose()
|
|
427
|
+
// await the running consolidation before the caller closes the store.
|
|
428
|
+
inFlight = Promise.resolve()
|
|
429
|
+
.then(() => (onRun ? onRun() : Promise.resolve({ ok: true, skipped: true })))
|
|
430
|
+
.then((result) => {
|
|
431
|
+
// Refresh the baseline only for a successful run (design §5.3: an
|
|
432
|
+
// LLM failure must not move the baseline, so the next write can
|
|
433
|
+
// immediately re-trigger a retry). A `{ok:false}` result or a throw
|
|
434
|
+
// keeps the old baseline. A run that reports nothing is treated as
|
|
435
|
+
// completed without failure (no-op hooks / minimal test doubles).
|
|
436
|
+
if (result && result.ok) {
|
|
437
|
+
try {
|
|
438
|
+
baseline = shouldTrigger(service);
|
|
439
|
+
} catch (error) {
|
|
440
|
+
// Store closed mid-flight: keep the last known baseline.
|
|
441
|
+
logger?.warn?.(`dsh-mneme dream: baseline refresh failed: ${String(error)}`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
})
|
|
445
|
+
.catch((error) => {
|
|
446
|
+
logger?.warn?.(`dsh-mneme dream: run failed: ${error?.message ?? error}`);
|
|
447
|
+
// Failed runs do not refresh the baseline.
|
|
448
|
+
})
|
|
449
|
+
.finally(() => {
|
|
450
|
+
running = false;
|
|
451
|
+
inFlight = null;
|
|
452
|
+
});
|
|
453
|
+
}, delayMs);
|
|
454
|
+
return true;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
async function dispose() {
|
|
458
|
+
disposed = true;
|
|
459
|
+
if (pendingTimer) { clearTimeout(pendingTimer); pendingTimer = null; }
|
|
460
|
+
// An in-flight run is left to complete naturally (its LLM calls are
|
|
461
|
+
// already paid for and aborting would discard the work). Await it so the
|
|
462
|
+
// caller can close the store only after every write has landed.
|
|
463
|
+
if (inFlight) await inFlight.catch(() => {});
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
async function runDream(ctx, service, config) {
|
|
467
|
+
const logger = ctx.logger;
|
|
468
|
+
let memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
469
|
+
if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
|
|
470
|
+
// v0.4.4 滑动窗口:只 consolidation 最近 dreamMaxSnapshotSize 条记忆,
|
|
471
|
+
// 窗口外的旧记忆不进 snapshot(大记忆量下全量快照会撑爆 LLM 输入,配合
|
|
472
|
+
// 隐式 keep 让 run 始终可收敛)。按 updated_at 倒序取前 maxSize 条。
|
|
473
|
+
const maxSize = Number.isInteger(config.dreamMaxSnapshotSize) ? config.dreamMaxSnapshotSize : 200;
|
|
474
|
+
memories = [...memories]
|
|
475
|
+
.sort((a, b) => {
|
|
476
|
+
const ta = String(a.updated_at ?? "");
|
|
477
|
+
const tb = String(b.updated_at ?? "");
|
|
478
|
+
if (ta < tb) return 1;
|
|
479
|
+
if (ta > tb) return -1;
|
|
480
|
+
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
481
|
+
})
|
|
482
|
+
.slice(0, Math.max(1, maxSize));
|
|
483
|
+
const snapshot = new Map(memories.map((m) => [m.id, m]));
|
|
484
|
+
const route = resolveRoute(ctx, config, logger);
|
|
485
|
+
const runId = randomUUID();
|
|
486
|
+
const snapshotHash = hashSnapshot([...snapshot.values()]);
|
|
487
|
+
// Conflict freeze (opt-in): when enabled, conflict decisions are parked for
|
|
488
|
+
// manual review instead of auto-adjudicated. Read once up front so the
|
|
489
|
+
// prompt hint and the apply-split agree on the same gate.
|
|
490
|
+
const freezeEnabled = config.conflictFreezeEnabled === true;
|
|
491
|
+
// Every exit (success or failure) funnels through `finish`, which writes
|
|
492
|
+
// the audit row + receipt. A record failure is logged, never thrown —
|
|
493
|
+
// auditing must not break the consolidation path. Failed runs still
|
|
494
|
+
// capture their decisions/outcome when the LLM produced a validated list
|
|
495
|
+
// (e.g. summary step failed after consolidation), so the partial write is
|
|
496
|
+
// replayable too.
|
|
497
|
+
const finish = (result) => {
|
|
498
|
+
// status is derived from what actually committed: ok only when the full
|
|
499
|
+
// decision list landed (or a summary was refreshed); noop when nothing
|
|
500
|
+
// changed; degraded when real changes landed without a summary;
|
|
501
|
+
// reconcile when decisions were validated but some did not commit (CAS
|
|
502
|
+
// conflict / rollback); failed on any LLM/validation error. No fake "ok"
|
|
503
|
+
// for an empty or partial run.
|
|
504
|
+
const status = result.status ?? (result.ok ? "ok" : "failed");
|
|
505
|
+
const applied = result.applied ?? 0;
|
|
506
|
+
const summaryStored = result.summary ?? false;
|
|
507
|
+
const receipt = buildReceipt({ runId, status, snapshotHash, inputCount: snapshot.size, applied, summaryStored });
|
|
508
|
+
try {
|
|
509
|
+
service.saveDreamRun({
|
|
510
|
+
id: runId,
|
|
511
|
+
status,
|
|
512
|
+
error: result.error,
|
|
513
|
+
provider: route?.provider,
|
|
514
|
+
model: route?.model,
|
|
515
|
+
snapshot_hash: snapshotHash,
|
|
516
|
+
input_count: snapshot.size,
|
|
517
|
+
// 裁决规则版本号:config.policyEpoch(默认 0)。规则升级后该行保留
|
|
518
|
+
// 当时的 epoch,旧裁决据此降级为历史证据(store 层 getLatestPolicyEpoch
|
|
519
|
+
// 只负责读取当前生效版本,写入由这里完成)。
|
|
520
|
+
policy_epoch: config.policyEpoch ?? 0,
|
|
521
|
+
// Full input snapshot (canonical fields) so the exact arbitration
|
|
522
|
+
// input can be rebuilt offline from the audit row alone — the
|
|
523
|
+
// digest + decisions + outcome triple makes silent errors locatable
|
|
524
|
+
// even after the store has moved on.
|
|
525
|
+
input: [...snapshot.values()].map((m) => ({
|
|
526
|
+
id: m.id,
|
|
527
|
+
type: m.type,
|
|
528
|
+
title: m.title,
|
|
529
|
+
content: m.content,
|
|
530
|
+
importance: m.importance,
|
|
531
|
+
updated_at: m.updated_at
|
|
532
|
+
})),
|
|
533
|
+
decisions: result.decisions,
|
|
534
|
+
outcome: result.outcome,
|
|
535
|
+
applied,
|
|
536
|
+
summary_stored: summaryStored,
|
|
537
|
+
receipt
|
|
538
|
+
});
|
|
539
|
+
} catch (error) {
|
|
540
|
+
logger?.warn?.(`dsh-mneme dream: failed to record audit run: ${String(error)}`);
|
|
541
|
+
}
|
|
542
|
+
return { ...result, runId, receipt, snapshotHash };
|
|
543
|
+
};
|
|
544
|
+
if (!route) {
|
|
545
|
+
logger?.warn?.("dsh-mneme dream: no llm route available");
|
|
546
|
+
return finish({ ok: false, error: "no llm route", summary: false });
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
let listText;
|
|
550
|
+
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
551
|
+
try {
|
|
552
|
+
const vectors = await collectVectors(memories, semantic);
|
|
553
|
+
if (vectors) {
|
|
554
|
+
const k = Math.min(10, Math.max(1, Math.floor(Math.sqrt(memories.length / 2))));
|
|
555
|
+
const clusters = clusterMemories(memories, vectors, k);
|
|
556
|
+
const conflicts = findPotentialConflicts(memories, vectors, 0.85);
|
|
557
|
+
const conflictIds = new Set(conflicts.flatMap((c) => [c.a.id, c.b.id]));
|
|
558
|
+
const parts = [];
|
|
559
|
+
clusters.forEach((cluster, ci) => {
|
|
560
|
+
parts.push(`# 聚类 ${ci + 1}`);
|
|
561
|
+
for (const m of cluster) {
|
|
562
|
+
parts.push(
|
|
563
|
+
`id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}` +
|
|
564
|
+
(conflictIds.has(m.id) ? " | [潜在冲突]" : "")
|
|
565
|
+
);
|
|
566
|
+
}
|
|
567
|
+
});
|
|
568
|
+
listText = parts.join("\n");
|
|
569
|
+
logger?.info?.(`[dsh-mneme] dream semantic pre-group: ${clusters.length} clusters, ${conflicts.length} conflict pairs`);
|
|
570
|
+
}
|
|
571
|
+
} catch (error) {
|
|
572
|
+
logger?.warn?.(`[dsh-mneme] dream semantic pre-group failed: ${String(error)}`);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
if (!listText) {
|
|
576
|
+
listText = [...snapshot.values()].map((m) =>
|
|
577
|
+
`id=${m.id} | type=${m.type} | importance=${m.importance} | updated=${m.updated_at} | title=${m.title} | content=${m.content}`
|
|
578
|
+
).join("\n");
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// Freeze-aware prompt: in freeze mode the conflict branch still outputs
|
|
582
|
+
// winner/loser (validation requires them) but they are treated as tentative
|
|
583
|
+
// candidates — the human makes the final call, not the model.
|
|
584
|
+
const consolidationPrompt = freezeEnabled
|
|
585
|
+
? CONSOLIDATION_PROMPT + `\n\n当前为「冲突冻结」模式:检测到内容矛盾的条目时,仍请输出 conflict,并以 winner/loser 作为候选、reason 说明理由;冲突不会被自动裁决,而会冻结待人工确认。`
|
|
586
|
+
: CONSOLIDATION_PROMPT;
|
|
587
|
+
let decisionText;
|
|
588
|
+
try {
|
|
589
|
+
// Bug8: the consolidation call is audited (tokens/time/status). A throw
|
|
590
|
+
// re-propagates to the catch below; an aborted stream returns undefined
|
|
591
|
+
// and is treated as a failed run after the check below.
|
|
592
|
+
decisionText = await runAuditedLlm(ctx, service, config, {
|
|
593
|
+
triggerSource: "autoDream",
|
|
594
|
+
operationType: "dream_consolidate",
|
|
595
|
+
modelId: `${route.provider}:${route.model}`,
|
|
596
|
+
relatedMemoryIds: [...snapshot.keys()]
|
|
597
|
+
}, (reportUsage) => streamText(ctx, {
|
|
598
|
+
provider: route.provider,
|
|
599
|
+
model: route.model,
|
|
600
|
+
purpose: "compaction",
|
|
601
|
+
maxTokens: config.dreamMaxTokens ?? 4096,
|
|
602
|
+
...(config.dreamReasoningEffort && config.dreamReasoningEffort !== "none"
|
|
603
|
+
? { reasoningEffort: config.dreamReasoningEffort }
|
|
604
|
+
: {}),
|
|
605
|
+
messages: [
|
|
606
|
+
{ role: "system", content: [{ type: "text", text: consolidationPrompt }] },
|
|
607
|
+
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
608
|
+
]
|
|
609
|
+
}, reportUsage));
|
|
610
|
+
} catch (error) {
|
|
611
|
+
logger?.warn?.(`dsh-mneme dream: consolidation llm call failed: ${String(error)}`);
|
|
612
|
+
return finish({ ok: false, error: "llm failed", summary: false });
|
|
613
|
+
}
|
|
614
|
+
if (decisionText === undefined) {
|
|
615
|
+
logger?.warn?.("dsh-mneme dream: consolidation llm stream aborted or errored");
|
|
616
|
+
return finish({ ok: false, error: "llm failed", summary: false });
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
const decisions = extractJsonArray(decisionText);
|
|
620
|
+
if (!Array.isArray(decisions)) {
|
|
621
|
+
logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0})`);
|
|
622
|
+
return finish({ ok: false, error: "no json array in llm output", summary: false });
|
|
623
|
+
}
|
|
624
|
+
const { ok, errors } = validateDecisions(decisions, snapshot, {
|
|
625
|
+
maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
|
|
626
|
+
minAgeHours: config.reflectionUpdateMinAgeHours,
|
|
627
|
+
// v0.4.4 fix:显式透传,用户配 dreamImplicitKeep:false 时严格模式必须
|
|
628
|
+
// 真正生效,dreamMinExplicitCoverage 决定隐式 keep 下的覆盖率下限。
|
|
629
|
+
dreamImplicitKeep: config.dreamImplicitKeep,
|
|
630
|
+
dreamMinExplicitCoverage: config.dreamMinExplicitCoverage
|
|
631
|
+
});
|
|
632
|
+
if (!ok) {
|
|
633
|
+
logger?.warn?.(`dsh-mneme dream: invalid decisions: ${errors.join("; ")}`);
|
|
634
|
+
return finish({ ok: false, error: `invalid decisions: ${errors.length} errors`, summary: false });
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// Capture pre-update snapshots so the audit records what each update changed.
|
|
638
|
+
const updateSnapshots = {};
|
|
639
|
+
for (const d of decisions) {
|
|
640
|
+
if (d.action === "update") {
|
|
641
|
+
const mem = snapshot.get(d.ids[0]);
|
|
642
|
+
if (mem) updateSnapshots[d.ids[0]] = { title: mem.title, content: mem.content, importance: mem.importance };
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Conflict freeze (opt-in): when enabled, conflict decisions are not
|
|
647
|
+
// auto-adjudicated — no winner kept, no loser archived. The pair is parked
|
|
648
|
+
// in conflict_pending for human review instead. Best-effort: a store
|
|
649
|
+
// failure here must never block the run (fail-safe — the memories are left
|
|
650
|
+
// untouched and nothing is arbitrated). The cap (conflictFreezeMaxPending)
|
|
651
|
+
// bounds the review queue; overflow is skipped with a warning.
|
|
652
|
+
let frozenCount = 0;
|
|
653
|
+
const frozenIds = [];
|
|
654
|
+
const applyList = freezeEnabled ? decisions.filter((d) => d.action !== "conflict") : decisions;
|
|
655
|
+
if (freezeEnabled) {
|
|
656
|
+
const conflictsToFreeze = decisions.filter((d) => d.action === "conflict");
|
|
657
|
+
if (conflictsToFreeze.length > 0) {
|
|
658
|
+
try {
|
|
659
|
+
const maxPending = Number.isInteger(config.conflictFreezeMaxPending) ? config.conflictFreezeMaxPending : 100;
|
|
660
|
+
const pendingNow = service.countConflictPending();
|
|
661
|
+
const budget = Math.max(0, maxPending - pendingNow);
|
|
662
|
+
const toFreeze = conflictsToFreeze.slice(0, budget);
|
|
663
|
+
if (conflictsToFreeze.length > budget) {
|
|
664
|
+
logger?.warn?.(`dsh-mneme dream: conflict freeze queue full (${pendingNow}/${maxPending}), skipped ${conflictsToFreeze.length - budget} conflict(s)`);
|
|
665
|
+
}
|
|
666
|
+
for (const d of toFreeze) {
|
|
667
|
+
try {
|
|
668
|
+
service.saveConflictPending({ run_id: runId, memory_a: d.winner, memory_b: d.loser, reason: d.reason });
|
|
669
|
+
frozenCount++;
|
|
670
|
+
frozenIds.push(d.winner, d.loser);
|
|
671
|
+
} catch (error) {
|
|
672
|
+
logger?.warn?.(`dsh-mneme dream: failed to freeze conflict ${d.winner}/${d.loser}: ${String(error)}`);
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
} catch (error) {
|
|
676
|
+
logger?.warn?.(`dsh-mneme dream: conflict freeze lookup failed: ${String(error)}`);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
// CAS-guarded, per-decision-transactional apply against the run snapshot:
|
|
682
|
+
// a target changed during the LLM call is skipped and reported as a
|
|
683
|
+
// conflict instead of being overwritten (item ①). Frozen conflicts are
|
|
684
|
+
// excluded from this list (they are parked, not applied).
|
|
685
|
+
const { applied, conflicts, failures, committed } = applyDecisions(applyList, service, logger, snapshot, config);
|
|
686
|
+
// Per-record receipt chain: one row per actually-committed merge/conflict/
|
|
687
|
+
// update verdict, stamped with the decision-basis digest + idempotency
|
|
688
|
+
// counters (count_before → count_after). Written here, before the run audit
|
|
689
|
+
// row, so the verdict trail always precedes the run trail it belongs to.
|
|
690
|
+
// Bookkeeping: a write failure is logged and swallowed — it must never
|
|
691
|
+
// block the consolidation flow.
|
|
692
|
+
try {
|
|
693
|
+
for (const r of buildRecordReceipts({ runId, committed, snapshot, policyEpoch: config.policyEpoch ?? 0 })) {
|
|
694
|
+
service.saveReceipt(r);
|
|
695
|
+
}
|
|
696
|
+
} catch (error) {
|
|
697
|
+
logger?.warn?.(`dsh-mneme dream: failed to write per-record receipt: ${String(error)}`);
|
|
698
|
+
}
|
|
699
|
+
// Attach the pre-update snapshot to the audit copy of each update decision
|
|
700
|
+
// so the recorded row shows the before/after delta, not just the target.
|
|
701
|
+
const auditDecisions = decisions.map((d) =>
|
|
702
|
+
d.action === "update" && updateSnapshots[d.ids[0]]
|
|
703
|
+
? { ...d, _before: updateSnapshots[d.ids[0]] }
|
|
704
|
+
: d
|
|
705
|
+
);
|
|
706
|
+
// Outcome is derived from the ACTUALLY committed sub-steps, never from the
|
|
707
|
+
// raw LLM decision list — a merge whose archive step rolled back must not
|
|
708
|
+
// claim "merge-archived" (item ②). Conflicts/failures ride along so the
|
|
709
|
+
// audit row records why the run diverged.
|
|
710
|
+
const outcome = { ...buildOutcome(committed), conflicts, failures };
|
|
711
|
+
// Frozen conflicts were not adjudicated: mark both sides pending in the
|
|
712
|
+
// per-id outcome so the audit row shows they were parked, not skipped.
|
|
713
|
+
if (frozenIds.length) {
|
|
714
|
+
for (const id of frozenIds) outcome.byId[id] = "conflict-pending";
|
|
715
|
+
}
|
|
716
|
+
// Decisions validated but not fully committed → reconcile (not ok).
|
|
717
|
+
const partial = conflicts.length > 0 || failures.length > 0;
|
|
718
|
+
// No decision landed (all-keep, or every decision skipped as an idempotent
|
|
719
|
+
// replay) → nothing substantive changed. Distinct from a success: such a
|
|
720
|
+
// run must never be reported as ok, or the audit claims work that never
|
|
721
|
+
// happened and the scheduler refreshes the baseline on a false positive.
|
|
722
|
+
// Frozen conflicts are substantive output (parked for review), so a run
|
|
723
|
+
// that only froze conflicts is not a noop.
|
|
724
|
+
const noChange = frozenCount === 0 && applied === 0 && committed.every((c) => c.action === "keep");
|
|
725
|
+
|
|
726
|
+
// Keep the vector index consistent with the post-dream store state.
|
|
727
|
+
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
728
|
+
try {
|
|
729
|
+
await maintainIndexAfterDream(applyList, service, semantic);
|
|
730
|
+
} catch (error) {
|
|
731
|
+
logger?.warn?.(`[dsh-mneme] dream index maintenance failed: ${String(error)}`);
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
// Summary generation (second LLM call). A throwing stream is reported as
|
|
736
|
+
// a failed run; summary:false marks a run that produced no summary.
|
|
737
|
+
let summaryText;
|
|
738
|
+
try {
|
|
739
|
+
// Bug8: the summary call is audited too (operation dream_summarize).
|
|
740
|
+
summaryText = await runAuditedLlm(ctx, service, config, {
|
|
741
|
+
triggerSource: "autoDream",
|
|
742
|
+
operationType: "dream_summarize",
|
|
743
|
+
modelId: `${route.provider}:${route.model}`,
|
|
744
|
+
relatedMemoryIds: []
|
|
745
|
+
}, (reportUsage) => streamText(ctx, {
|
|
746
|
+
provider: route.provider,
|
|
747
|
+
model: route.model,
|
|
748
|
+
purpose: "compaction",
|
|
749
|
+
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
750
|
+
...(config.dreamReasoningEffort && config.dreamReasoningEffort !== "none"
|
|
751
|
+
? { reasoningEffort: config.dreamReasoningEffort }
|
|
752
|
+
: {}),
|
|
753
|
+
messages: [
|
|
754
|
+
{ role: "system", content: [{ type: "text", text: SUMMARY_PROMPT }] },
|
|
755
|
+
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
|
|
756
|
+
]
|
|
757
|
+
}, reportUsage));
|
|
758
|
+
} catch (error) {
|
|
759
|
+
logger?.warn?.(`dsh-mneme dream: summary llm call failed: ${String(error)}`);
|
|
760
|
+
return finish({ ok: false, error: "llm failed", applied, decisions: auditDecisions, outcome, frozen: frozenCount, summary: false });
|
|
761
|
+
}
|
|
762
|
+
let summaryStored = false;
|
|
763
|
+
if (summaryText !== undefined && summaryText.trim()) {
|
|
764
|
+
// Bug5 carve-out: the library overview is regenerated every run, so it
|
|
765
|
+
// must REPLACE the previous overview (not append — that would grow the
|
|
766
|
+
// summary unboundedly). `_overwrite` still archives the old overview into
|
|
767
|
+
// content_history before replacing it.
|
|
768
|
+
service.saveWithDedupe({ type: "summary", title: "记忆库总览", content: summaryText.trim(), importance: 5, source: "dream", _overwrite: true });
|
|
769
|
+
summaryStored = true;
|
|
770
|
+
// Re-embed the fresh summary so the index stays in sync with the store.
|
|
771
|
+
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
772
|
+
try {
|
|
773
|
+
const summary = service.all().find((m) => m.type === "summary");
|
|
774
|
+
if (summary) {
|
|
775
|
+
const v = await semantic.embedder.embedSingle([summary.title, summary.content].filter(Boolean).join("\n"));
|
|
776
|
+
if (v?.length) semantic.vectorIndex.saveEmbedding(summary.id, v);
|
|
777
|
+
if (semantic.embedder.modelHash) semantic.vectorIndex.markModel?.(semantic.embedder.modelHash, semantic.embedder.dimension);
|
|
778
|
+
}
|
|
779
|
+
} catch { /* best-effort */ }
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
// Honest status assignment (never a fake ok):
|
|
783
|
+
// reconcile — some decisions validated but did not commit (CAS/rollback).
|
|
784
|
+
// noop — nothing changed and no summary persisted: truly an empty
|
|
785
|
+
// run. ok:false keeps the scheduler from moving the baseline.
|
|
786
|
+
// ok — either real changes landed, or a fresh summary was stored
|
|
787
|
+
// (all-keep + summary is a substantive summary refresh).
|
|
788
|
+
// degraded — real consolidation landed but the summary came back empty/
|
|
789
|
+
// missing: the store was absorbed (ok for the baseline) but
|
|
790
|
+
// the run did not produce its full output (marked, not faked).
|
|
791
|
+
let status;
|
|
792
|
+
let okResult;
|
|
793
|
+
if (partial) {
|
|
794
|
+
status = "reconcile";
|
|
795
|
+
okResult = false;
|
|
796
|
+
} else if (noChange) {
|
|
797
|
+
status = summaryStored ? "ok" : "noop";
|
|
798
|
+
okResult = summaryStored;
|
|
799
|
+
} else {
|
|
800
|
+
status = summaryStored ? "ok" : "degraded";
|
|
801
|
+
okResult = true;
|
|
802
|
+
}
|
|
803
|
+
return finish({
|
|
804
|
+
ok: okResult,
|
|
805
|
+
status,
|
|
806
|
+
applied,
|
|
807
|
+
decisions: auditDecisions,
|
|
808
|
+
outcome,
|
|
809
|
+
conflicts,
|
|
810
|
+
failures,
|
|
811
|
+
frozen: frozenCount,
|
|
812
|
+
summary: summaryStored
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
return { maybeSchedule, runDream, dispose };
|
|
817
|
+
}
|