@modusensus/dsh-mneme 0.8.2 → 0.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +718 -633
  3. package/bin/cli.mjs +0 -0
  4. package/bin/dsh-mneme-mcp.mjs +502 -0
  5. package/cordis.patch.yml +21 -21
  6. package/lib/api-standalone.js +495 -316
  7. package/lib/api.js +1382 -1316
  8. package/lib/bootstrap.js +211 -0
  9. package/lib/client.js +4475 -4362
  10. package/lib/commands.js +95 -95
  11. package/lib/config.js +533 -500
  12. package/lib/dream/clustering.js +118 -118
  13. package/lib/dream/decisions.js +658 -658
  14. package/lib/dream/narratives.js +44 -0
  15. package/lib/dream/sleep.js +700 -700
  16. package/lib/dream.js +1319 -1216
  17. package/lib/embedding.js +177 -177
  18. package/lib/entities/extractor.js +249 -249
  19. package/lib/heat.js +142 -136
  20. package/lib/hot-memory.js +46 -46
  21. package/lib/index.js +515 -513
  22. package/lib/inject.js +299 -253
  23. package/lib/lang.js +202 -179
  24. package/lib/local-embedder.js +320 -308
  25. package/lib/mirror.js +198 -198
  26. package/lib/quality-filter.js +126 -126
  27. package/lib/recall-stats.js +103 -0
  28. package/lib/reranker.js +269 -260
  29. package/lib/runtime/adopt.js +176 -176
  30. package/lib/runtime/closure.js +172 -172
  31. package/lib/runtime/download.js +247 -247
  32. package/lib/runtime/layout.js +235 -235
  33. package/lib/runtime/loader.js +249 -249
  34. package/lib/runtime/model-fetch.js +361 -0
  35. package/lib/runtime/provision.js +237 -237
  36. package/lib/runtime/tarball.js +130 -130
  37. package/lib/runtime/verify.js +191 -191
  38. package/lib/scope.js +118 -118
  39. package/lib/search/adaptive.js +22 -22
  40. package/lib/search/bm25.js +96 -96
  41. package/lib/service.js +2171 -2071
  42. package/lib/settings.js +430 -419
  43. package/lib/store.js +2448 -2367
  44. package/lib/summarize.js +575 -449
  45. package/lib/tools.js +574 -574
  46. package/lib/vector-index.js +110 -110
  47. package/lib/version-check.js +122 -122
  48. package/package.json +109 -108
  49. package/runtime-manifest.json +596 -596
  50. package/scripts/benchmark-embed.js +201 -201
  51. package/scripts/benchmark-recall.js +199 -199
  52. package/scripts/benchmark-rerank.js +166 -166
  53. package/scripts/build-runtime-manifest.mjs +217 -217
  54. package/scripts/check-sync.js +42 -42
  55. package/scripts/e2e-dsh.js +220 -220
  56. package/scripts/mneme-runtime.mjs +277 -277
  57. package/scripts/stress-dsh.js +255 -255
  58. package/scripts/sync-lib.js +47 -47
  59. package/scripts/sync-test-badge.mjs +61 -61
  60. package/src/api-standalone.js +495 -316
  61. package/src/api.js +1382 -1316
  62. package/src/bootstrap.js +211 -0
  63. package/src/commands.js +95 -95
  64. package/src/config.js +533 -500
  65. package/src/dream/clustering.js +118 -118
  66. package/src/dream/decisions.js +658 -658
  67. package/src/dream/narratives.js +44 -0
  68. package/src/dream/sleep.js +700 -700
  69. package/src/dream.js +1319 -1216
  70. package/src/embedding.js +177 -177
  71. package/src/entities/extractor.js +249 -249
  72. package/src/heat.js +142 -136
  73. package/src/hot-memory.js +46 -46
  74. package/src/index.js +515 -513
  75. package/src/inject.js +299 -253
  76. package/src/lang.js +202 -179
  77. package/src/local-embedder.js +320 -308
  78. package/src/mirror.js +198 -198
  79. package/src/quality-filter.js +126 -126
  80. package/src/recall-stats.js +103 -0
  81. package/src/reranker.js +269 -260
  82. package/src/runtime/adopt.js +176 -176
  83. package/src/runtime/closure.js +172 -172
  84. package/src/runtime/download.js +247 -247
  85. package/src/runtime/layout.js +235 -235
  86. package/src/runtime/loader.js +249 -249
  87. package/src/runtime/model-fetch.js +361 -0
  88. package/src/runtime/provision.js +237 -237
  89. package/src/runtime/tarball.js +130 -130
  90. package/src/runtime/verify.js +191 -191
  91. package/src/scope.js +118 -118
  92. package/src/search/adaptive.js +22 -22
  93. package/src/search/bm25.js +96 -96
  94. package/src/service.js +2171 -2071
  95. package/src/settings.js +430 -419
  96. package/src/store.js +2448 -2367
  97. package/src/summarize.js +575 -449
  98. package/src/tools.js +574 -574
  99. package/src/vector-index.js +110 -110
  100. package/src/version-check.js +122 -122
  101. package/test/api-routes-gaps.test.js +218 -218
  102. package/test/api.test.js +1578 -1481
  103. package/test/audit.test.js +448 -448
  104. package/test/benchmark.test.js +94 -94
  105. package/test/bootstrap.test.js +220 -0
  106. package/test/client.test.js +799 -736
  107. package/test/clustering.test.js +100 -100
  108. package/test/commands.test.js +106 -106
  109. package/test/config.test.js +66 -66
  110. package/test/conflict-freeze.test.js +300 -300
  111. package/test/dream-archive-guardrail.test.js +116 -116
  112. package/test/dream-candidate-hybrid.test.js +281 -281
  113. package/test/dream-coverage-degrade.test.js +86 -86
  114. package/test/dream.test.js +1257 -1254
  115. package/test/embedding-ready.test.js +196 -196
  116. package/test/entities.test.js +547 -547
  117. package/test/entity-recall.test.js +179 -0
  118. package/test/epistemic.test.js +298 -298
  119. package/test/fnew-0112.test.js +311 -311
  120. package/test/fnew-03.test.js +422 -422
  121. package/test/graph-api.test.js +175 -175
  122. package/test/heat.test.js +175 -148
  123. package/test/helpers/dream-mock.js +82 -82
  124. package/test/helpers/peer-worker.mjs +25 -25
  125. package/test/helpers/runtime-source.js +60 -60
  126. package/test/helpers/tar-builder.js +80 -80
  127. package/test/hot-memory.test.js +145 -145
  128. package/test/inject-content-cap.test.js +89 -0
  129. package/test/inject-first-round-bm25.test.js +89 -89
  130. package/test/inject-heat.test.js +61 -0
  131. package/test/inject-rotation.test.js +161 -0
  132. package/test/inject.test.js +249 -249
  133. package/test/lang.test.js +87 -87
  134. package/test/lib-smoke.test.js +96 -96
  135. package/test/llm-audit.test.js +309 -309
  136. package/test/llm-message-source.test.js +177 -177
  137. package/test/local-embedder.test.js +249 -229
  138. package/test/mcp-stdio.test.js +385 -0
  139. package/test/mirror-dirty.test.js +424 -424
  140. package/test/mirror-edit-digest.test.js +187 -187
  141. package/test/mirror-generation.test.js +499 -499
  142. package/test/mirror.test.js +249 -249
  143. package/test/narrative-bars.test.js +162 -0
  144. package/test/peer-blockers.test.js +193 -193
  145. package/test/policy-epoch.test.js +259 -259
  146. package/test/quality-filter.test.js +186 -186
  147. package/test/reasoning-effort.test.js +746 -746
  148. package/test/recall-evals.test.js +235 -235
  149. package/test/recall-layer.test.js +326 -326
  150. package/test/recall-stats.test.js +117 -0
  151. package/test/receipt-chain.test.js +451 -451
  152. package/test/reflection.test.js +226 -226
  153. package/test/reindex-backfill.test.js +62 -62
  154. package/test/reranker-single-logit.test.js +73 -73
  155. package/test/reranker.test.js +197 -197
  156. package/test/resident-status.test.js +83 -0
  157. package/test/runtime-adopt.test.js +206 -206
  158. package/test/runtime-cli.test.js +89 -89
  159. package/test/runtime-closure.test.js +193 -193
  160. package/test/runtime-download.test.js +302 -302
  161. package/test/runtime-layout.test.js +163 -163
  162. package/test/runtime-loader.test.js +237 -237
  163. package/test/runtime-manifest.test.js +158 -158
  164. package/test/runtime-model-fetch.test.js +348 -0
  165. package/test/runtime-provision.test.js +101 -101
  166. package/test/runtime-tarball.test.js +81 -81
  167. package/test/runtime-verify.test.js +214 -214
  168. package/test/scope-candidates.test.js +239 -239
  169. package/test/scope-explicit.test.js +452 -452
  170. package/test/scope-retrieval.test.js +197 -197
  171. package/test/scope-storage.test.js +255 -255
  172. package/test/scope-strict.test.js +231 -231
  173. package/test/search-fusion.test.js +90 -90
  174. package/test/semantic.test.js +124 -124
  175. package/test/service-search.test.js +174 -174
  176. package/test/service.test.js +405 -405
  177. package/test/settings.test.js +261 -261
  178. package/test/skip-invalid-restore.test.js +262 -262
  179. package/test/sleep-action-set.test.js +422 -422
  180. package/test/sleep-heat.test.js +126 -125
  181. package/test/sleep-scheduler-rearm.test.js +89 -89
  182. package/test/sleep.test.js +373 -373
  183. package/test/standalone-api.test.js +544 -359
  184. package/test/store-vector-cache.test.js +98 -98
  185. package/test/store.test.js +383 -383
  186. package/test/stress.test.js +209 -209
  187. package/test/summarize.test.js +884 -575
  188. package/test/tools-memory-get-schema.test.js +65 -65
  189. package/test/tools.test.js +341 -341
  190. package/test/updated-at-semantics.test.js +113 -113
  191. package/test/vector-index.test.js +221 -221
  192. package/test/version-check.test.js +152 -152
  193. package/test/webServer-optional.test.js +82 -82
package/lib/summarize.js CHANGED
@@ -1,449 +1,575 @@
1
- import { BlockAssembler, createUserMessage } from "@deepseek-ai/dsh-llm";
2
- import { STR, langOf } from "./lang.js";
3
-
4
- // 编码记忆蒸馏 prompt(codingRetrospect 开启时启用):在通用记忆之外,额外提取
5
- // 三类编码专属记忆,专治重复踩坑 / 遗忘被否决方案 / 丢失工程约束。字段仍沿用
6
- // title/content 单列结构(store 无结构化字段),信息浓缩进 content。
7
- /** Extract a JSON array from LLM output that may contain prose around it. */
8
- export function parseSummaryJson(raw) {
9
- const text = String(raw ?? "");
10
- const start = text.indexOf("[");
11
- const end = text.lastIndexOf("]");
12
- if (start === -1 || end === -1 || end <= start) return [];
13
- let arr;
14
- try {
15
- arr = JSON.parse(text.slice(start, end + 1));
16
- } catch {
17
- return [];
18
- }
19
- if (!Array.isArray(arr)) return [];
20
- const VALID = new Set(["preference", "project", "decision", "history", "rejected_solution", "pitfall", "constraint"]);
21
- return arr.filter(
22
- (item) =>
23
- item &&
24
- typeof item === "object" &&
25
- VALID.has(item.type) &&
26
- typeof item.title === "string" &&
27
- item.title.trim() &&
28
- typeof item.content === "string" &&
29
- item.content.trim()
30
- ).map((item) => ({
31
- type: item.type,
32
- title: item.title.trim(),
33
- content: item.content.trim(),
34
- importance: Number.isInteger(item.importance) ? Math.min(5, Math.max(1, item.importance)) : 3
35
- }));
36
- }
37
-
38
- // The dsh-llm StreamChunk protocol BlockAssembler.push() consumes:
39
- // block-start {index, blockType}, text-delta {index, text},
40
- // block-end {index, block}, finish {reason}. Some consumers observe a
41
- // looser shape ({block} / {delta} / {kind}); normalize before pushing so
42
- // both real adapter streams and shape-tolerant test doubles assemble.
43
- const STREAM_CHUNK_TYPES = new Set([
44
- "block-start",
45
- "text-delta",
46
- "reasoning-delta",
47
- "tool-call-delta",
48
- "block-end",
49
- "usage",
50
- "finish"
51
- ]);
52
-
53
- function toProtocolChunk(chunk) {
54
- switch (chunk.type) {
55
- case "block-start":
56
- return { type: "block-start", index: chunk.index ?? 0, blockType: chunk.blockType ?? chunk.block?.type ?? "text" };
57
- case "text-delta":
58
- return { type: "text-delta", index: chunk.index ?? 0, text: chunk.text ?? chunk.delta ?? "" };
59
- case "reasoning-delta":
60
- return { type: "reasoning-delta", index: chunk.index ?? 0, text: chunk.text ?? chunk.delta ?? "" };
61
- case "block-end":
62
- return { type: "block-end", index: chunk.index ?? 0, block: chunk.block ?? { type: "text" } };
63
- case "finish":
64
- return {
65
- type: "finish",
66
- reason: chunk.reason ?? { kind: chunk.kind === "error" ? "error" : "stop" },
67
- replayState: chunk.replayState
68
- };
69
- default:
70
- return chunk;
71
- }
72
- }
73
-
74
- // Only direct human prompts are summarized: plugin-injected context
75
- // (AGENTS.md, skill bodies, file-change notices) and other machine-originated
76
- // events must not leak into the memory store. Events without a data payload
77
- // (minimal test doubles) pass the kind check and are handled by the content
78
- // check below.
79
- //
80
- // codingRetrospect: the distill context is the FULL turn transcript —
81
- // user prompts plus assistant public replies, tool calls + results and code
82
- // dispatch output — so the summarizer can see tool errors and extract pitfall
83
- // root causes, not just what the user typed. The same filtering stays: only
84
- // source.kind === "user" prompts enter (plugin/machine content is excluded).
85
- // The result is a single text transcript passed to the LLM as one user message
86
- // (SUMMARY_PROMPT already says "根据下面的会话内容").
87
- //
88
- // Privacy: assistant `reasoning` (private thought) blocks are deliberately NOT
89
- // collected — distilled memories must never sink private reasoning chains.
90
- // Only public text blocks (type "text") reach the summarizer.
91
- function collectMessages(session, maxChars = 8000, language = "zh") {
92
- // DSH 0.1.2-rc.1 起 Session 改用 snapshotEvents(),兼容旧版 .events
93
- const events = session.snapshotEvents?.() ?? session.events ?? [];
94
- const lines = [];
95
- // 兼容严格形状 [{type:"text",text}] 与宽松形状 ["字符串", ...](lib-smoke 用例
96
- // 直接传字符串数组)。只取公开文本块;reasoning 私有推理块不进蒸馏上下文。
97
- const textOf = (content) => {
98
- if (typeof content === "string") return content;
99
- if (!Array.isArray(content)) return "";
100
- return content
101
- .map((block) => (typeof block === "string" ? block : (block && block.type === "text" && typeof block.text === "string" ? block.text : "")))
102
- .filter((s) => s)
103
- .join("\n");
104
- };
105
- const trim = (s, n) => (typeof s === "string" && s.length > n ? `${s.slice(0, n)}…` : s);
106
- for (const event of events) {
107
- const data = event?.data ?? {};
108
- const kind = data?.source?.kind;
109
- switch (event.type) {
110
- case "user/message": {
111
- if (kind !== undefined && kind !== "user") break;
112
- const text = textOf(data?.content);
113
- if (text.trim()) lines.push(STR.transcriptUser[language](text));
114
- break;
115
- }
116
- case "assistant/message": {
117
- const msg = data?.message;
118
- const blocks = Array.isArray(msg?.content) ? msg.content : [];
119
- const text = textOf(blocks);
120
- if (text.trim()) lines.push(STR.transcriptAssistant[language](text));
121
- // 私有推理块(reasoning)刻意不采集:蒸馏记忆不得沉淀模型私有思考链。
122
- break;
123
- }
124
- case "tool/call": {
125
- const args = typeof data.arguments === "string"
126
- ? data.arguments
127
- : data.arguments ? JSON.stringify(data.arguments) : "";
128
- lines.push(STR.transcriptToolCall[language](data.name ?? "?", trim(args, 300)));
129
- break;
130
- }
131
- case "tool/result": {
132
- const out = typeof data.output === "string" ? data.output : data.output ? JSON.stringify(data.output) : "";
133
- const status = data.ok === false ? STR.statusFail[language] : STR.statusOk[language];
134
- lines.push(STR.transcriptToolResult[language](status, trim(out, 500)));
135
- break;
136
- }
137
- case "tool/code-dispatch": {
138
- const out = typeof data.output === "string" ? data.output : data.output ? JSON.stringify(data.output) : "";
139
- const status = data.ok === false ? STR.statusFail[language] : STR.statusOk[language];
140
- lines.push(STR.transcriptCode[language](status, trim(out, 500)));
141
- break;
142
- }
143
- default:
144
- break;
145
- }
146
- }
147
- return lines.length ? [createUserMessage({ content: [{ type: "text", text: trim(lines.join("\n"), maxChars) }], source: { kind: "plugin", plugin: "dsh-mneme" } })] : [];
148
- }
149
-
150
- // ── 智能调速器(429 保护)─────────────────────────────────────────────
151
- // 对话一多时 turn/end 会批量触发蒸馏,多个 LLM 请求"一拥而上"正是 429 的
152
- // 来源。这里借鉴机场安检的思路:所有蒸馏调用进同一个全局串行队列,按间隔
153
- // distillRateLimitIntervalMs 分批放行;命中 429 时按 distillRateLimitBaseDelayMs
154
- // 指数退避(1s→2s→4s…)自动重试 distillRateLimitRetries 次,全程对用户透明,
155
- // 不把 429 错误码直接抛出去。
156
- const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
157
-
158
- function isRateLimited(error) {
159
- const status = error?.status ?? error?.statusCode ?? error?.response?.status;
160
- if (status === 429) return true;
161
- const msg = String(error?.message ?? error ?? "");
162
- return /429|rate.?limit|too many requests|请求过于频繁/i.test(msg);
163
- }
164
-
165
- // 全局串行链:每个蒸馏任务在前一个结束后才开始,单次失败不阻塞后续。
166
- // 相邻请求间隔按"距上一个结束不足 intervalMs 就补齐等待"实现——单次蒸馏
167
- // 零延迟(上次结束距今已超过间隔,直接放行),只有连续批量蒸馏才触发限速。
168
- let distillQueue = Promise.resolve();
169
- let lastDistillEnd = 0;
170
- function enqueueDistill(task, intervalMs = 0) {
171
- const run = distillQueue.then(async () => {
172
- if (intervalMs > 0) {
173
- const wait = Math.max(0, intervalMs - (Date.now() - lastDistillEnd));
174
- if (wait > 0) await sleep(wait);
175
- }
176
- lastDistillEnd = Date.now();
177
- return task();
178
- });
179
- distillQueue = run.catch(() => {});
180
- return run;
181
- }
182
-
183
- export function createSummarizer(ctx, service, config) {
184
- if (!config.autoSummarize) return { dispose: () => {} };
185
-
186
- const inFlight = new Map();
187
- // Issue #127:per-session 上一次实际开跑时刻(最小间隔闸门用)。与 inFlight
188
- // 同生命周期,dispose 时一并清空。
189
- const lastRunAt = new Map();
190
- let disposed = false;
191
-
192
- /** 写一条 autoSummarize 的 LLM 审计行(审计失败绝不阻塞蒸馏本身)。 */
193
- function writeAudit(entry) {
194
- if (config?.llmAudit?.enabled === false || typeof service.saveLlmAudit !== "function") return;
195
- try {
196
- service.saveLlmAudit({
197
- trigger_source: "autoSummarize",
198
- operation_type: "summarize_compress",
199
- related_memory_ids: [],
200
- ...entry
201
- });
202
- } catch (auditError) {
203
- ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
204
- }
205
- }
206
-
207
- async function summarize(session) {
208
- if (disposed || inFlight.has(session.id)) return;
209
-
210
- const header = session.requestHeader?.()?.config;
211
- // Config override takes priority, then session header, then nothing.
212
- const route = (config.summarizeProvider && config.summarizeModel)
213
- ? { provider: config.summarizeProvider, model: config.summarizeModel }
214
- : (header?.provider && header?.model)
215
- ? { provider: header.provider, model: header.model }
216
- : undefined;
217
-
218
- // Issue #127 间隔门:同一会话两次蒸馏之间的最小间隔(0 = 现状,不限)。进入
219
- // 即打点——失败/degraded 的 run 也占用间隔,与 dreamMinIntervalMinutes 语义
220
- // 一致,防止失败连发。被挡下的 turn/end 静默跳过,但留一行 status='skipped'
221
- // 审计,否则「这一轮为什么没蒸馏」对用户完全不可观测。
222
- const gapMs = (config.summarizeMinIntervalMinutes ?? 0) * 60000;
223
- const prevClaim = lastRunAt.get(session.id);
224
- if (gapMs > 0 && Date.now() - (prevClaim ?? 0) < gapMs) {
225
- writeAudit({
226
- timestamp: new Date().toISOString(),
227
- model_id: route ? `${route.provider}:${route.model}` : "unknown",
228
- status: "skipped",
229
- error_message: "min-interval"
230
- });
231
- return;
232
- }
233
- lastRunAt.set(session.id, Date.now());
234
-
235
- const controller = new AbortController();
236
- inFlight.set(session.id, controller);
237
- // audit state for the compression call. null = no audit for this run
238
- // (disabled, or no LLM call was actually made). The audit row is written in
239
- // the finally below — once, regardless of which exit path the call took —
240
- // so a failed/aborted stream still leaves a status='error' trail without
241
- // ever blocking the summarization itself.
242
- let audit = null;
243
- let abortedRun = false;
244
- try {
245
- if (!route) return;
246
- // 完整转录(Codex 式):蒸馏把整轮对话交给 LLM 提炼原子记忆,不再硬裁
247
- // 8000 字截断语义;上限由 distillMaxChars 控制(默认 24000,可调大)。
248
- const messages = collectMessages(session, config.distillMaxChars ?? 24000, langOf(config));
249
- if (!messages.length) return;
250
-
251
- if (config?.llmAudit?.enabled !== false && typeof service.saveLlmAudit === "function") {
252
- audit = {
253
- route,
254
- timestamp: new Date().toISOString(),
255
- startedAt: Date.now(),
256
- inputTokens: 0,
257
- outputTokens: 0,
258
- status: "success",
259
- errorMessage: null
260
- };
261
- }
262
-
263
- const options = {
264
- provider: route.provider,
265
- model: route.model,
266
- purpose: "summarization",
267
- messages: [
268
- { role: "system", content: [{ type: "text", text: config.codingRetrospect ? STR.prompts.codingSummary[langOf(config)] : STR.prompts.summary[langOf(config)] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
269
- ...messages
270
- ],
271
- signal: controller.signal
272
- };
273
- // 智能调速器:整段蒸馏 LLM 调用进全局串行队列,按间隔分批放行;429 时
274
- // 指数退避自动重试,全程对用户透明,不把 429 错误码直接抛出去。
275
- const intervalMs = config.distillRateLimitIntervalMs ?? 1000;
276
- const { text, assembledText, aborted } = await enqueueDistill(async () => {
277
- const retries = config.distillRateLimitRetries ?? 3;
278
- const baseDelayMs = config.distillRateLimitBaseDelayMs ?? 1000;
279
- for (let attempt = 0; ; attempt++) {
280
- const assembler = new BlockAssembler();
281
- let text = "";
282
- let aborted = false;
283
- try {
284
- for await (const chunk of ctx.llm.stream(options)) {
285
- if (STREAM_CHUNK_TYPES.has(chunk.type)) assembler.push(toProtocolChunk(chunk));
286
- if (chunk.type === "text-delta") {
287
- text += chunk.text ?? chunk.delta ?? "";
288
- }
289
- if (chunk.type === "usage" && audit) {
290
- const i = chunk.input_tokens ?? chunk.inputTokens ?? chunk.prompt_tokens ?? chunk.promptTokens;
291
- const o = chunk.output_tokens ?? chunk.outputTokens ?? chunk.completion_tokens ?? chunk.completionTokens;
292
- if (Number.isFinite(i)) audit.inputTokens = i;
293
- if (Number.isFinite(o)) audit.outputTokens = o;
294
- }
295
- if (chunk.type === "finish") {
296
- const reasonKind = chunk.reason?.kind ?? chunk.kind;
297
- if (reasonKind === "error" || reasonKind === "aborted") {
298
- // 429 也可能以 finish reason error 携带 rate-limit 信息,统一
299
- // 转抛错走指数退避重试。
300
- if (isRateLimited(chunk.reason ?? chunk)) {
301
- throw Object.assign(new Error("rate limited"), { status: 429 });
302
- }
303
- if (audit) {
304
- audit.status = "error";
305
- audit.errorMessage = `llm stream ${reasonKind}`;
306
- }
307
- aborted = true;
308
- break;
309
- }
310
- }
311
- }
312
- // Direct delta accumulation is the primary extraction path (it works
313
- // for real protocol chunks {index,text} and looser {delta} shapes
314
- // alike); the assembler blocks are a fallback for streams that only
315
- // deliver text inside block-end. This dsh-llm exposes no public
316
- // no-arg assemble() — blocks() is the message-level API.
317
- const blocks = assembler.blocks();
318
- const assembledText = blocks
319
- .filter((b) => b.type === "text")
320
- .map((b) => b.text ?? "")
321
- .join("");
322
- return { text, assembledText, aborted };
323
- } catch (error) {
324
- if (error?.name === "AbortError" || controller.signal.aborted) throw error; // dispose 中止直接放行
325
- if (isRateLimited(error) && attempt < retries) {
326
- const delay = baseDelayMs * 2 ** attempt;
327
- ctx.logger?.warn?.(
328
- `dsh-mneme: 蒸馏请求过于频繁(429),为避免限流等待 ${delay}ms 后自动重试(第 ${attempt + 1}/${retries} 次)`
329
- );
330
- await sleep(delay);
331
- continue;
332
- }
333
- // 429 重试耗尽或非 429 错误:记 audit 后抛出,保持原失败路径。
334
- if (audit) {
335
- audit.status = "error";
336
- audit.errorMessage = String(error?.message ?? error);
337
- }
338
- throw error;
339
- }
340
- }
341
- }, intervalMs);
342
- if (aborted) {
343
- abortedRun = true;
344
- return;
345
- }
346
- const parsed = parseSummaryJson(text || assembledText);
347
- // Issue #127 条数上限:0 = 不限(现状)。被截断的条数与去重命中数一并进
348
- // 审计 metadata——此前产出条数完全由模型输出决定,用户无从自查。
349
- const cap = config.summarizeMaxEntriesPerRun ?? 0;
350
- const entries = cap > 0 ? parsed.slice(0, cap) : parsed;
351
- const capped = parsed.length - entries.length;
352
- const dedupeMode = config.summarizeDedupeMode ?? "off";
353
- let deduped = 0;
354
- let dedupeMaxSim = 0;
355
- for (const entry of entries) {
356
- // Provenance: the summarizer runs on a real session (turn/end hook), so
357
- // session.id is always available here — it rides the human-readable
358
- // source label.
359
- // 编码记忆类型(codingRetrospect)不带 tag:读取侧门控/加权靠 m.type
360
- // (rejected_solution/pitfall/constraint)区分即可,tag 体系
361
- // sanitizeTags 不认 `type:` 前缀反而会清空 tags 列(额外一次 UPDATE)。
362
- const source = `session:${session.id}`;
363
- // Issue #127 落库前去重(opt-in,默认 off = 现状):命中则并入既有条目
364
- // (saveWithDedupe 的 _mergeInto 分支),不再新造一行。作用域 = 同会话 +
365
- // 时间窗;跨会话的同类事实仍交给 autoDream / sleep 的批量裁决。
366
- const dup = (dedupeMode !== "off" && typeof service.findSessionDuplicate === "function")
367
- ? await service.findSessionDuplicate(
368
- { ...entry, source },
369
- {
370
- mode: dedupeMode,
371
- minSim: config.summarizeDedupeMinSim ?? 0.92,
372
- windowHours: config.summarizeDedupeWindowHours ?? 24,
373
- source
374
- }
375
- )
376
- : undefined;
377
- if (dup) {
378
- deduped++;
379
- dedupeMaxSim = Math.max(dedupeMaxSim, dup.sim ?? 0);
380
- }
381
- service.saveWithDedupe({
382
- ...entry,
383
- source,
384
- ...(dup ? { _mergeInto: dup.memory.id } : {})
385
- });
386
- }
387
- if (audit && (capped > 0 || deduped > 0)) {
388
- audit.metadata = {
389
- parsed: parsed.length,
390
- ...(capped > 0 ? { capped } : {}),
391
- ...(deduped > 0 ? { deduped, mode: dedupeMode, maxSim: Number(dedupeMaxSim.toFixed(4)) } : {})
392
- };
393
- }
394
- } finally {
395
- // Issue #127:aborted(会话关闭 / 插件 dispose)不占间隔,避免误伤该会话的
396
- // 下一次蒸馏;其余情况(含失败)的打点保留,与 dreamMinIntervalMinutes 一致。
397
- if (abortedRun || controller.signal.aborted) {
398
- if (prevClaim === undefined) lastRunAt.delete(session.id);
399
- else lastRunAt.set(session.id, prevClaim);
400
- }
401
- if (audit) {
402
- try {
403
- service.saveLlmAudit({
404
- timestamp: audit.timestamp,
405
- trigger_source: "autoSummarize",
406
- operation_type: "summarize_compress",
407
- model_id: `${audit.route.provider}:${audit.route.model}`,
408
- input_tokens: audit.inputTokens,
409
- output_tokens: audit.outputTokens,
410
- total_tokens: audit.inputTokens + audit.outputTokens,
411
- cost_usd: 0,
412
- duration_ms: Date.now() - audit.startedAt,
413
- status: audit.status,
414
- error_message: audit.errorMessage,
415
- related_memory_ids: [],
416
- // Issue #127:parsed / capped / deduped 的观测数据。零 schema 变更
417
- // (llm_audit_logs.metadata 已存在),一条 SQL 即可验证节流是否生效。
418
- metadata: audit.metadata
419
- });
420
- } catch (auditError) {
421
- ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
422
- }
423
- }
424
- inFlight.delete(session.id);
425
- }
426
- }
427
-
428
- const unsubscribe = ctx.on("session/event", (session, event) => {
429
- if (disposed || event.type !== "turn/end") return;
430
- // Return the summarization promise so awaiters observe the writes; the
431
- // catch keeps listener dispatch from rejecting. Dispose-initiated aborts
432
- // and external AbortErrors are silent.
433
- return summarize(session).catch((error) => {
434
- if (disposed || error?.name === "AbortError") return;
435
- ctx.logger?.warn?.(`dsh-mneme: summarization failed: ${String(error)}`);
436
- });
437
- });
438
-
439
- return {
440
- dispose() {
441
- if (disposed) return;
442
- disposed = true;
443
- unsubscribe?.();
444
- for (const controller of inFlight.values()) controller.abort();
445
- inFlight.clear();
446
- lastRunAt.clear();
447
- }
448
- };
449
- }
1
+ import { BlockAssembler, createUserMessage } from "@deepseek-ai/dsh-llm";
2
+ import { STR, langOf } from "./lang.js";
3
+
4
+ // 编码记忆蒸馏 prompt(codingRetrospect 开启时启用):在通用记忆之外,额外提取
5
+ // 三类编码专属记忆,专治重复踩坑 / 遗忘被否决方案 / 丢失工程约束。字段仍沿用
6
+ // title/content 单列结构(store 无结构化字段),信息浓缩进 content。
7
+ /** 解析 LLM 输出中的 JSON 数组,并保留数组是否有效的结果。 */
8
+ function parseSummaryJsonResult(raw) {
9
+ const text = String(raw ?? "");
10
+ const start = text.indexOf("[");
11
+ const end = text.lastIndexOf("]");
12
+ if (start === -1 || end === -1 || end <= start) return { ok: false, entries: [] };
13
+ let arr;
14
+ try {
15
+ arr = JSON.parse(text.slice(start, end + 1));
16
+ } catch {
17
+ return { ok: false, entries: [] };
18
+ }
19
+ if (!Array.isArray(arr)) return { ok: false, entries: [] };
20
+ const VALID = new Set(["preference", "project", "decision", "history", "rejected_solution", "pitfall", "constraint"]);
21
+ const entries = arr.filter(
22
+ (item) =>
23
+ item &&
24
+ typeof item === "object" &&
25
+ VALID.has(item.type) &&
26
+ typeof item.title === "string" &&
27
+ item.title.trim() &&
28
+ typeof item.content === "string" &&
29
+ item.content.trim()
30
+ ).map((item) => ({
31
+ type: item.type,
32
+ title: item.title.trim(),
33
+ content: item.content.trim(),
34
+ importance: Number.isInteger(item.importance) ? Math.min(5, Math.max(1, item.importance)) : 3
35
+ }));
36
+ // 空数组表示模型明确判断本轮没有可沉淀内容;非空数组若全部无效,
37
+ // 则不能消费窗口,否则无效输出会永久推进 seq 游标。
38
+ return { ok: arr.length === 0 || entries.length > 0, entries };
39
+ }
40
+
41
+ /** Extract a JSON array from LLM output that may contain prose around it. */
42
+ export function parseSummaryJson(raw) {
43
+ return parseSummaryJsonResult(raw).entries;
44
+ }
45
+
46
+ // The dsh-llm StreamChunk protocol BlockAssembler.push() consumes:
47
+ // block-start {index, blockType}, text-delta {index, text},
48
+ // block-end {index, block}, finish {reason}. Some consumers observe a
49
+ // looser shape ({block} / {delta} / {kind}); normalize before pushing so
50
+ // both real adapter streams and shape-tolerant test doubles assemble.
51
+ const STREAM_CHUNK_TYPES = new Set([
52
+ "block-start",
53
+ "text-delta",
54
+ "reasoning-delta",
55
+ "tool-call-delta",
56
+ "block-end",
57
+ "usage",
58
+ "finish"
59
+ ]);
60
+
61
+ function toProtocolChunk(chunk) {
62
+ switch (chunk.type) {
63
+ case "block-start":
64
+ return { type: "block-start", index: chunk.index ?? 0, blockType: chunk.blockType ?? chunk.block?.type ?? "text" };
65
+ case "text-delta":
66
+ return { type: "text-delta", index: chunk.index ?? 0, text: chunk.text ?? chunk.delta ?? "" };
67
+ case "reasoning-delta":
68
+ return { type: "reasoning-delta", index: chunk.index ?? 0, text: chunk.text ?? chunk.delta ?? "" };
69
+ case "block-end":
70
+ return { type: "block-end", index: chunk.index ?? 0, block: chunk.block ?? { type: "text" } };
71
+ case "finish":
72
+ return {
73
+ type: "finish",
74
+ reason: chunk.reason ?? { kind: chunk.kind === "error" ? "error" : "stop" },
75
+ replayState: chunk.replayState
76
+ };
77
+ default:
78
+ return chunk;
79
+ }
80
+ }
81
+
82
+ // Only direct human prompts are summarized: plugin-injected context
83
+ // (AGENTS.md, skill bodies, file-change notices) and other machine-originated
84
+ // events must not leak into the memory store. Events without a data payload
85
+ // (minimal test doubles) pass the kind check and are handled by the content
86
+ // check below.
87
+ //
88
+ // codingRetrospect: each new distill window contains the complete available
89
+ // transcript for that window — user prompts plus assistant public replies, tool
90
+ // calls + results and code dispatch output — so the summarizer can see tool
91
+ // errors and extract pitfall root causes, not just what the user typed. The same
92
+ // filtering stays: only source.kind === "user" prompts enter (plugin/machine
93
+ // content is excluded). The result is a single text transcript passed to the
94
+ // LLM as one user message (SUMMARY_PROMPT already says "根据下面的会话内容").
95
+ //
96
+ // Privacy: assistant `reasoning` (private thought) blocks are deliberately NOT
97
+ // collected — distilled memories must never sink private reasoning chains.
98
+ // Only public text blocks (type "text") reach the summarizer.
99
+ function eventSeq(event) {
100
+ return Number.isSafeInteger(event?.seq) ? event.seq : undefined;
101
+ }
102
+
103
+ const SUBAGENT_MESSAGE_KINDS = new Set(["agent-message", "subagent-settled"]);
104
+
105
+ // DSH 0.1.2-rc.1 起 Session 改用 snapshotEvents(),兼容旧版 .events。
106
+ // snapshotEvents 的范围参数按 seq 传入;再做一次事件级过滤,是为了兼容
107
+ // 旧实现忽略参数、或把边界解释为闭区间的情况,同时避免用数组下标充当游标。
108
+ function snapshotSessionEvents(session, afterSeq, throughSeq) {
109
+ let events;
110
+ if (typeof session?.snapshotEvents === "function") {
111
+ events = afterSeq === undefined && throughSeq === undefined
112
+ ? session.snapshotEvents()
113
+ : session.snapshotEvents(afterSeq, throughSeq);
114
+ } else {
115
+ events = session?.events ?? [];
116
+ }
117
+ if (!Array.isArray(events)) events = session?.events ?? [];
118
+ if (!Array.isArray(events)) return [];
119
+ return events.filter((event) => {
120
+ const seq = eventSeq(event);
121
+ // 一旦已有 seq 游标,没有 seq 的事件就无法证明是新增事件,不能重新打开
122
+ // 已经成功消费过的窗口。
123
+ if (afterSeq !== undefined && (seq === undefined || seq <= afterSeq)) return false;
124
+ if (throughSeq !== undefined && seq !== undefined && seq > throughSeq) return false;
125
+ return true;
126
+ });
127
+ }
128
+
129
+ function collectMessages(session, maxChars = 8000, language = "zh", afterSeq, throughSeq) {
130
+ const events = snapshotSessionEvents(session, afterSeq, throughSeq);
131
+ const lines = [];
132
+ const seenSubagentMessages = new Set();
133
+ // 兼容严格形状 [{type:"text",text}] 与宽松形状 ["字符串", ...](lib-smoke 用例
134
+ // 直接传字符串数组)。只取公开文本块;reasoning 私有推理块不进蒸馏上下文。
135
+ const textOf = (content) => {
136
+ if (typeof content === "string") return content;
137
+ if (!Array.isArray(content)) return "";
138
+ return content
139
+ .map((block) => (typeof block === "string" ? block : (block && block.type === "text" && typeof block.text === "string" ? block.text : "")))
140
+ .filter((s) => s)
141
+ .join("\n");
142
+ };
143
+ const trim = (s, n) => (typeof s === "string" && s.length > n ? `${s.slice(0, n)}…` : s);
144
+ const trimTranscript = (s, n) => {
145
+ if (typeof s !== "string" || s.length <= n) return s;
146
+ if (n <= 1) return "…".slice(0, n);
147
+ return `…${s.slice(-(n - 1))}`;
148
+ };
149
+ const appendSubagentMessage = (message) => {
150
+ const kind = message?.source?.kind;
151
+ if (!SUBAGENT_MESSAGE_KINDS.has(kind)) return;
152
+ const text = textOf(message?.content);
153
+ if (!text.trim()) return;
154
+ const key = typeof message?.id === "string" && message.id
155
+ ? `id:${message.id}`
156
+ : `content:${kind}:${text}`;
157
+ if (seenSubagentMessages.has(key)) return;
158
+ seenSubagentMessages.add(key);
159
+ lines.push(STR.transcriptAgent[language](text));
160
+ };
161
+ for (const event of events) {
162
+ const data = event?.data ?? {};
163
+ const kind = data?.source?.kind;
164
+ switch (event.type) {
165
+ case "user/message": {
166
+ if (kind !== undefined && kind !== "user" && !SUBAGENT_MESSAGE_KINDS.has(kind)) break;
167
+ const text = textOf(data?.content);
168
+ if (!text.trim()) break;
169
+ if (SUBAGENT_MESSAGE_KINDS.has(kind)) appendSubagentMessage(data);
170
+ else lines.push(STR.transcriptUser[language](text));
171
+ break;
172
+ }
173
+ case "agent/inbox/spliced": {
174
+ const inserted = Array.isArray(data?.inserted) ? data.inserted : [];
175
+ for (const message of inserted) appendSubagentMessage(message);
176
+ break;
177
+ }
178
+ case "assistant/message": {
179
+ const msg = data?.message;
180
+ const blocks = Array.isArray(msg?.content) ? msg.content : [];
181
+ const text = textOf(blocks);
182
+ if (text.trim()) lines.push(STR.transcriptAssistant[language](text));
183
+ // 私有推理块(reasoning)刻意不采集:蒸馏记忆不得沉淀模型私有思考链。
184
+ break;
185
+ }
186
+ case "tool/call": {
187
+ const args = typeof data.arguments === "string"
188
+ ? data.arguments
189
+ : data.arguments ? JSON.stringify(data.arguments) : "";
190
+ lines.push(STR.transcriptToolCall[language](data.name ?? "?", trim(args, 300)));
191
+ break;
192
+ }
193
+ case "tool/result": {
194
+ const result = Array.isArray(data?.message?.content)
195
+ ? data.message.content.find((block) => block?.type === "tool-result")
196
+ : undefined;
197
+ const out = textOf(result?.content);
198
+ const status = result?.isError === true ? STR.statusFail[language] : STR.statusOk[language];
199
+ lines.push(STR.transcriptToolResult[language](status, trim(out, 500)));
200
+ break;
201
+ }
202
+ case "tool/code-dispatch": {
203
+ const out = textOf(data.content);
204
+ const status = data.isError === true ? STR.statusFail[language] : STR.statusOk[language];
205
+ lines.push(STR.transcriptCode[language](status, trim(out, 500)));
206
+ break;
207
+ }
208
+ default:
209
+ break;
210
+ }
211
+ }
212
+ const lastSeq = events.reduce((max, event) => {
213
+ const seq = eventSeq(event);
214
+ if (seq === undefined) return max;
215
+ return max === undefined ? seq : Math.max(max, seq);
216
+ }, undefined);
217
+ return {
218
+ messages: lines.length
219
+ ? [createUserMessage({ content: [{ type: "text", text: trimTranscript(lines.join("\n"), maxChars) }], source: { kind: "plugin", plugin: "dsh-mneme" } })]
220
+ : [],
221
+ hasEvents: events.length > 0,
222
+ lastSeq
223
+ };
224
+ }
225
+
226
+ // ── 智能调速器(429 保护)─────────────────────────────────────────────
227
+ // 对话一多时 turn/end 会批量触发蒸馏,多个 LLM 请求"一拥而上"正是 429 的
228
+ // 来源。这里借鉴机场安检的思路:所有蒸馏调用进同一个全局串行队列,按间隔
229
+ // distillRateLimitIntervalMs 分批放行;命中 429 时按 distillRateLimitBaseDelayMs
230
+ // 指数退避(1s→2s→4s…)自动重试 distillRateLimitRetries 次,全程对用户透明,
231
+ // 不把 429 错误码直接抛出去。
232
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
233
+
234
+ function isRateLimited(error) {
235
+ const status = error?.status ?? error?.statusCode ?? error?.response?.status;
236
+ if (status === 429) return true;
237
+ const msg = String(error?.message ?? error ?? "");
238
+ return /429|rate.?limit|too many requests|请求过于频繁/i.test(msg);
239
+ }
240
+
241
+ // 全局串行链:每个蒸馏任务在前一个结束后才开始,单次失败不阻塞后续。
242
+ // 相邻请求间隔按"距上一个结束不足 intervalMs 就补齐等待"实现——单次蒸馏
243
+ // 零延迟(上次结束距今已超过间隔,直接放行),只有连续批量蒸馏才触发限速。
244
+ let distillQueue = Promise.resolve();
245
+ let lastDistillEnd = 0;
246
+ function enqueueDistill(task, intervalMs = 0) {
247
+ const run = distillQueue.then(async () => {
248
+ if (intervalMs > 0) {
249
+ const wait = Math.max(0, intervalMs - (Date.now() - lastDistillEnd));
250
+ if (wait > 0) await sleep(wait);
251
+ }
252
+ lastDistillEnd = Date.now();
253
+ return task();
254
+ });
255
+ distillQueue = run.catch(() => {});
256
+ return run;
257
+ }
258
+
259
+ export function createSummarizer(ctx, service, config) {
260
+ if (!config.autoSummarize) return { dispose: () => {} };
261
+
262
+ const inFlight = new Map();
263
+ // Issue #127:per-session 上一次实际开跑时刻(最小间隔闸门用)。与 inFlight
264
+ // 同生命周期,dispose 时一并清空。
265
+ const lastRunAt = new Map();
266
+ // Issue #210:只在一次蒸馏完整成功后记录已消费的最后事件 seq。
267
+ const lastDistilledSeq = new Map();
268
+ let disposed = false;
269
+
270
+ /** 写一条 autoSummarize 的 LLM 审计行(审计失败绝不阻塞蒸馏本身)。 */
271
+ function writeAudit(entry) {
272
+ if (config?.llmAudit?.enabled === false || typeof service.saveLlmAudit !== "function") return;
273
+ try {
274
+ service.saveLlmAudit({
275
+ trigger_source: "autoSummarize",
276
+ operation_type: "summarize_compress",
277
+ related_memory_ids: [],
278
+ ...entry
279
+ });
280
+ } catch (auditError) {
281
+ ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
282
+ }
283
+ }
284
+
285
+ async function summarize(session, triggerEvent) {
286
+ if (disposed || inFlight.has(session.id)) return;
287
+
288
+ const header = session.requestHeader?.()?.config;
289
+ // Config override takes priority, then session header, then nothing.
290
+ const route = (config.summarizeProvider && config.summarizeModel)
291
+ ? { provider: config.summarizeProvider, model: config.summarizeModel }
292
+ : (header?.provider && header?.model)
293
+ ? { provider: header.provider, model: header.model }
294
+ : undefined;
295
+
296
+ // Issue #127 间隔门:同一会话两次蒸馏之间的最小间隔(0 = 现状,不限)。进入
297
+ // 即打点——失败/degraded 的 run 也占用间隔,与 dreamMinIntervalMinutes 语义
298
+ // 一致,防止失败连发。被挡下的 turn/end 静默跳过,但留一行 status='skipped'
299
+ // 审计,否则「这一轮为什么没蒸馏」对用户完全不可观测。
300
+ const gapMs = (config.summarizeMinIntervalMinutes ?? 0) * 60000;
301
+ const prevClaim = lastRunAt.get(session.id);
302
+ if (gapMs > 0 && Date.now() - (prevClaim ?? 0) < gapMs) {
303
+ writeAudit({
304
+ timestamp: new Date().toISOString(),
305
+ model_id: route ? `${route.provider}:${route.model}` : "unknown",
306
+ status: "skipped",
307
+ error_message: "min-interval"
308
+ });
309
+ return;
310
+ }
311
+ const controller = new AbortController();
312
+ inFlight.set(session.id, controller);
313
+ // audit state for the compression call. null = no audit for this run
314
+ // (disabled, or no LLM call was actually made). The audit row is written in
315
+ // the finally below — once, regardless of which exit path the call took —
316
+ // so a failed/aborted stream still leaves a status='error' trail without
317
+ // ever blocking the summarization itself.
318
+ let audit = null;
319
+ let abortedRun = false;
320
+ try {
321
+ if (!route) return;
322
+ const previousSeq = lastDistilledSeq.get(session.id);
323
+ const triggerSeq = eventSeq(triggerEvent);
324
+ // 只读取上次成功游标之后、当前 turn/end 之前的事件。旧版 snapshotEvents
325
+ // 即使忽略范围参数,collectMessages 仍会按事件 seq 二次过滤。
326
+ const collected = collectMessages(
327
+ session,
328
+ config.distillMaxChars ?? 24000,
329
+ langOf(config),
330
+ previousSeq,
331
+ triggerSeq
332
+ );
333
+ const hasNewEvents = collected.hasEvents
334
+ || (triggerSeq !== undefined && (previousSeq === undefined || triggerSeq > previousSeq));
335
+ if (!hasNewEvents) return;
336
+
337
+ const nextSeq = Math.max(collected.lastSeq ?? Number.NEGATIVE_INFINITY, triggerSeq ?? Number.NEGATIVE_INFINITY);
338
+ if (!collected.messages.length) {
339
+ // 没有可蒸馏的公开文本也算成功消费当前事件窗口,避免每个 turn/end
340
+ // 都重新扫描同一批无内容事件;没有 seq 时则不提交不可验证的游标。
341
+ if (Number.isFinite(nextSeq)) lastDistilledSeq.set(session.id, nextSeq);
342
+ return;
343
+ }
344
+ const messages = collected.messages;
345
+ // 只有真正准备发起一次 LLM 蒸馏时才占用最小间隔;失败路径仍按原语义
346
+ // 占用,aborted 则由 finally 回滚。
347
+ lastRunAt.set(session.id, Date.now());
348
+
349
+ if (config?.llmAudit?.enabled !== false && typeof service.saveLlmAudit === "function") {
350
+ audit = {
351
+ route,
352
+ timestamp: new Date().toISOString(),
353
+ startedAt: Date.now(),
354
+ inputTokens: 0,
355
+ outputTokens: 0,
356
+ status: "success",
357
+ errorMessage: null
358
+ };
359
+ }
360
+
361
+ const options = {
362
+ provider: route.provider,
363
+ model: route.model,
364
+ purpose: "summarization",
365
+ messages: [
366
+ { role: "system", content: [{ type: "text", text: config.codingRetrospect ? STR.prompts.codingSummary[langOf(config)] : STR.prompts.summary[langOf(config)] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
367
+ ...messages
368
+ ],
369
+ signal: controller.signal
370
+ };
371
+ // 智能调速器:整段蒸馏 LLM 调用进全局串行队列,按间隔分批放行;429 时
372
+ // 指数退避自动重试,全程对用户透明,不把 429 错误码直接抛出去。
373
+ const intervalMs = config.distillRateLimitIntervalMs ?? 1000;
374
+ const { text, assembledText, aborted } = await enqueueDistill(async () => {
375
+ const retries = config.distillRateLimitRetries ?? 3;
376
+ const baseDelayMs = config.distillRateLimitBaseDelayMs ?? 1000;
377
+ for (let attempt = 0; ; attempt++) {
378
+ const assembler = new BlockAssembler();
379
+ let text = "";
380
+ let aborted = false;
381
+ try {
382
+ for await (const chunk of ctx.llm.stream(options)) {
383
+ if (STREAM_CHUNK_TYPES.has(chunk.type)) assembler.push(toProtocolChunk(chunk));
384
+ if (chunk.type === "text-delta") {
385
+ text += chunk.text ?? chunk.delta ?? "";
386
+ }
387
+ if (chunk.type === "usage" && audit) {
388
+ const i = chunk.input_tokens ?? chunk.inputTokens ?? chunk.prompt_tokens ?? chunk.promptTokens;
389
+ const o = chunk.output_tokens ?? chunk.outputTokens ?? chunk.completion_tokens ?? chunk.completionTokens;
390
+ if (Number.isFinite(i)) audit.inputTokens = i;
391
+ if (Number.isFinite(o)) audit.outputTokens = o;
392
+ }
393
+ if (chunk.type === "finish") {
394
+ const reasonKind = chunk.reason?.kind ?? chunk.kind;
395
+ if (reasonKind === "error" || reasonKind === "aborted") {
396
+ // 429 也可能以 finish reason error 携带 rate-limit 信息,统一
397
+ // 转抛错走指数退避重试。
398
+ if (isRateLimited(chunk.reason ?? chunk)) {
399
+ throw Object.assign(new Error("rate limited"), { status: 429 });
400
+ }
401
+ if (audit) {
402
+ audit.status = "error";
403
+ audit.errorMessage = `llm stream ${reasonKind}`;
404
+ }
405
+ aborted = true;
406
+ break;
407
+ }
408
+ }
409
+ }
410
+ // Direct delta accumulation is the primary extraction path (it works
411
+ // for real protocol chunks {index,text} and looser {delta} shapes
412
+ // alike); the assembler blocks are a fallback for streams that only
413
+ // deliver text inside block-end. This dsh-llm exposes no public
414
+ // no-arg assemble() — blocks() is the message-level API.
415
+ const blocks = assembler.blocks();
416
+ const assembledText = blocks
417
+ .filter((b) => b.type === "text")
418
+ .map((b) => b.text ?? "")
419
+ .join("");
420
+ return { text, assembledText, aborted };
421
+ } catch (error) {
422
+ if (error?.name === "AbortError" || controller.signal.aborted) throw error; // dispose 中止直接放行
423
+ if (isRateLimited(error) && attempt < retries) {
424
+ const delay = baseDelayMs * 2 ** attempt;
425
+ ctx.logger?.warn?.(
426
+ `dsh-mneme: 蒸馏请求过于频繁(429),为避免限流等待 ${delay}ms 后自动重试(第 ${attempt + 1}/${retries} 次)`
427
+ );
428
+ await sleep(delay);
429
+ continue;
430
+ }
431
+ // 429 重试耗尽或非 429 错误:记 audit 后抛出,保持原失败路径。
432
+ if (audit) {
433
+ audit.status = "error";
434
+ audit.errorMessage = String(error?.message ?? error);
435
+ }
436
+ throw error;
437
+ }
438
+ }
439
+ }, intervalMs);
440
+ if (aborted) {
441
+ abortedRun = true;
442
+ return;
443
+ }
444
+ const parsedResult = parseSummaryJsonResult(text || assembledText);
445
+ // 解析失败不推进 seq 游标,下一次 turn/end 仍会重试同一窗口。
446
+ if (!parsedResult.ok) {
447
+ if (audit) {
448
+ audit.status = "error";
449
+ audit.errorMessage = "invalid summary JSON";
450
+ }
451
+ return;
452
+ }
453
+ const parsed = parsedResult.entries;
454
+ // Issue #127 条数上限:0 = 不限(现状)。被截断的条数与去重命中数一并进
455
+ // 审计 metadata——此前产出条数完全由模型输出决定,用户无从自查。
456
+ const cap = config.summarizeMaxEntriesPerRun ?? 0;
457
+ const entries = cap > 0 ? parsed.slice(0, cap) : parsed;
458
+ const capped = parsed.length - entries.length;
459
+ const dedupeMode = config.summarizeDedupeMode ?? "off";
460
+ let deduped = 0;
461
+ let dedupeMaxSim = 0;
462
+ const writes = [];
463
+ for (const entry of entries) {
464
+ // Provenance: the summarizer runs on a real session (turn/end hook), so
465
+ // session.id is always available here — it rides the human-readable
466
+ // source label.
467
+ // 编码记忆类型(codingRetrospect)不带 tag:读取侧门控/加权靠 m.type
468
+ // (rejected_solution/pitfall/constraint)区分即可,tag 体系
469
+ // sanitizeTags 不认 `type:` 前缀反而会清空 tags 列(额外一次 UPDATE)。
470
+ const source = `session:${session.id}`;
471
+ // Issue #127 落库前去重(opt-in,默认 off = 现状):命中则并入既有条目
472
+ // (saveWithDedupe 的 _mergeInto 分支),不再新造一行。作用域 = 同会话 +
473
+ // 时间窗;跨会话的同类事实仍交给 autoDream / sleep 的批量裁决。
474
+ const dup = (dedupeMode !== "off" && typeof service.findSessionDuplicate === "function")
475
+ ? await service.findSessionDuplicate(
476
+ { ...entry, source },
477
+ {
478
+ mode: dedupeMode,
479
+ minSim: config.summarizeDedupeMinSim ?? 0.92,
480
+ windowHours: config.summarizeDedupeWindowHours ?? 24,
481
+ source
482
+ }
483
+ )
484
+ : undefined;
485
+ if (dup) {
486
+ deduped++;
487
+ dedupeMaxSim = Math.max(dedupeMaxSim, dup.sim ?? 0);
488
+ }
489
+ writes.push({ entry, source, dup });
490
+ }
491
+ if (writes.length) {
492
+ // 查询去重目标可以异步执行,但记忆写入必须在同一个事务中完成。
493
+ // 否则第 N 条写入失败时,前 N-1 条会残留,而重试同一 seq 窗口会重复
494
+ // 追加它们。createService 始终提供 transaction;缺失时直接失败并保留
495
+ // 游标窗口,避免退回到不具备原子性的部分写入。
496
+ if (typeof service.transaction !== "function") {
497
+ throw new Error("dsh-mneme: atomic summarization writes require service.transaction");
498
+ }
499
+ service.transaction(() => {
500
+ for (const { entry, source, dup } of writes) {
501
+ service.saveWithDedupe({
502
+ ...entry,
503
+ source,
504
+ ...(dup ? { _mergeInto: dup.memory.id } : {})
505
+ });
506
+ }
507
+ });
508
+ }
509
+ if (audit && (capped > 0 || deduped > 0)) {
510
+ audit.metadata = {
511
+ parsed: parsed.length,
512
+ ...(capped > 0 ? { capped } : {}),
513
+ ...(deduped > 0 ? { deduped, mode: dedupeMode, maxSim: Number(dedupeMaxSim.toFixed(4)) } : {})
514
+ };
515
+ }
516
+ // 记忆写入和解析都成功后才提交窗口;流失败、中止、解析失败或写入异常
517
+ // 都会在此之前退出,从而保留窗口供下一次重试。
518
+ if (Number.isFinite(nextSeq)) lastDistilledSeq.set(session.id, nextSeq);
519
+ } finally {
520
+ // Issue #127:aborted(会话关闭 / 插件 dispose)不占间隔,避免误伤该会话的
521
+ // 下一次蒸馏;其余情况(含失败)的打点保留,与 dreamMinIntervalMinutes 一致。
522
+ if (abortedRun || controller.signal.aborted) {
523
+ if (prevClaim === undefined) lastRunAt.delete(session.id);
524
+ else lastRunAt.set(session.id, prevClaim);
525
+ }
526
+ if (audit) {
527
+ try {
528
+ service.saveLlmAudit({
529
+ timestamp: audit.timestamp,
530
+ trigger_source: "autoSummarize",
531
+ operation_type: "summarize_compress",
532
+ model_id: `${audit.route.provider}:${audit.route.model}`,
533
+ input_tokens: audit.inputTokens,
534
+ output_tokens: audit.outputTokens,
535
+ total_tokens: audit.inputTokens + audit.outputTokens,
536
+ cost_usd: 0,
537
+ duration_ms: Date.now() - audit.startedAt,
538
+ status: audit.status,
539
+ error_message: audit.errorMessage,
540
+ related_memory_ids: [],
541
+ // Issue #127:parsed / capped / deduped 的观测数据。零 schema 变更
542
+ // (llm_audit_logs.metadata 已存在),一条 SQL 即可验证节流是否生效。
543
+ metadata: audit.metadata
544
+ });
545
+ } catch (auditError) {
546
+ ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
547
+ }
548
+ }
549
+ inFlight.delete(session.id);
550
+ }
551
+ }
552
+
553
+ const unsubscribe = ctx.on("session/event", (session, event) => {
554
+ if (disposed || event.type !== "turn/end") return;
555
+ // Return the summarization promise so awaiters observe the writes; the
556
+ // catch keeps listener dispatch from rejecting. Dispose-initiated aborts
557
+ // and external AbortErrors are silent.
558
+ return summarize(session, event).catch((error) => {
559
+ if (disposed || error?.name === "AbortError") return;
560
+ ctx.logger?.warn?.(`dsh-mneme: summarization failed: ${String(error)}`);
561
+ });
562
+ });
563
+
564
+ return {
565
+ dispose() {
566
+ if (disposed) return;
567
+ disposed = true;
568
+ unsubscribe?.();
569
+ for (const controller of inFlight.values()) controller.abort();
570
+ inFlight.clear();
571
+ lastRunAt.clear();
572
+ lastDistilledSeq.clear();
573
+ }
574
+ };
575
+ }