@modusensus/dsh-mneme 0.7.31 → 0.7.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/lib/api.js +79 -6
- package/lib/client.js +217 -30
- package/lib/commands.js +6 -2
- package/lib/config.js +80 -14
- package/lib/dream/decisions.js +147 -14
- package/lib/dream/sleep.js +84 -34
- package/lib/dream.js +250 -60
- package/lib/embedding.js +22 -2
- package/lib/index.js +68 -37
- package/lib/inject.js +16 -9
- package/lib/lang.js +156 -0
- package/lib/local-embedder.js +23 -9
- package/lib/mirror.js +17 -14
- package/lib/reranker.js +23 -8
- package/lib/runtime/adopt.js +176 -0
- package/lib/runtime/closure.js +172 -0
- package/lib/runtime/download.js +247 -0
- package/lib/runtime/layout.js +235 -0
- package/lib/runtime/loader.js +249 -0
- package/lib/runtime/provision.js +237 -0
- package/lib/runtime/tarball.js +130 -0
- package/lib/runtime/verify.js +191 -0
- package/lib/service.js +102 -8
- package/lib/settings.js +36 -2
- package/lib/store.js +26 -8
- package/lib/summarize.js +108 -35
- package/lib/tools.js +119 -0
- package/package.json +7 -2
- package/runtime-manifest.json +596 -0
- package/scripts/build-runtime-manifest.mjs +217 -0
- package/scripts/mneme-runtime.mjs +277 -0
- package/src/api.js +79 -6
- package/src/commands.js +6 -2
- package/src/config.js +80 -14
- package/src/dream/decisions.js +147 -14
- package/src/dream/sleep.js +84 -34
- package/src/dream.js +250 -60
- package/src/embedding.js +22 -2
- package/src/index.js +68 -37
- package/src/inject.js +16 -9
- package/src/lang.js +156 -0
- package/src/local-embedder.js +23 -9
- package/src/mirror.js +17 -14
- package/src/reranker.js +23 -8
- package/src/runtime/adopt.js +176 -0
- package/src/runtime/closure.js +172 -0
- package/src/runtime/download.js +247 -0
- package/src/runtime/layout.js +235 -0
- package/src/runtime/loader.js +249 -0
- package/src/runtime/provision.js +237 -0
- package/src/runtime/tarball.js +130 -0
- package/src/runtime/verify.js +191 -0
- package/src/service.js +102 -8
- package/src/settings.js +36 -2
- package/src/store.js +26 -8
- package/src/summarize.js +108 -35
- package/src/tools.js +119 -0
- package/test/api.test.js +89 -9
- package/test/client.test.js +94 -10
- package/test/config.test.js +17 -1
- package/test/dream-candidate-hybrid.test.js +281 -0
- package/test/dream.test.js +172 -0
- package/test/embedding-ready.test.js +196 -0
- package/test/helpers/runtime-source.js +60 -0
- package/test/helpers/tar-builder.js +80 -0
- package/test/inject.test.js +37 -1
- package/test/lang.test.js +87 -0
- package/test/quality-filter.test.js +68 -0
- package/test/reasoning-effort.test.js +106 -15
- package/test/reindex-backfill.test.js +62 -0
- package/test/runtime-adopt.test.js +206 -0
- package/test/runtime-cli.test.js +89 -0
- package/test/runtime-closure.test.js +193 -0
- package/test/runtime-download.test.js +302 -0
- package/test/runtime-layout.test.js +163 -0
- package/test/runtime-loader.test.js +237 -0
- package/test/runtime-manifest.test.js +158 -0
- package/test/runtime-provision.test.js +101 -0
- package/test/runtime-tarball.test.js +82 -0
- package/test/runtime-verify.test.js +214 -0
- package/test/sleep-action-set.test.js +422 -0
- package/test/store.test.js +14 -0
- package/test/summarize.test.js +224 -0
- package/test/tools.test.js +30 -5
- package/test/webServer-optional.test.js +34 -1
package/src/service.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { TYPE_FILE } from "./mirror.js";
|
|
3
|
+
import { STR, langOf } from "./lang.js";
|
|
3
4
|
import { computeHeat } from "./heat.js";
|
|
4
5
|
import { evaluateMemoryQuality } from "./quality-filter.js";
|
|
5
6
|
import { createBM25Index } from "./search/bm25.js";
|
|
@@ -32,6 +33,11 @@ const EPISTEMIC_WEIGHTS = { observation: 1.0, inferred: 0.85, subjective: 0.7 };
|
|
|
32
33
|
// how the version was superseded (auto_merge | human_override | overwrite).
|
|
33
34
|
const CONTENT_HISTORY_MAX = 20;
|
|
34
35
|
|
|
36
|
+
// Issue #135 附属发现 2:质量过滤器写下的系统信号标签(「为什么被降权/归档」的
|
|
37
|
+
// 唯一审计线索)。更新路径整组替换 tags 会把它们抹掉——更新时按此清单并集保留。
|
|
38
|
+
// 清单必须与 quality-filter.js 的写入端对齐(6 个,含 type 自指的 self_referential)。
|
|
39
|
+
const SIGNAL_TAGS = ["low_quality", "duplicate", "meta", "repetitive", "short_content", "self_referential"];
|
|
40
|
+
|
|
35
41
|
/** Prepend the previous content to a memory's content_history (FIFO capped). */
|
|
36
42
|
function pushContentHistory(existing, source) {
|
|
37
43
|
const history = Array.isArray(existing?.content_history) ? existing.content_history : [];
|
|
@@ -78,6 +84,7 @@ export function computeRetrievalMetrics(actualIds, expectedIds) {
|
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
export function createService({ store, mirror, config, onWrite, logger }) {
|
|
87
|
+
const language = langOf(config);
|
|
81
88
|
// Optional dream scheduler hook, installed via setDreamHook after creation
|
|
82
89
|
// (the scheduler holds a reference back to the service, so it cannot be
|
|
83
90
|
// passed in the constructor). Fired on the same write events as onWrite.
|
|
@@ -865,6 +872,60 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
865
872
|
}
|
|
866
873
|
}
|
|
867
874
|
|
|
875
|
+
/**
|
|
876
|
+
* 写入端同会话语义去重(Issue #127,opt-in,默认 off):落库前把新条目与
|
|
877
|
+
* 「同会话 + 时间窗」内已落库的记忆比对,命中即并入既有条目,不再新造一行。
|
|
878
|
+
*
|
|
879
|
+
* 分层:title 档走零成本的标题归一化;vector 档复用已有 embedding 列做近邻,
|
|
880
|
+
* 新条目的向量由 embedder 现算(embedQuery),全程无 LLM 调用。embedder 不可用
|
|
881
|
+
* 或向量缺失一律返回 undefined 回落正常写入——去重是增强,绝不能成为写入依赖。
|
|
882
|
+
*
|
|
883
|
+
* 作用域刻意限定「同会话」:跨会话的同类事实仍交给 autoDream / sleep 的批量裁决。
|
|
884
|
+
* 比对逻辑与并入动作分别复用 cosineVec 与 saveWithDedupe 的 _mergeInto 分支。
|
|
885
|
+
*
|
|
886
|
+
* @returns {{action: "merged", memory: object, sim: number}|undefined} 命中则返回
|
|
887
|
+
* 并入目标与相似度;未命中(含任何异常)返回 undefined。
|
|
888
|
+
*/
|
|
889
|
+
async function findSessionDuplicate(memory, { mode = "off", minSim = 0.92, windowHours = 24, source } = {}) {
|
|
890
|
+
if (mode === "off" || !source || !memory?.title) return undefined;
|
|
891
|
+
try {
|
|
892
|
+
const sinceMs = windowHours > 0 ? Date.now() - windowHours * 3600000 : 0;
|
|
893
|
+
const inWindow = (m) => {
|
|
894
|
+
if (sinceMs <= 0) return true;
|
|
895
|
+
const t = Date.parse(m.created_at ?? "");
|
|
896
|
+
return !Number.isFinite(t) || t >= sinceMs; // 时间戳损坏时不误杀
|
|
897
|
+
};
|
|
898
|
+
const candidates = store
|
|
899
|
+
.list({ type: memory.type, source, limit: 200 })
|
|
900
|
+
.filter((m) => !m.archived && inWindow(m));
|
|
901
|
+
if (!candidates.length) return undefined;
|
|
902
|
+
|
|
903
|
+
if (mode === "title") {
|
|
904
|
+
const norm = (s) => String(s ?? "").toLowerCase().replace(/[\s\p{P}]+/gu, "");
|
|
905
|
+
const key = norm(memory.title);
|
|
906
|
+
if (!key) return undefined;
|
|
907
|
+
const hit = candidates.find((m) => norm(m.title) === key);
|
|
908
|
+
return hit ? { action: "merged", memory: hit, sim: 1 } : undefined;
|
|
909
|
+
}
|
|
910
|
+
if (mode !== "vector") return undefined;
|
|
911
|
+
|
|
912
|
+
const vecs = store.getEmbeddings(candidates.map((m) => m.id));
|
|
913
|
+
if (!vecs.size) return undefined;
|
|
914
|
+
const probe = await embedQuery([memory.title, memory.content].filter(Boolean).join("\n"));
|
|
915
|
+
if (!probe) return undefined;
|
|
916
|
+
let best;
|
|
917
|
+
for (const m of candidates) {
|
|
918
|
+
const v = vecs.get(m.id);
|
|
919
|
+
if (!v) continue; // 无向量的既有行不参与比对(无信号 = 不判定)
|
|
920
|
+
const sim = cosineVec(probe, v);
|
|
921
|
+
if (sim >= minSim && (!best || sim > best.sim)) best = { action: "merged", memory: m, sim };
|
|
922
|
+
}
|
|
923
|
+
return best;
|
|
924
|
+
} catch {
|
|
925
|
+
return undefined; // 去重失败绝不阻塞写入
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
|
|
868
929
|
/**
|
|
869
930
|
* Save a memory, merging into an existing one when title matches within the same type.
|
|
870
931
|
*
|
|
@@ -900,9 +961,19 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
900
961
|
});
|
|
901
962
|
} catch { /* quality scoring is best-effort */ }
|
|
902
963
|
}
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
964
|
+
// dream 总览按 source 身份去重,而非本地化标题:memory.language 切换后
|
|
965
|
+
// 标题变化,若仍按 (type, title) 匹配会出现两个活跃的 summary 同时注入。
|
|
966
|
+
// 其余类型维持 (type, title) 匹配不变。
|
|
967
|
+
const candidates = store.list({ type: memory.type, limit: 100 });
|
|
968
|
+
// Issue #127:写入端语义去重命中时,调用方用 _mergeInto 指定并入目标——复用
|
|
969
|
+
// 下面这段并入逻辑(appendContent + content_history + 质量处置),不另写第二份
|
|
970
|
+
// 实现。目标已被并发删除时回落到常规匹配。
|
|
971
|
+
const existing = memory._mergeInto
|
|
972
|
+
? store.getById(memory._mergeInto)
|
|
973
|
+
: (memory.type === "summary" && memory.source === "dream")
|
|
974
|
+
? (candidates.find((m) => m.source === "dream")
|
|
975
|
+
?? candidates.find((m) => m.title.trim() === String(memory.title).trim()))
|
|
976
|
+
: candidates.find((m) => m.title.trim() === String(memory.title).trim());
|
|
906
977
|
if (existing) {
|
|
907
978
|
const newContent = String(memory.content ?? "");
|
|
908
979
|
if (!newContent.trim()) {
|
|
@@ -956,6 +1027,12 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
956
1027
|
* (importance × score/100). Best-effort: a disposition write failure must
|
|
957
1028
|
* never fail the save. Returns the (possibly refreshed) memory row so callers
|
|
958
1029
|
* see the archived/tagged state, not the pre-disposition snapshot.
|
|
1030
|
+
*
|
|
1031
|
+
* Issue #135 附属发现 1:importance ≥ memoryQualityFilter.exemptImportance
|
|
1032
|
+
* (默认 4)的记忆不参与静默自动归档——评分与信号标签照常写入(可观测),
|
|
1033
|
+
* 注入排序照常按 importance × score/100 降权,但 setArchived 跳过。报告实测
|
|
1034
|
+
* 150 条低分归档里 128 条 importance ≥ 4(51 条 = 5):一条被显式标为重要的
|
|
1035
|
+
* 记忆不该被规则分静默归档、无感知无豁免。
|
|
959
1036
|
*/
|
|
960
1037
|
function applyQualityDisposition(memory, quality, qf) {
|
|
961
1038
|
if (!quality || qf?.enabled !== true) return memory;
|
|
@@ -963,14 +1040,21 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
963
1040
|
// Signal tags (meta / repetitive / duplicate / short_content / low_quality)
|
|
964
1041
|
// are merged onto the stored row in every assessed band so the verdict is
|
|
965
1042
|
// observable, not just the numeric score. Below the archive threshold the
|
|
966
|
-
// memory is additionally archived (still explicitly searchable)
|
|
1043
|
+
// memory is additionally archived (still explicitly searchable) — unless
|
|
1044
|
+
// its importance reaches the exemption floor (kept active, demoted only).
|
|
967
1045
|
const tags = [...new Set([...(memory.tags ?? []), ...(quality.tags ?? [])])];
|
|
968
|
-
|
|
1046
|
+
const exemptImportance = qf.exemptImportance ?? 4;
|
|
1047
|
+
const importanceExempt = Number.isInteger(memory.importance) && memory.importance >= exemptImportance;
|
|
1048
|
+
if (tags.length === (memory.tags?.length ?? 0) && (quality.score >= archiveThreshold || importanceExempt)) {
|
|
969
1049
|
return memory; // no tag drift and not archived → nothing extra to write
|
|
970
1050
|
}
|
|
971
1051
|
try {
|
|
972
1052
|
store.update(memory.id, { tags, quality_score: quality.score });
|
|
973
|
-
if (quality.score < archiveThreshold
|
|
1053
|
+
if (quality.score < archiveThreshold && !importanceExempt) {
|
|
1054
|
+
store.setArchived(memory.id, true);
|
|
1055
|
+
} else if (quality.score < archiveThreshold) {
|
|
1056
|
+
logger?.info?.(`[dsh-mneme] quality filter: memory ${memory.id} scored ${quality.score} < ${archiveThreshold} but importance ${memory.importance} >= exemptImportance ${exemptImportance}; demoted, not archived`);
|
|
1057
|
+
}
|
|
974
1058
|
return store.getById(memory.id);
|
|
975
1059
|
} catch {
|
|
976
1060
|
return memory;
|
|
@@ -1195,7 +1279,8 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1195
1279
|
// (保留现有 storeChanged 逻辑)
|
|
1196
1280
|
const storeChanged = edit.updated_at !== undefined && m.updated_at !== edit.updated_at;
|
|
1197
1281
|
if (storeChanged) {
|
|
1198
|
-
|
|
1282
|
+
// 三方合并保留双方时的冲突批注:随实例语言,写进记忆正文。
|
|
1283
|
+
const marker = STR.serviceConflictMarker[language](m.updated_at, m.content);
|
|
1199
1284
|
store.update(m.id, { title: edit.title, content: `${edit.content}${marker}` });
|
|
1200
1285
|
} else {
|
|
1201
1286
|
store.update(m.id, { title: edit.title, content: edit.content });
|
|
@@ -1467,6 +1552,7 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1467
1552
|
setRecallRecorder(fn) { recallRecorder = fn; },
|
|
1468
1553
|
searchMemories,
|
|
1469
1554
|
embedQuery,
|
|
1555
|
+
findSessionDuplicate,
|
|
1470
1556
|
evaluateRetrieval,
|
|
1471
1557
|
computeRetrievalMetrics,
|
|
1472
1558
|
// passthroughs used by tools and api layers; mutations keep the mirror in sync
|
|
@@ -1484,7 +1570,15 @@ export function createService({ store, mirror, config, onWrite, logger }) {
|
|
|
1484
1570
|
},
|
|
1485
1571
|
update: (id, p, ctx = {}) => {
|
|
1486
1572
|
const old = store.getById(id);
|
|
1487
|
-
|
|
1573
|
+
// Issue #135 附属发现 2:质量过滤器把「为什么被降权/归档」写在系统信号标签
|
|
1574
|
+
// 上(SIGNAL_TAGS,applyQualityDisposition 以并集写入),而这里整组替换
|
|
1575
|
+
// tags——任何带 tags 的更新都会抹掉这条唯一的审计线索(报告实测 150 条低分
|
|
1576
|
+
// 记忆里恰好缺的 2 条就是被 update 过的,并因此误判过归档来源)。把 existing
|
|
1577
|
+
// 的系统信号标签并集保留;用户自传的标签照常生效。
|
|
1578
|
+
const patch = (p && Array.isArray(p.tags) && old && Array.isArray(old.tags))
|
|
1579
|
+
? { ...p, tags: [...new Set([...p.tags, ...old.tags.filter((t) => SIGNAL_TAGS.includes(t))])] }
|
|
1580
|
+
: p;
|
|
1581
|
+
const updated = store.update(id, patch);
|
|
1488
1582
|
// Record a user correction when any meaningful field changed and the
|
|
1489
1583
|
// reflection failure tracker is enabled. expected = what it became,
|
|
1490
1584
|
// actual = what it was before; query (when provided) captures the
|
package/src/settings.js
CHANGED
|
@@ -78,7 +78,21 @@ const FEATURE_FLAG_INT_RANGES = {
|
|
|
78
78
|
distillMaxChars: [1000, 200000],
|
|
79
79
|
codingBoostFactor: [1, 5],
|
|
80
80
|
dreamMinIntervalMinutes: [0, 10080],
|
|
81
|
-
dreamMaxTokens: [256, 131072]
|
|
81
|
+
dreamMaxTokens: [256, 131072],
|
|
82
|
+
// Issue #127:autoSummarize 节流三键(0 = 零行为变化,等同现状)。
|
|
83
|
+
summarizeMinIntervalMinutes: [0, 10080],
|
|
84
|
+
summarizeMaxEntriesPerRun: [0, 50],
|
|
85
|
+
summarizeDedupeWindowHours: [0, 168],
|
|
86
|
+
// Issue #125:hybrid 候选量上限(0 = 复用 dreamMaxSnapshotSize)。
|
|
87
|
+
dreamCandidateMax: [0, 5000]
|
|
88
|
+
};
|
|
89
|
+
// 浮点开关的闭区间(与 config.js 的 z.number().min().max() 对齐)。与整数开关
|
|
90
|
+
// 分开:面板的整数控件要求 Number.isInteger,而余弦相似度阈值必须允许小数。
|
|
91
|
+
const FEATURE_FLAG_NUMBER_RANGES = {
|
|
92
|
+
// Issue #127:vector 去重档的并入阈值。
|
|
93
|
+
summarizeDedupeMinSim: [0.5, 0.99],
|
|
94
|
+
// Issue #125:hybrid 判"高相似"的阈值。
|
|
95
|
+
dreamCandidateMinSim: [0.5, 0.99]
|
|
82
96
|
};
|
|
83
97
|
// 自由字符串开关(与 config.js 的 z.string() 同名同型):trim 后 ≤200 字符,
|
|
84
98
|
// 空串合法(= 跟随主对话模型/默认路径,面板显示 placeholder)。
|
|
@@ -106,7 +120,13 @@ const FEATURE_FLAG_ENUMS = {
|
|
|
106
120
|
// rank/scale-aware alternatives selected by the panel.
|
|
107
121
|
recallFusion: ["blend", "rrf", "minmax"],
|
|
108
122
|
// 实体抽取思考强度(issue #109):与 dreamReasoningEffort 枚举对齐。
|
|
109
|
-
entityExtractionReasoning: ["low", "medium", "high", "none"]
|
|
123
|
+
entityExtractionReasoning: ["low", "medium", "high", "none"],
|
|
124
|
+
// Issue #127:落库前去重档位(off 默认,等同现状)。
|
|
125
|
+
summarizeDedupeMode: ["off", "title", "vector"],
|
|
126
|
+
// Issue #126:sleep 冲突阶段的动作集(conflict 默认 = 现状;full = 六分支)。
|
|
127
|
+
sleepActionSet: ["conflict", "full"],
|
|
128
|
+
// Issue #125:dream 候选集构造方式(window 默认 = 现状;hybrid 并入向量组)。
|
|
129
|
+
dreamCandidateMode: ["window", "hybrid"]
|
|
110
130
|
};
|
|
111
131
|
const FEATURE_FLAG_STRING_MAX = 200;
|
|
112
132
|
|
|
@@ -114,6 +134,7 @@ const FEATURE_FLAG_STRING_MAX = 200;
|
|
|
114
134
|
export const FEATURE_FLAG_SPEC = {
|
|
115
135
|
booleans: FEATURE_FLAG_BOOLEANS,
|
|
116
136
|
ints: FEATURE_FLAG_INT_RANGES,
|
|
137
|
+
numbers: FEATURE_FLAG_NUMBER_RANGES,
|
|
117
138
|
strings: FEATURE_FLAG_STRINGS,
|
|
118
139
|
urls: FEATURE_FLAG_URLS,
|
|
119
140
|
enums: FEATURE_FLAG_ENUMS
|
|
@@ -145,6 +166,14 @@ function validateFlag(key, value) {
|
|
|
145
166
|
}
|
|
146
167
|
return value;
|
|
147
168
|
}
|
|
169
|
+
const numRange = FEATURE_FLAG_NUMBER_RANGES[key];
|
|
170
|
+
if (numRange) {
|
|
171
|
+
const [min, max] = numRange;
|
|
172
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) {
|
|
173
|
+
throw new TypeError(`feature flag "${key}" must be a number in [${min}, ${max}]`);
|
|
174
|
+
}
|
|
175
|
+
return value;
|
|
176
|
+
}
|
|
148
177
|
if (FEATURE_FLAG_STRINGS.includes(key)) {
|
|
149
178
|
if (typeof value !== "string") {
|
|
150
179
|
throw new TypeError(`feature flag "${key}" must be a string`);
|
|
@@ -185,6 +214,11 @@ function sanitizeFlags(raw) {
|
|
|
185
214
|
for (const [key, [min, max]] of Object.entries(FEATURE_FLAG_INT_RANGES)) {
|
|
186
215
|
if (Number.isInteger(raw[key]) && raw[key] >= min && raw[key] <= max) out[key] = raw[key];
|
|
187
216
|
}
|
|
217
|
+
for (const [key, [min, max]] of Object.entries(FEATURE_FLAG_NUMBER_RANGES)) {
|
|
218
|
+
if (typeof raw[key] === "number" && Number.isFinite(raw[key]) && raw[key] >= min && raw[key] <= max) {
|
|
219
|
+
out[key] = raw[key];
|
|
220
|
+
}
|
|
221
|
+
}
|
|
188
222
|
for (const key of FEATURE_FLAG_STRINGS) {
|
|
189
223
|
if (typeof raw[key] === "string" && raw[key].trim().length <= FEATURE_FLAG_STRING_MAX) {
|
|
190
224
|
out[key] = raw[key].trim();
|
package/src/store.js
CHANGED
|
@@ -43,7 +43,8 @@ CREATE TABLE IF NOT EXISTS dream_runs (
|
|
|
43
43
|
summary_stored INTEGER NOT NULL DEFAULT 0,
|
|
44
44
|
receipt TEXT NOT NULL,
|
|
45
45
|
policy_epoch INTEGER NOT NULL DEFAULT 0, -- 裁决规则版本:规则升级后旧裁决降级为历史证据
|
|
46
|
-
run_type TEXT NOT NULL DEFAULT 'auto'
|
|
46
|
+
run_type TEXT NOT NULL DEFAULT 'auto', -- auto | sleep:睡眠周期的审计区分
|
|
47
|
+
skipped TEXT -- JSON: degraded 轮被跳过的逐条明细(index/action/ids/error)
|
|
47
48
|
);
|
|
48
49
|
CREATE INDEX IF NOT EXISTS idx_dream_runs_created ON dream_runs(created_at);
|
|
49
50
|
|
|
@@ -371,7 +372,9 @@ function toDreamRun(row) {
|
|
|
371
372
|
summary_stored: row.summary_stored === 1,
|
|
372
373
|
receipt: row.receipt,
|
|
373
374
|
policy_epoch: row.policy_epoch ?? 0,
|
|
374
|
-
run_type: row.run_type ?? "auto"
|
|
375
|
+
run_type: row.run_type ?? "auto",
|
|
376
|
+
// Issue #104:degraded 轮的逐条跳过明细(JSON 列,NULL = 该轮无跳过项)。
|
|
377
|
+
skipped: row.skipped ? JSON.parse(row.skipped) : undefined
|
|
375
378
|
};
|
|
376
379
|
}
|
|
377
380
|
|
|
@@ -597,6 +600,9 @@ export function createStore(path) {
|
|
|
597
600
|
// Legacy dream_runs without policy_epoch → backfill with the default epoch.
|
|
598
601
|
addColumn("dream_runs", "policy_epoch", "ALTER TABLE dream_runs ADD COLUMN policy_epoch INTEGER NOT NULL DEFAULT 0");
|
|
599
602
|
addColumn("dream_runs", "run_type", "ALTER TABLE dream_runs ADD COLUMN run_type TEXT NOT NULL DEFAULT 'auto'");
|
|
603
|
+
// Issue #104:degraded(合法子集已应用)轮被跳过的决策明细。此前只进 logger.warn,
|
|
604
|
+
// 离线回放 dream_runs 无法定位 degraded 成因(跨类型 merge / update 保护期 / unknown id)。
|
|
605
|
+
addColumn("dream_runs", "skipped", "ALTER TABLE dream_runs ADD COLUMN skipped TEXT");
|
|
600
606
|
|
|
601
607
|
// Legacy mirror_state without v0.3.6 generation columns → add each missing
|
|
602
608
|
// column idempotently (old DBs open cleanly, no data loss).
|
|
@@ -1031,11 +1037,18 @@ export function createStore(path) {
|
|
|
1031
1037
|
).get().c;
|
|
1032
1038
|
}
|
|
1033
1039
|
|
|
1034
|
-
/**
|
|
1040
|
+
/**
|
|
1041
|
+
* Candidate rows still missing an embedding, for incremental re-indexing.
|
|
1042
|
+
* Issue #128: active rows only — archived/forgotten rows never participate in
|
|
1043
|
+
* recall or dream clustering, so backfill quota must not be eaten by them
|
|
1044
|
+
* (report measured 25/50 reindex slots landing on archived rows while 88
|
|
1045
|
+
* active rows stayed un-embedded). Same 口径 as embeddedCount()/count().
|
|
1046
|
+
*/
|
|
1035
1047
|
function needsEmbedding(limit = 50) {
|
|
1036
1048
|
return db.prepare(
|
|
1037
1049
|
`SELECT id, title, content FROM memories
|
|
1038
|
-
WHERE embedding IS NULL OR embedding = ''
|
|
1050
|
+
WHERE (embedding IS NULL OR embedding = '')
|
|
1051
|
+
AND forgotten = 0 AND archived = 0
|
|
1039
1052
|
ORDER BY updated_at DESC LIMIT ?`
|
|
1040
1053
|
).all(limit);
|
|
1041
1054
|
}
|
|
@@ -1119,16 +1132,20 @@ export function createStore(path) {
|
|
|
1119
1132
|
const now = nowIso();
|
|
1120
1133
|
const policyEpoch = Number.isInteger(run.policy_epoch) ? run.policy_epoch : 0;
|
|
1121
1134
|
const runType = run.run_type ?? "auto";
|
|
1135
|
+
// Issue #104:degraded 轮的跳过明细(JSON)。与 decisions/outcome 同为可选
|
|
1136
|
+
// 载荷,未提供时落 NULL(而不是 "[]"),审计行只记真实发生过的跳过。
|
|
1137
|
+
const skipped = run.skipped !== undefined ? JSON.stringify(run.skipped) : null;
|
|
1122
1138
|
db.prepare(
|
|
1123
1139
|
`INSERT INTO dream_runs (id, created_at, status, error, provider, model, snapshot_hash,
|
|
1124
|
-
input_count, input, decisions, outcome, applied, summary_stored, receipt, policy_epoch, run_type)
|
|
1125
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1140
|
+
input_count, input, decisions, outcome, applied, summary_stored, receipt, policy_epoch, run_type, skipped)
|
|
1141
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1126
1142
|
ON CONFLICT(id) DO UPDATE SET
|
|
1127
1143
|
created_at=excluded.created_at, status=excluded.status, error=excluded.error,
|
|
1128
1144
|
provider=excluded.provider, model=excluded.model, snapshot_hash=excluded.snapshot_hash,
|
|
1129
1145
|
input_count=excluded.input_count, input=excluded.input, decisions=excluded.decisions,
|
|
1130
1146
|
outcome=excluded.outcome, applied=excluded.applied, summary_stored=excluded.summary_stored,
|
|
1131
|
-
receipt=excluded.receipt, policy_epoch=excluded.policy_epoch, run_type=excluded.run_type
|
|
1147
|
+
receipt=excluded.receipt, policy_epoch=excluded.policy_epoch, run_type=excluded.run_type,
|
|
1148
|
+
skipped=excluded.skipped`
|
|
1132
1149
|
).run(
|
|
1133
1150
|
id,
|
|
1134
1151
|
run.created_at ?? now,
|
|
@@ -1145,7 +1162,8 @@ export function createStore(path) {
|
|
|
1145
1162
|
run.summary_stored ? 1 : 0,
|
|
1146
1163
|
run.receipt,
|
|
1147
1164
|
policyEpoch,
|
|
1148
|
-
runType
|
|
1165
|
+
runType,
|
|
1166
|
+
skipped
|
|
1149
1167
|
);
|
|
1150
1168
|
return getDreamRun(id);
|
|
1151
1169
|
}
|
package/src/summarize.js
CHANGED
|
@@ -1,22 +1,9 @@
|
|
|
1
1
|
import { BlockAssembler, createUserMessage } from "@deepseek-ai/dsh-llm";
|
|
2
|
-
|
|
3
|
-
const SUMMARY_PROMPT = `你是记忆库提炼助手。根据下面的会话内容,提炼值得跨会话记住的原子记忆。
|
|
4
|
-
原子记忆原则:每条记忆只装一个独立事实/偏好/决策,短小、自带完整上下文(把数字、名字、路径、结论等原始细节保留在 content 里,不要抽象概括);宁可拆成多条也绝不合并丢细节。信息量一般提 2-4 条,信息密集的对话可提 4-8 条。
|
|
5
|
-
只输出 JSON 数组,每项形如 {"type":"preference|project|decision|history","title":"简短标题","content":"保留原始细节的一句话","importance":1-5}。
|
|
6
|
-
不要输出任何其他文字。`;
|
|
2
|
+
import { STR, langOf } from "./lang.js";
|
|
7
3
|
|
|
8
4
|
// 编码记忆蒸馏 prompt(codingRetrospect 开启时启用):在通用记忆之外,额外提取
|
|
9
5
|
// 三类编码专属记忆,专治重复踩坑 / 遗忘被否决方案 / 丢失工程约束。字段仍沿用
|
|
10
6
|
// title/content 单列结构(store 无结构化字段),信息浓缩进 content。
|
|
11
|
-
const CODING_SUMMARY_PROMPT = `你是记忆库提炼助手。根据下面的会话内容(含用户输入、助手回答、工具调用与结果),提炼值得跨会话记住的原子记忆。
|
|
12
|
-
原子记忆原则:每条记忆只装一个独立事实/偏好/决策,短小、自带完整上下文(把数字、报错信息、命令、路径、结论等原始细节保留在 content 里,不要抽象概括);宁可拆成多条也绝不合并丢细节。信息量一般提 2-4 条,信息密集的对话可提 4-8 条。
|
|
13
|
-
只输出 JSON 数组,每项形如 {"type":"preference|project|decision|history|rejected_solution|pitfall|constraint","title":"简短标题","content":"保留原始细节的一句话","importance":1-5}。
|
|
14
|
-
若对话涉及编码/调试,可额外提取编码类记忆:
|
|
15
|
-
- rejected_solution:被否决/废弃的实现方案(content 含方案简述 + 被否决原因 + 最终采用方案)
|
|
16
|
-
- pitfall:调试踩坑记录(content 含现象/报错 + 根因 + 解决/规避方法)
|
|
17
|
-
- constraint:项目工程约束(content 含约束描述 + 来源)
|
|
18
|
-
普通闲聊、临时无关对话一律不提取编码类记忆。不要输出任何其他文字。`;
|
|
19
|
-
|
|
20
7
|
/** Extract a JSON array from LLM output that may contain prose around it. */
|
|
21
8
|
export function parseSummaryJson(raw) {
|
|
22
9
|
const text = String(raw ?? "");
|
|
@@ -101,7 +88,7 @@ function toProtocolChunk(chunk) {
|
|
|
101
88
|
// Privacy: assistant `reasoning` (private thought) blocks are deliberately NOT
|
|
102
89
|
// collected — distilled memories must never sink private reasoning chains.
|
|
103
90
|
// Only public text blocks (type "text") reach the summarizer.
|
|
104
|
-
function collectMessages(session, maxChars = 8000) {
|
|
91
|
+
function collectMessages(session, maxChars = 8000, language = "zh") {
|
|
105
92
|
// DSH 0.1.2-rc.1 起 Session 改用 snapshotEvents(),兼容旧版 .events
|
|
106
93
|
const events = session.snapshotEvents?.() ?? session.events ?? [];
|
|
107
94
|
const lines = [];
|
|
@@ -123,14 +110,14 @@ function collectMessages(session, maxChars = 8000) {
|
|
|
123
110
|
case "user/message": {
|
|
124
111
|
if (kind !== undefined && kind !== "user") break;
|
|
125
112
|
const text = textOf(data?.content);
|
|
126
|
-
if (text.trim()) lines.push(
|
|
113
|
+
if (text.trim()) lines.push(STR.transcriptUser[language](text));
|
|
127
114
|
break;
|
|
128
115
|
}
|
|
129
116
|
case "assistant/message": {
|
|
130
117
|
const msg = data?.message;
|
|
131
118
|
const blocks = Array.isArray(msg?.content) ? msg.content : [];
|
|
132
119
|
const text = textOf(blocks);
|
|
133
|
-
if (text.trim()) lines.push(
|
|
120
|
+
if (text.trim()) lines.push(STR.transcriptAssistant[language](text));
|
|
134
121
|
// 私有推理块(reasoning)刻意不采集:蒸馏记忆不得沉淀模型私有思考链。
|
|
135
122
|
break;
|
|
136
123
|
}
|
|
@@ -138,19 +125,19 @@ function collectMessages(session, maxChars = 8000) {
|
|
|
138
125
|
const args = typeof data.arguments === "string"
|
|
139
126
|
? data.arguments
|
|
140
127
|
: data.arguments ? JSON.stringify(data.arguments) : "";
|
|
141
|
-
lines.push(
|
|
128
|
+
lines.push(STR.transcriptToolCall[language](data.name ?? "?", trim(args, 300)));
|
|
142
129
|
break;
|
|
143
130
|
}
|
|
144
131
|
case "tool/result": {
|
|
145
132
|
const out = typeof data.output === "string" ? data.output : data.output ? JSON.stringify(data.output) : "";
|
|
146
|
-
const status = data.ok === false ?
|
|
147
|
-
lines.push(
|
|
133
|
+
const status = data.ok === false ? STR.statusFail[language] : STR.statusOk[language];
|
|
134
|
+
lines.push(STR.transcriptToolResult[language](status, trim(out, 500)));
|
|
148
135
|
break;
|
|
149
136
|
}
|
|
150
137
|
case "tool/code-dispatch": {
|
|
151
138
|
const out = typeof data.output === "string" ? data.output : data.output ? JSON.stringify(data.output) : "";
|
|
152
|
-
const status = data.ok === false ?
|
|
153
|
-
lines.push(
|
|
139
|
+
const status = data.ok === false ? STR.statusFail[language] : STR.statusOk[language];
|
|
140
|
+
lines.push(STR.transcriptCode[language](status, trim(out, 500)));
|
|
154
141
|
break;
|
|
155
142
|
}
|
|
156
143
|
default:
|
|
@@ -197,10 +184,54 @@ export function createSummarizer(ctx, service, config) {
|
|
|
197
184
|
if (!config.autoSummarize) return { dispose: () => {} };
|
|
198
185
|
|
|
199
186
|
const inFlight = new Map();
|
|
187
|
+
// Issue #127:per-session 上一次实际开跑时刻(最小间隔闸门用)。与 inFlight
|
|
188
|
+
// 同生命周期,dispose 时一并清空。
|
|
189
|
+
const lastRunAt = new Map();
|
|
200
190
|
let disposed = false;
|
|
201
191
|
|
|
192
|
+
/** 写一条 autoSummarize 的 LLM 审计行(审计失败绝不阻塞蒸馏本身)。 */
|
|
193
|
+
function writeAudit(entry) {
|
|
194
|
+
if (config?.llmAudit?.enabled === false || typeof service.saveLlmAudit !== "function") return;
|
|
195
|
+
try {
|
|
196
|
+
service.saveLlmAudit({
|
|
197
|
+
trigger_source: "autoSummarize",
|
|
198
|
+
operation_type: "summarize_compress",
|
|
199
|
+
related_memory_ids: [],
|
|
200
|
+
...entry
|
|
201
|
+
});
|
|
202
|
+
} catch (auditError) {
|
|
203
|
+
ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
202
207
|
async function summarize(session) {
|
|
203
208
|
if (disposed || inFlight.has(session.id)) return;
|
|
209
|
+
|
|
210
|
+
const header = session.requestHeader?.()?.config;
|
|
211
|
+
// Config override takes priority, then session header, then nothing.
|
|
212
|
+
const route = (config.summarizeProvider && config.summarizeModel)
|
|
213
|
+
? { provider: config.summarizeProvider, model: config.summarizeModel }
|
|
214
|
+
: (header?.provider && header?.model)
|
|
215
|
+
? { provider: header.provider, model: header.model }
|
|
216
|
+
: undefined;
|
|
217
|
+
|
|
218
|
+
// Issue #127 间隔门:同一会话两次蒸馏之间的最小间隔(0 = 现状,不限)。进入
|
|
219
|
+
// 即打点——失败/degraded 的 run 也占用间隔,与 dreamMinIntervalMinutes 语义
|
|
220
|
+
// 一致,防止失败连发。被挡下的 turn/end 静默跳过,但留一行 status='skipped'
|
|
221
|
+
// 审计,否则「这一轮为什么没蒸馏」对用户完全不可观测。
|
|
222
|
+
const gapMs = (config.summarizeMinIntervalMinutes ?? 0) * 60000;
|
|
223
|
+
const prevClaim = lastRunAt.get(session.id);
|
|
224
|
+
if (gapMs > 0 && Date.now() - (prevClaim ?? 0) < gapMs) {
|
|
225
|
+
writeAudit({
|
|
226
|
+
timestamp: new Date().toISOString(),
|
|
227
|
+
model_id: route ? `${route.provider}:${route.model}` : "unknown",
|
|
228
|
+
status: "skipped",
|
|
229
|
+
error_message: "min-interval"
|
|
230
|
+
});
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
lastRunAt.set(session.id, Date.now());
|
|
234
|
+
|
|
204
235
|
const controller = new AbortController();
|
|
205
236
|
inFlight.set(session.id, controller);
|
|
206
237
|
// audit state for the compression call. null = no audit for this run
|
|
@@ -209,18 +240,12 @@ export function createSummarizer(ctx, service, config) {
|
|
|
209
240
|
// so a failed/aborted stream still leaves a status='error' trail without
|
|
210
241
|
// ever blocking the summarization itself.
|
|
211
242
|
let audit = null;
|
|
243
|
+
let abortedRun = false;
|
|
212
244
|
try {
|
|
213
|
-
const header = session.requestHeader?.()?.config;
|
|
214
|
-
// Config override takes priority, then session header, then nothing.
|
|
215
|
-
const route = (config.summarizeProvider && config.summarizeModel)
|
|
216
|
-
? { provider: config.summarizeProvider, model: config.summarizeModel }
|
|
217
|
-
: (header?.provider && header?.model)
|
|
218
|
-
? { provider: header.provider, model: header.model }
|
|
219
|
-
: undefined;
|
|
220
245
|
if (!route) return;
|
|
221
246
|
// 完整转录(Codex 式):蒸馏把整轮对话交给 LLM 提炼原子记忆,不再硬裁
|
|
222
247
|
// 8000 字截断语义;上限由 distillMaxChars 控制(默认 24000,可调大)。
|
|
223
|
-
const messages = collectMessages(session, config.distillMaxChars ?? 24000);
|
|
248
|
+
const messages = collectMessages(session, config.distillMaxChars ?? 24000, langOf(config));
|
|
224
249
|
if (!messages.length) return;
|
|
225
250
|
|
|
226
251
|
if (config?.llmAudit?.enabled !== false && typeof service.saveLlmAudit === "function") {
|
|
@@ -240,7 +265,7 @@ export function createSummarizer(ctx, service, config) {
|
|
|
240
265
|
model: route.model,
|
|
241
266
|
purpose: "summarization",
|
|
242
267
|
messages: [
|
|
243
|
-
{ role: "system", content: [{ type: "text", text: config.codingRetrospect ?
|
|
268
|
+
{ role: "system", content: [{ type: "text", text: config.codingRetrospect ? STR.prompts.codingSummary[langOf(config)] : STR.prompts.summary[langOf(config)] }] },
|
|
244
269
|
...messages
|
|
245
270
|
],
|
|
246
271
|
signal: controller.signal
|
|
@@ -314,8 +339,19 @@ export function createSummarizer(ctx, service, config) {
|
|
|
314
339
|
}
|
|
315
340
|
}
|
|
316
341
|
}, intervalMs);
|
|
317
|
-
if (aborted)
|
|
318
|
-
|
|
342
|
+
if (aborted) {
|
|
343
|
+
abortedRun = true;
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
const parsed = parseSummaryJson(text || assembledText);
|
|
347
|
+
// Issue #127 条数上限:0 = 不限(现状)。被截断的条数与去重命中数一并进
|
|
348
|
+
// 审计 metadata——此前产出条数完全由模型输出决定,用户无从自查。
|
|
349
|
+
const cap = config.summarizeMaxEntriesPerRun ?? 0;
|
|
350
|
+
const entries = cap > 0 ? parsed.slice(0, cap) : parsed;
|
|
351
|
+
const capped = parsed.length - entries.length;
|
|
352
|
+
const dedupeMode = config.summarizeDedupeMode ?? "off";
|
|
353
|
+
let deduped = 0;
|
|
354
|
+
let dedupeMaxSim = 0;
|
|
319
355
|
for (const entry of entries) {
|
|
320
356
|
// Provenance: the summarizer runs on a real session (turn/end hook), so
|
|
321
357
|
// session.id is always available here — it rides the human-readable
|
|
@@ -323,12 +359,45 @@ export function createSummarizer(ctx, service, config) {
|
|
|
323
359
|
// 编码记忆类型(codingRetrospect)不带 tag:读取侧门控/加权靠 m.type
|
|
324
360
|
// (rejected_solution/pitfall/constraint)区分即可,tag 体系
|
|
325
361
|
// sanitizeTags 不认 `type:` 前缀反而会清空 tags 列(额外一次 UPDATE)。
|
|
362
|
+
const source = `session:${session.id}`;
|
|
363
|
+
// Issue #127 落库前去重(opt-in,默认 off = 现状):命中则并入既有条目
|
|
364
|
+
// (saveWithDedupe 的 _mergeInto 分支),不再新造一行。作用域 = 同会话 +
|
|
365
|
+
// 时间窗;跨会话的同类事实仍交给 autoDream / sleep 的批量裁决。
|
|
366
|
+
const dup = (dedupeMode !== "off" && typeof service.findSessionDuplicate === "function")
|
|
367
|
+
? await service.findSessionDuplicate(
|
|
368
|
+
{ ...entry, source },
|
|
369
|
+
{
|
|
370
|
+
mode: dedupeMode,
|
|
371
|
+
minSim: config.summarizeDedupeMinSim ?? 0.92,
|
|
372
|
+
windowHours: config.summarizeDedupeWindowHours ?? 24,
|
|
373
|
+
source
|
|
374
|
+
}
|
|
375
|
+
)
|
|
376
|
+
: undefined;
|
|
377
|
+
if (dup) {
|
|
378
|
+
deduped++;
|
|
379
|
+
dedupeMaxSim = Math.max(dedupeMaxSim, dup.sim ?? 0);
|
|
380
|
+
}
|
|
326
381
|
service.saveWithDedupe({
|
|
327
382
|
...entry,
|
|
328
|
-
source
|
|
383
|
+
source,
|
|
384
|
+
...(dup ? { _mergeInto: dup.memory.id } : {})
|
|
329
385
|
});
|
|
330
386
|
}
|
|
387
|
+
if (audit && (capped > 0 || deduped > 0)) {
|
|
388
|
+
audit.metadata = {
|
|
389
|
+
parsed: parsed.length,
|
|
390
|
+
...(capped > 0 ? { capped } : {}),
|
|
391
|
+
...(deduped > 0 ? { deduped, mode: dedupeMode, maxSim: Number(dedupeMaxSim.toFixed(4)) } : {})
|
|
392
|
+
};
|
|
393
|
+
}
|
|
331
394
|
} finally {
|
|
395
|
+
// Issue #127:aborted(会话关闭 / 插件 dispose)不占间隔,避免误伤该会话的
|
|
396
|
+
// 下一次蒸馏;其余情况(含失败)的打点保留,与 dreamMinIntervalMinutes 一致。
|
|
397
|
+
if (abortedRun || controller.signal.aborted) {
|
|
398
|
+
if (prevClaim === undefined) lastRunAt.delete(session.id);
|
|
399
|
+
else lastRunAt.set(session.id, prevClaim);
|
|
400
|
+
}
|
|
332
401
|
if (audit) {
|
|
333
402
|
try {
|
|
334
403
|
service.saveLlmAudit({
|
|
@@ -343,7 +412,10 @@ export function createSummarizer(ctx, service, config) {
|
|
|
343
412
|
duration_ms: Date.now() - audit.startedAt,
|
|
344
413
|
status: audit.status,
|
|
345
414
|
error_message: audit.errorMessage,
|
|
346
|
-
related_memory_ids: []
|
|
415
|
+
related_memory_ids: [],
|
|
416
|
+
// Issue #127:parsed / capped / deduped 的观测数据。零 schema 变更
|
|
417
|
+
// (llm_audit_logs.metadata 已存在),一条 SQL 即可验证节流是否生效。
|
|
418
|
+
metadata: audit.metadata
|
|
347
419
|
});
|
|
348
420
|
} catch (auditError) {
|
|
349
421
|
ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
|
|
@@ -371,6 +443,7 @@ export function createSummarizer(ctx, service, config) {
|
|
|
371
443
|
unsubscribe?.();
|
|
372
444
|
for (const controller of inFlight.values()) controller.abort();
|
|
373
445
|
inFlight.clear();
|
|
446
|
+
lastRunAt.clear();
|
|
374
447
|
}
|
|
375
448
|
};
|
|
376
449
|
}
|