@modusensus/dsh-mneme 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -5
- package/lib/api-standalone.js +12 -0
- package/lib/api.js +52 -14
- package/lib/client.js +130 -8
- package/lib/config.js +8 -0
- package/lib/dream/decisions.js +55 -8
- package/lib/dream/sleep.js +79 -17
- package/lib/dream.js +61 -9
- package/lib/entities/extractor.js +2 -2
- package/lib/index.js +6 -0
- package/lib/lang.js +13 -2
- package/lib/local-embedder.js +9 -1
- package/lib/reranker.js +38 -8
- package/lib/scope.js +39 -4
- package/lib/service.js +121 -18
- package/lib/store.js +243 -40
- package/lib/summarize.js +2 -2
- package/lib/tools.js +64 -30
- package/lib/vector-index.js +2 -8
- package/lib/version-check.js +122 -0
- package/package.json +2 -1
- package/scripts/sync-test-badge.mjs +61 -0
- package/src/api-standalone.js +12 -0
- package/src/api.js +52 -14
- package/src/config.js +8 -0
- package/src/dream/decisions.js +55 -8
- package/src/dream/sleep.js +79 -17
- package/src/dream.js +61 -9
- package/src/entities/extractor.js +2 -2
- package/src/index.js +6 -0
- package/src/lang.js +13 -2
- package/src/local-embedder.js +9 -1
- package/src/reranker.js +38 -8
- package/src/scope.js +39 -4
- package/src/service.js +121 -18
- package/src/store.js +243 -40
- package/src/summarize.js +2 -2
- package/src/tools.js +64 -30
- package/src/vector-index.js +2 -8
- package/src/version-check.js +122 -0
- package/test/api.test.js +5 -0
- package/test/conflict-freeze.test.js +15 -5
- package/test/dream-archive-guardrail.test.js +116 -0
- package/test/dream-coverage-degrade.test.js +86 -0
- package/test/dream.test.js +15 -11
- package/test/inject-first-round-bm25.test.js +89 -0
- package/test/llm-message-source.test.js +177 -0
- package/test/reranker-single-logit.test.js +73 -0
- package/test/scope-candidates.test.js +239 -0
- package/test/scope-explicit.test.js +452 -0
- package/test/scope-storage.test.js +13 -5
- package/test/scope-strict.test.js +107 -41
- package/test/skip-invalid-restore.test.js +7 -5
- package/test/sleep-scheduler-rearm.test.js +89 -0
- package/test/sleep.test.js +2 -2
- package/test/standalone-api.test.js +34 -1
- package/test/store-vector-cache.test.js +98 -0
- package/test/tools-memory-get-schema.test.js +65 -0
- package/test/version-check.test.js +152 -0
package/src/dream.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { validateDecisions, applyDecisions } from "./dream/decisions.js";
|
|
2
2
|
import { clusterMemories, findPotentialConflicts, cosineSimilarity } from "./dream/clustering.js";
|
|
3
|
+
import { scopeKeyOf } from "./scope.js";
|
|
3
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
5
|
import { STR, langOf } from "./lang.js";
|
|
5
6
|
export { validateDecisions, applyDecisions, withEffortFallback, describeStreamFailure, resolveDreamEffort, resolveRoute };
|
|
@@ -909,8 +910,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
909
910
|
maxTokens: config.dreamMaxTokens ?? 4096,
|
|
910
911
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
911
912
|
messages: [
|
|
912
|
-
{ role: "system", content: [{ type: "text", text: consolidationPrompt }] },
|
|
913
|
-
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
913
|
+
{ role: "system", content: [{ type: "text", text: consolidationPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
914
|
+
{ role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
914
915
|
]
|
|
915
916
|
}, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
|
|
916
917
|
};
|
|
@@ -932,8 +933,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
932
933
|
logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0}; head: ${head})`);
|
|
933
934
|
return finish({ ok: false, error: "no json array in llm output", summary: false });
|
|
934
935
|
}
|
|
935
|
-
const { ok, errors, skipped, resolvedShortIds } = validateDecisions(decisions, snapshot, {
|
|
936
|
+
const { ok, errors, skipped, resolvedShortIds, coverageShortfall } = validateDecisions(decisions, snapshot, {
|
|
936
937
|
maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
|
|
938
|
+
// Issue #104 方向 2:archive 批量上限(全局闸门,超限整单拒绝)。
|
|
939
|
+
maxArchivePerRun: config.dreamMaxArchivePerRun,
|
|
937
940
|
minAgeHours: config.reflectionUpdateMinAgeHours,
|
|
938
941
|
// v0.4.4 fix:显式透传,用户配 dreamImplicitKeep:false 时严格模式必须
|
|
939
942
|
// 真正生效,dreamMinExplicitCoverage 决定隐式 keep 下的覆盖率下限。
|
|
@@ -971,6 +974,12 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
971
974
|
if (skippedInvalid) {
|
|
972
975
|
logger?.warn?.(`dsh-mneme dream: ${skipped.length} invalid decision(s) skipped (run degrades): ${skipped.map((s) => s.error).join("; ")}`);
|
|
973
976
|
}
|
|
977
|
+
// Issue #104 方向 1:覆盖率不足降级为 degraded(不再整单拒绝)——合法子集
|
|
978
|
+
// 照常应用、未提及条目隐式 keep,理由随 outcome.degradations 落库供离线定位。
|
|
979
|
+
const coverageDegraded = typeof coverageShortfall === "string" && coverageShortfall.length > 0;
|
|
980
|
+
if (coverageDegraded) {
|
|
981
|
+
logger?.warn?.(`dsh-mneme dream: ${coverageShortfall} (run degrades: valid subset applied, unclaimed ids kept implicitly)`);
|
|
982
|
+
}
|
|
974
983
|
|
|
975
984
|
// Capture pre-update snapshots so the audit records what each update changed.
|
|
976
985
|
const updateSnapshots = {};
|
|
@@ -989,7 +998,21 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
989
998
|
// bounds the review queue; overflow is skipped with a warning.
|
|
990
999
|
let frozenCount = 0;
|
|
991
1000
|
const frozenIds = [];
|
|
992
|
-
|
|
1001
|
+
// v0.8.1(issue #170 复核项 3):跨 scope 的 conflict 决策两种模式都一律
|
|
1002
|
+
// 停车人工裁决、绝不自动归档——否则普通 dream 的 LLM 能归档别的 scope 的
|
|
1003
|
+
// 唯一副本,sleep 路径(scope-candidate-queue)的保护在这里缺位。同 scope
|
|
1004
|
+
// 的 conflict 决策照旧:freeze 停车 / 非 freeze 自动裁决。
|
|
1005
|
+
const isCrossScopeDecision = (d) => {
|
|
1006
|
+
const a = snapshot.get(d.winner);
|
|
1007
|
+
const b = snapshot.get(d.loser);
|
|
1008
|
+
return !!a && !!b && (
|
|
1009
|
+
scopeKeyOf(a.agent_scope) !== scopeKeyOf(b.agent_scope)
|
|
1010
|
+
|| scopeKeyOf(a.workspace_scope) !== scopeKeyOf(b.workspace_scope)
|
|
1011
|
+
);
|
|
1012
|
+
};
|
|
1013
|
+
const applyList = freezeEnabled
|
|
1014
|
+
? decisions.filter((d) => d.action !== "conflict")
|
|
1015
|
+
: decisions.filter((d) => d.action !== "conflict" || !isCrossScopeDecision(d));
|
|
993
1016
|
if (freezeEnabled) {
|
|
994
1017
|
const conflictsToFreeze = decisions.filter((d) => d.action === "conflict");
|
|
995
1018
|
if (conflictsToFreeze.length > 0) {
|
|
@@ -1003,7 +1026,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1003
1026
|
}
|
|
1004
1027
|
for (const d of toFreeze) {
|
|
1005
1028
|
try {
|
|
1006
|
-
|
|
1029
|
+
// 返回 undefined = 该对已被人工裁决且两侧内容未变(issue #170 复核
|
|
1030
|
+
// 项 4:内容变了才重新入队)——不算停车、不进 frozenIds。
|
|
1031
|
+
const parked = service.saveConflictPending({ run_id: runId, memory_a: d.winner, memory_b: d.loser, reason: d.reason });
|
|
1032
|
+
if (parked == null) continue;
|
|
1007
1033
|
frozenCount++;
|
|
1008
1034
|
frozenIds.push(d.winner, d.loser);
|
|
1009
1035
|
} catch (error) {
|
|
@@ -1014,6 +1040,29 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1014
1040
|
logger?.warn?.(`dsh-mneme dream: conflict freeze lookup failed: ${String(error)}`);
|
|
1015
1041
|
}
|
|
1016
1042
|
}
|
|
1043
|
+
} else {
|
|
1044
|
+
// 非 freeze:跨 scope conflict 停车(绝不进 applyDecisions)。带专属
|
|
1045
|
+
// reason + 模型给出的裁决理由,队列里读起来是「归属裁决」而非重复项。
|
|
1046
|
+
const crossScope = decisions.filter((d) => d.action === "conflict" && isCrossScopeDecision(d));
|
|
1047
|
+
for (const d of crossScope) {
|
|
1048
|
+
try {
|
|
1049
|
+
const parked = service.saveConflictPending({
|
|
1050
|
+
run_id: runId,
|
|
1051
|
+
memory_a: d.winner,
|
|
1052
|
+
memory_b: d.loser,
|
|
1053
|
+
reason: STR.scopeConflictParkReason[language](d.reason)
|
|
1054
|
+
});
|
|
1055
|
+
if (parked != null) {
|
|
1056
|
+
frozenCount++;
|
|
1057
|
+
frozenIds.push(d.winner, d.loser);
|
|
1058
|
+
}
|
|
1059
|
+
} catch (error) {
|
|
1060
|
+
logger?.warn?.(`dsh-mneme dream: failed to park cross-scope conflict ${d.winner}/${d.loser}: ${String(error)}`);
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
if (crossScope.length > 0) {
|
|
1064
|
+
logger?.info?.(`[dsh-mneme] dream: parked ${crossScope.length} cross-scope conflict(s) for human ownership review`);
|
|
1065
|
+
}
|
|
1017
1066
|
}
|
|
1018
1067
|
|
|
1019
1068
|
// CAS-guarded, per-decision-transactional apply against the run snapshot:
|
|
@@ -1046,6 +1095,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1046
1095
|
// claim "merge-archived" (item ②). Conflicts/failures ride along so the
|
|
1047
1096
|
// audit row records why the run diverged.
|
|
1048
1097
|
const outcome = { ...buildOutcome(committed), conflicts, failures };
|
|
1098
|
+
if (coverageDegraded) outcome.degradations = [coverageShortfall];
|
|
1049
1099
|
// Frozen conflicts were not adjudicated: mark both sides pending in the
|
|
1050
1100
|
// per-id outcome so the audit row shows they were parked, not skipped.
|
|
1051
1101
|
if (frozenIds.length) {
|
|
@@ -1089,8 +1139,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1089
1139
|
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
1090
1140
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
1091
1141
|
messages: [
|
|
1092
|
-
{ role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }] },
|
|
1093
|
-
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
|
|
1142
|
+
{ role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
1143
|
+
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
1094
1144
|
]
|
|
1095
1145
|
}, reportUsage, (reason) => { summaryStreamFailure = describeStreamFailure(reason); }));
|
|
1096
1146
|
};
|
|
@@ -1130,7 +1180,9 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1130
1180
|
// degraded — real consolidation landed but the run did not produce its
|
|
1131
1181
|
// full output: the summary came back empty/missing, or
|
|
1132
1182
|
// skipInvalid dropped individually-invalid decisions
|
|
1133
|
-
// (Issue #89 — marked, not faked)
|
|
1183
|
+
// (Issue #89 — marked, not faked), or the explicit-decision
|
|
1184
|
+
// coverage fell below dreamMinExplicitCoverage (Issue #104
|
|
1185
|
+
// 方向 1 — degrade, don't reject). The valid subset was
|
|
1134
1186
|
// absorbed (ok for the baseline).
|
|
1135
1187
|
let status;
|
|
1136
1188
|
let okResult;
|
|
@@ -1141,7 +1193,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1141
1193
|
status = summaryStored ? "ok" : "noop";
|
|
1142
1194
|
okResult = summaryStored;
|
|
1143
1195
|
} else {
|
|
1144
|
-
status = summaryStored && !skippedInvalid ? "ok" : "degraded";
|
|
1196
|
+
status = summaryStored && !skippedInvalid && !coverageDegraded ? "ok" : "degraded";
|
|
1145
1197
|
okResult = true;
|
|
1146
1198
|
}
|
|
1147
1199
|
return finish({
|
|
@@ -150,8 +150,8 @@ export async function extractEntities(memory, { store, config, callLLM, logger }
|
|
|
150
150
|
const userText = buildUserMessage(memory.content);
|
|
151
151
|
|
|
152
152
|
const messages = [
|
|
153
|
-
{ role: "system", content: [{ type: "text", text: systemPrompt }] },
|
|
154
|
-
{ role: "user", content: [{ type: "text", text: userText }] }
|
|
153
|
+
{ role: "system", content: [{ type: "text", text: systemPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
154
|
+
{ role: "user", content: [{ type: "text", text: userText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
155
155
|
];
|
|
156
156
|
|
|
157
157
|
// Issue #109: optional provider/model override + reasoning effort are
|
package/src/index.js
CHANGED
|
@@ -275,6 +275,8 @@ export const apply = (ctx, config) => {
|
|
|
275
275
|
batchSize: cfg.localEmbedBatchSize,
|
|
276
276
|
cacheDir: cfg.embedModelCacheDir,
|
|
277
277
|
runtimeDir: cfg.runtimeDir,
|
|
278
|
+
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
279
|
+
remoteHost: cfg.embedModelMirror,
|
|
278
280
|
baseUrl: cfg.ollamaBaseUrl,
|
|
279
281
|
logger: ctx.logger
|
|
280
282
|
});
|
|
@@ -327,6 +329,10 @@ export const apply = (ctx, config) => {
|
|
|
327
329
|
device: cfg.localEmbedDevice,
|
|
328
330
|
cacheDir: cfg.embedModelCacheDir,
|
|
329
331
|
runtimeDir: cfg.runtimeDir,
|
|
332
|
+
// #188:量化档默认 q8(此前不传 dtype 会去要 1GB 级 fp32 模型);
|
|
333
|
+
// embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
|
|
334
|
+
useDtype: cfg.rerankDtype,
|
|
335
|
+
remoteHost: cfg.embedModelMirror,
|
|
330
336
|
logger: ctx.logger
|
|
331
337
|
});
|
|
332
338
|
service.setReranker(reranker);
|
package/src/lang.js
CHANGED
|
@@ -23,8 +23,8 @@ const PROMPTS = {
|
|
|
23
23
|
"en": "You are the memory overview assistant. From the consolidated memories below, produce a 150-200 word memory-library overview covering: user preferences, active projects, key decisions. This will be injected as conversation context. Output only the summary text, nothing else."
|
|
24
24
|
},
|
|
25
25
|
"consolidation": {
|
|
26
|
-
"zh": "你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。\n请执行记忆巩固(consolidation),输出一个决策 JSON 数组。\n\n【决策格式(必须严格遵守)】\n每个决策必须是对象,字段固定:\n- \"action\":必填。取值只能是 \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" 之一(字段名必须是 action,严禁写成 type)\n- \"ids\":必填,数组,本决策涉及的记忆 id 列表\n- \"reason\":可选,字符串,决策理由\n- \"importance\":可选,整数 1-5\n- merge 额外字段:\"keepSource\"(单个 id 字符串,必须是 ids 之一)+ 合并后的 \"title\"、\"content\"\n- conflict 额外字段:\"winner\" 与 \"loser\",都是【单个 id 字符串,不是数组】\n- update 额外字段:修正后的 \"title\" 和/或 \"content\";\"ids\" 只能包含一个 id\n\n【决策 JSON 示例】\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"合并标题\", \"content\": \"合并后的摘要内容\", \"importance\": 4, \"reason\": \"主题相近\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"内容矛盾,保留更新的信息\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"修正后的内容\", \"reason\": \"信息过时\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"重复或过时\" }\n]\n\n【任务】\n1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)\n2. 识别重复/过时信息 → archive\n3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)\n4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)\n - update 的 ids 只能包含一个 id\n - 必须提供修正后的 title 和/或 content\n - 仅当内容确实需要修正时才使用,不要滥用\n - 每次整理最多输出 2 个 update\n - 24 小时内新建的记忆不可 update\n5. 无问题的条目无需输出(未提及的条目将自动保留 keep)\n\n【硬性规则】\n- 字段名必须精确为 \"action\",严禁写成 \"type\";字段名统一用双引号\n- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】\n- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)\n- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep\n- merge 的 keepSource 必须是 ids 之一\n- 仅合并同类型条目(type 相同)\n- 不要编造 ids;只使用提供的 id\n- 重要性 1-5,合并后取最高\n- 只输出 JSON 数组,不要其他文字",
|
|
27
|
-
"en": "You are the memory consolidation assistant. Below are all memory entries (id, type, title, content, importance, updated time).\nPerform memory consolidation and output a JSON array of decisions.\n\n[Decision format (must be followed strictly)]\nEach decision must be an object with fixed fields:\n- \"action\": required. Value must be one of \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" (the field name must be action, never type)\n- \"ids\": required, array, the list of memory ids involved in this decision\n- \"reason\": optional, string, rationale for the decision\n- \"importance\": optional, integer 1-5\n- merge extra fields: \"keepSource\" (a single id string, must be one of ids) plus the merged \"title\" and \"content\"\n- conflict extra fields: \"winner\" and \"loser\", each a [single id string, never an array]\n- update extra fields: the corrected \"title\" and/or \"content\"; \"ids\" may contain only one id\n\n[Decision JSON examples]\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"Merged title\", \"content\": \"Merged summary content\", \"importance\": 4, \"reason\": \"similar topics\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"content contradicts; keep the newer information\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"corrected content\", \"reason\": \"information outdated\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"duplicate or outdated\" }\n]\n\n[Task]\n1. Identify entries with similar topics -> merge (merge into a more refined summary; keep the id with the most complete information as keepSource)\n2. Identify duplicate/outdated information -> archive\n3. Identify entries with contradictory content -> conflict (judge winner/loser by recency, source completeness, and information specificity)\n4. Find outdated, wrong, or missing information in a single memory -> update (correct the content directly)\n - update's ids may contain only one id\n - must provide the corrected title and/or content\n - use only when the content truly needs correction; do not abuse\n - output at most 2 updates per consolidation run\n - memories created within the last 24 hours must not be updated\n5. Entries without issues need no output (unmentioned entries are automatically kept)\n\n[Hard rules]\n- Field names must be exactly \"action\", never \"type\"; use double quotes for all field names\n- conflict winner/loser and merge keepSource are [single id strings, never arrays]\n- Each memory may be claimed at most once: the same id must not appear in multiple decisions (an id must not be claimed by both merge and conflict/archive, etc.)\n- Memories not mentioned in any decision are automatically kept (keep); no need to output keep for every memory\n- merge keepSource must be one of ids\n- Only merge entries of the same type (identical type)\n- Never invent ids; use only the provided ids\n- Importance is 1-5; after a merge take the highest\n- Write every title, content, and reason in English\n- Output only the JSON array, nothing else"
|
|
26
|
+
"zh": "你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。\n请执行记忆巩固(consolidation),输出一个决策 JSON 数组。\n\n【决策格式(必须严格遵守)】\n每个决策必须是对象,字段固定:\n- \"action\":必填。取值只能是 \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" 之一(字段名必须是 action,严禁写成 type)\n- \"ids\":必填,数组,本决策涉及的记忆 id 列表\n- \"reason\":可选,字符串,决策理由\n- \"importance\":可选,整数 1-5\n- merge 额外字段:\"keepSource\"(单个 id 字符串,必须是 ids 之一)+ 合并后的 \"title\"、\"content\"\n- conflict 额外字段:\"winner\" 与 \"loser\",都是【单个 id 字符串,不是数组】\n- update 额外字段:修正后的 \"title\" 和/或 \"content\";\"ids\" 只能包含一个 id\n\n【决策 JSON 示例】\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"合并标题\", \"content\": \"合并后的摘要内容\", \"importance\": 4, \"reason\": \"主题相近\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"内容矛盾,保留更新的信息\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"修正后的内容\", \"reason\": \"信息过时\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"重复或过时\" }\n]\n\n【任务】\n1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)\n2. 识别重复/过时信息 → archive\n3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)\n4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)\n - update 的 ids 只能包含一个 id\n - 必须提供修正后的 title 和/或 content\n - 仅当内容确实需要修正时才使用,不要滥用\n - 每次整理最多输出 2 个 update\n - 24 小时内新建的记忆不可 update\n5. 无问题的条目无需输出(未提及的条目将自动保留 keep)\n\n【硬性规则】\n- 字段名必须精确为 \"action\",严禁写成 \"type\";字段名统一用双引号\n- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】\n- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)\n- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep\n- merge 的 keepSource 必须是 ids 之一\n- 仅合并同类型条目(type 相同)\n- 不要编造 ids;只使用提供的 id\n- 重要性 1-5,合并后取最高\n- archive 仅用于确属「重复或已过时」的条目;rejected_solution / constraint / pitfall / preference / pattern 属长保留类型,除非确属重复或过时,否则不要归档\n- 每次整理最多输出 8 个 archive\n- 只输出 JSON 数组,不要其他文字",
|
|
27
|
+
"en": "You are the memory consolidation assistant. Below are all memory entries (id, type, title, content, importance, updated time).\nPerform memory consolidation and output a JSON array of decisions.\n\n[Decision format (must be followed strictly)]\nEach decision must be an object with fixed fields:\n- \"action\": required. Value must be one of \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" (the field name must be action, never type)\n- \"ids\": required, array, the list of memory ids involved in this decision\n- \"reason\": optional, string, rationale for the decision\n- \"importance\": optional, integer 1-5\n- merge extra fields: \"keepSource\" (a single id string, must be one of ids) plus the merged \"title\" and \"content\"\n- conflict extra fields: \"winner\" and \"loser\", each a [single id string, never an array]\n- update extra fields: the corrected \"title\" and/or \"content\"; \"ids\" may contain only one id\n\n[Decision JSON examples]\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"Merged title\", \"content\": \"Merged summary content\", \"importance\": 4, \"reason\": \"similar topics\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"content contradicts; keep the newer information\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"corrected content\", \"reason\": \"information outdated\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"duplicate or outdated\" }\n]\n\n[Task]\n1. Identify entries with similar topics -> merge (merge into a more refined summary; keep the id with the most complete information as keepSource)\n2. Identify duplicate/outdated information -> archive\n3. Identify entries with contradictory content -> conflict (judge winner/loser by recency, source completeness, and information specificity)\n4. Find outdated, wrong, or missing information in a single memory -> update (correct the content directly)\n - update's ids may contain only one id\n - must provide the corrected title and/or content\n - use only when the content truly needs correction; do not abuse\n - output at most 2 updates per consolidation run\n - memories created within the last 24 hours must not be updated\n5. Entries without issues need no output (unmentioned entries are automatically kept)\n\n[Hard rules]\n- Field names must be exactly \"action\", never \"type\"; use double quotes for all field names\n- conflict winner/loser and merge keepSource are [single id strings, never arrays]\n- Each memory may be claimed at most once: the same id must not appear in multiple decisions (an id must not be claimed by both merge and conflict/archive, etc.)\n- Memories not mentioned in any decision are automatically kept (keep); no need to output keep for every memory\n- merge keepSource must be one of ids\n- Only merge entries of the same type (identical type)\n- Never invent ids; use only the provided ids\n- Importance is 1-5; after a merge take the highest\n- Write every title, content, and reason in English\n- archive is only for entries that are genuinely \"duplicate or outdated\"; rejected_solution / constraint / pitfall / preference / pattern are long-retention types — do not archive them unless they are truly duplicate or outdated\n- output at most 8 archives per consolidation run\n- Output only the JSON array, nothing else"
|
|
28
28
|
},
|
|
29
29
|
"conflict": {
|
|
30
30
|
"zh": "你是记忆库冲突仲裁助手。下面是检测到的高相似度记忆对,可能内容矛盾或重复。\n对每一对输出一个 decision 对象:\n- 两条确实矛盾/重复 → { \"action\": \"conflict\", \"winner\": <保留的id>, \"loser\": <归档的id>, \"reason\": \"理由\" }\n- 两条只是主题相近、并无矛盾 → { \"action\": \"keep\", \"ids\": [<两个id>] }\n规则:\n- winner 应为信息更完整、更新或更可信的一条\n- 只使用提供的 id,不要编造\n- 每对必须输出一个 decision\n- 只输出 JSON 数组,不要其他文字",
|
|
@@ -103,6 +103,17 @@ export const STR = {
|
|
|
103
103
|
|
|
104
104
|
// --- dream/sleep.js:冲突候选列表 --------------------------------------------
|
|
105
105
|
similarityReason: { zh: (s) => `相似度 ${s}`, en: (s) => `similarity ${s}` },
|
|
106
|
+
// v0.8.1(issue #170 第 2 步):跨 scope 相似对停车到冲突队列的专属 reason。
|
|
107
|
+
scopeCandidateReason: {
|
|
108
|
+
zh: (s) => `跨作用域相似 ${s}(疑似同一内容落在两个归属下,请裁决归属/去重)`,
|
|
109
|
+
en: (s) => `cross-scope similar ${s} (same content under two scopes — review ownership)`
|
|
110
|
+
},
|
|
111
|
+
// v0.8.1(issue #170 复核项 3):普通 dream 的 LLM 输出跨 scope conflict 时
|
|
112
|
+
// 停车用(相似度未知,带模型给的裁决理由)。
|
|
113
|
+
scopeConflictParkReason: {
|
|
114
|
+
zh: (r) => `跨作用域冲突(归属不同,不自动裁决)${r ? `:${r}` : ""}`,
|
|
115
|
+
en: (r) => `cross-scope conflict (different ownership — not auto-adjudicated)${r ? `: ${r}` : ""}`
|
|
116
|
+
},
|
|
106
117
|
candidateConflicts: {
|
|
107
118
|
zh: (p) => `候选冲突:\nid=${p.a.id} | type=${p.a.type} | title=${p.a.title}\n${p.a.content}\n---\nid=${p.b.id} | type=${p.b.type} | title=${p.b.title}\n${p.b.content}\n(相似度 ${p.similarity.toFixed(2)})`,
|
|
108
119
|
en: (p) => `Candidate conflicts:\nid=${p.a.id} | type=${p.a.type} | title=${p.a.title}\n${p.a.content}\n---\nid=${p.b.id} | type=${p.b.type} | title=${p.b.title}\n${p.b.content}\n(similarity ${p.similarity.toFixed(2)})`
|
package/src/local-embedder.js
CHANGED
|
@@ -30,7 +30,7 @@ function modelHash(model) {
|
|
|
30
30
|
* transformers 的选项里。
|
|
31
31
|
*/
|
|
32
32
|
async function defaultPipelineLoader(task, model, options) {
|
|
33
|
-
const { runtimeDir, ...pipelineOptions } = options ?? {};
|
|
33
|
+
const { runtimeDir, remoteHost, ...pipelineOptions } = options ?? {};
|
|
34
34
|
const { module } = await loadTransformers({
|
|
35
35
|
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
36
36
|
});
|
|
@@ -41,6 +41,11 @@ async function defaultPipelineLoader(task, model, options) {
|
|
|
41
41
|
// model is fully cached locally. Mirroring the cache_dir onto env.cacheDir
|
|
42
42
|
// makes that pre-check resolve locally too — fully offline loading.
|
|
43
43
|
if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
|
|
44
|
+
// issue #188 补充:embedModelMirror 此前是死配置(定义了但全仓无人读取)。
|
|
45
|
+
// transformers.js 不认 HF_ENDPOINT(那是 Python huggingface_hub 的变量),
|
|
46
|
+
// 模型下载源由 env.remoteHost 决定——把镜像接到真正的开关上,否则
|
|
47
|
+
// huggingface.co 不可达的网络连模型都下不下来。
|
|
48
|
+
if (remoteHost) env.remoteHost = remoteHost;
|
|
44
49
|
return pipeline(task, model, pipelineOptions);
|
|
45
50
|
}
|
|
46
51
|
|
|
@@ -72,6 +77,8 @@ export class LocalEmbedder {
|
|
|
72
77
|
// 自管运行时根目录(issue #131):空表示用默认位置,具体解析交给 loader。
|
|
73
78
|
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
74
79
|
this.useDtype = opts.useDtype || "q8";
|
|
80
|
+
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
81
|
+
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
75
82
|
this.logger = opts.logger ?? null;
|
|
76
83
|
// Test hook: replace the pipeline factory without touching modules.
|
|
77
84
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
@@ -91,6 +98,7 @@ export class LocalEmbedder {
|
|
|
91
98
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
92
99
|
// 传给注入的 engineFactory,由 defaultPipelineLoader 取走后交给三层解析。
|
|
93
100
|
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
101
|
+
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
94
102
|
this.extractor = await this.engineFactory("feature-extraction", this.model, options);
|
|
95
103
|
this.ready = true; // service reads this to flush queued re-embeds
|
|
96
104
|
this.logger?.info?.(
|
package/src/reranker.js
CHANGED
|
@@ -27,7 +27,7 @@ function modelHash(model) {
|
|
|
27
27
|
* runtimeDir 取走后不混进 transformers 的选项。
|
|
28
28
|
*/
|
|
29
29
|
async function defaultPipelineLoader(task, model, options) {
|
|
30
|
-
const { runtimeDir, ...pipelineOptions } = options ?? {};
|
|
30
|
+
const { runtimeDir, remoteHost, ...pipelineOptions } = options ?? {};
|
|
31
31
|
const { module } = await loadTransformers({
|
|
32
32
|
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
33
33
|
});
|
|
@@ -36,6 +36,10 @@ async function defaultPipelineLoader(task, model, options) {
|
|
|
36
36
|
// 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
|
|
37
37
|
// embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
|
|
38
38
|
if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
|
|
39
|
+
// issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
|
|
40
|
+
// HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
|
|
41
|
+
// 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
|
|
42
|
+
if (remoteHost) env.remoteHost = remoteHost;
|
|
39
43
|
return pipeline(task, model, pipelineOptions);
|
|
40
44
|
}
|
|
41
45
|
|
|
@@ -85,6 +89,11 @@ export class LocalReranker {
|
|
|
85
89
|
this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
|
|
86
90
|
// 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
|
|
87
91
|
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
92
|
+
// 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
|
|
93
|
+
// 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
|
|
94
|
+
this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
|
|
95
|
+
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
96
|
+
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
88
97
|
this.logger = opts.logger ?? null;
|
|
89
98
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
90
99
|
// Injectable seam: async (query, passage) => number. When set, init()
|
|
@@ -94,6 +103,7 @@ export class LocalReranker {
|
|
|
94
103
|
this._batchScorer = null;
|
|
95
104
|
this._queryVec = null;
|
|
96
105
|
this._queryKey = null;
|
|
106
|
+
this._constantScoreWarned = false;
|
|
97
107
|
}
|
|
98
108
|
|
|
99
109
|
/** Load the model; throws when no strategy can be bound. */
|
|
@@ -119,9 +129,10 @@ export class LocalReranker {
|
|
|
119
129
|
}
|
|
120
130
|
|
|
121
131
|
_engineOptions() {
|
|
122
|
-
const options = { device: this.device };
|
|
132
|
+
const options = { device: this.device, dtype: this.useDtype };
|
|
123
133
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
124
134
|
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
135
|
+
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
125
136
|
return options;
|
|
126
137
|
}
|
|
127
138
|
|
|
@@ -136,8 +147,14 @@ export class LocalReranker {
|
|
|
136
147
|
return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
|
|
137
148
|
};
|
|
138
149
|
} else if (strategy === "tc") {
|
|
139
|
-
// Classification head:
|
|
140
|
-
//
|
|
150
|
+
// Classification head: score so relevance lands in [0, 1]. Branch on the
|
|
151
|
+
// output width (issue #188): a dual-logit head (num_labels=2 exports,
|
|
152
|
+
// e.g. bge-reranker-v2-m3) gives the positive class as the softmax
|
|
153
|
+
// probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
|
|
154
|
+
// Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
|
|
155
|
+
// the positive class is simply sigmoid(logit) — reading l1 as l0 there
|
|
156
|
+
// collapses every score to sigmoid(0) = 0.5 and silently disables
|
|
157
|
+
// reranking (no error, status still "ready").
|
|
141
158
|
this._batchScorer = async (query, passages) => {
|
|
142
159
|
const { tokenizer, model } = this.pipeline;
|
|
143
160
|
const inputs = tokenizer(passages.map(() => query), {
|
|
@@ -151,10 +168,10 @@ export class LocalReranker {
|
|
|
151
168
|
const rows = [];
|
|
152
169
|
for (let i = 0; i < dims[0]; i++) {
|
|
153
170
|
const base = i * cols;
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
rows.push(clamp01(
|
|
171
|
+
const score = cols > 1
|
|
172
|
+
? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
|
|
173
|
+
: 1 / (1 + Math.exp(-logits.data[base]));
|
|
174
|
+
rows.push(clamp01(score));
|
|
158
175
|
}
|
|
159
176
|
return rows;
|
|
160
177
|
};
|
|
@@ -208,6 +225,19 @@ export class LocalReranker {
|
|
|
208
225
|
results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
|
|
209
226
|
}
|
|
210
227
|
}
|
|
228
|
+
// Silent-failure tripwire (issue #188): a reranker whose every candidate
|
|
229
|
+
// scores identically is not ranking anything — e.g. a single-logit head
|
|
230
|
+
// read through the dual-logit formula. Warn once so the degradation is
|
|
231
|
+
// observable instead of a constant 0.5 column hiding behind a "ready"
|
|
232
|
+
// status forever.
|
|
233
|
+
if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
|
|
234
|
+
if (!this._constantScoreWarned) {
|
|
235
|
+
this._constantScoreWarned = true;
|
|
236
|
+
this.logger?.warn?.(
|
|
237
|
+
`[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
211
241
|
return results
|
|
212
242
|
.filter((r) => r.score >= this.scoreThreshold)
|
|
213
243
|
.sort((a, b) => b.score - a.score);
|
package/src/scope.js
CHANGED
|
@@ -8,8 +8,11 @@
|
|
|
8
8
|
// 硬约束:解析失败绝不阻塞写入——任何一步取不到就落 NULL(读侧 NULL = 未标注
|
|
9
9
|
// = 全局可见),任何异常都被吞掉(registry 故障 warn 一次后降级)。
|
|
10
10
|
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
11
|
+
// v0.8.1(issue #170 复核项 2):scopeEnabled 的闸门从解析器挪到「写标注」处
|
|
12
|
+
// (memory_save 的 auto 分支)。解析器恒返回会话身份(真取不到才是 null)——
|
|
13
|
+
// 读取侧(strictVisibility / memory_get / inject)与显式声明都依赖它:flag 关
|
|
14
|
+
// 只关「自动标注」,不能连读取身份一起关,否则 strictScope 硬墙被静默拆掉
|
|
15
|
+
// (fail-open)。
|
|
13
16
|
|
|
14
17
|
/**
|
|
15
18
|
* @param {object} deps
|
|
@@ -31,8 +34,6 @@ export function createScopeResolver({ ctx, config, logger } = {}) {
|
|
|
31
34
|
};
|
|
32
35
|
|
|
33
36
|
return function resolveWriteScope(exec) {
|
|
34
|
-
if (config?.scopeEnabled !== true) return null;
|
|
35
|
-
|
|
36
37
|
let session = null;
|
|
37
38
|
try {
|
|
38
39
|
session = exec?.agent?.session ?? null;
|
|
@@ -81,3 +82,37 @@ export function createScopeResolver({ ctx, config, logger } = {}) {
|
|
|
81
82
|
return { agent_scope: agentScope, workspace_scope: workspaceScope };
|
|
82
83
|
};
|
|
83
84
|
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* v0.8.1 底座(issue #170):显式 scope 参数(memory_save / memory_update /
|
|
88
|
+
* 面板编辑)的归一化。约定:
|
|
89
|
+
* - 省略(undefined,由调用方判断)= 该维不声明,回落载体自动标注;
|
|
90
|
+
* - "global"(大小写不敏感)/ "*" / 空白 = 显式声明全局 → 存储 NULL
|
|
91
|
+
* (行上的 agent_scope_source='explicit' 承担「显式全局」与「从未标注」
|
|
92
|
+
* 的区分,NULL 本身继续表示全局可见);
|
|
93
|
+
* - 其余非空 trim 字符串 = 收窄到该标签。
|
|
94
|
+
* 脏输入(非字符串)落 NULL(等同声明全局)——与 resolveWriteScope 的「解析
|
|
95
|
+
* 绝不阻塞写入」同一哲学:显式声明是用户意图,形状不对时宁可放宽不报错。
|
|
96
|
+
*
|
|
97
|
+
* @param {unknown} raw 工具/HTTP 参数原值
|
|
98
|
+
* @returns {string|null} 归一化后的标签值(null=显式全局)
|
|
99
|
+
*/
|
|
100
|
+
export function normalizeExplicitScope(raw) {
|
|
101
|
+
if (typeof raw !== "string") return null;
|
|
102
|
+
const s = raw.trim();
|
|
103
|
+
if (!s || s === "*" || s.toLowerCase() === "global") return null;
|
|
104
|
+
return s;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* scope 标注的比较键(去重键/检索加权/跨 scope 配对共用口径):非空 trim 字符串
|
|
109
|
+
* 原样返回,其余(NULL/undefined/空串/非字符串)统一为 null(「未标注」)。
|
|
110
|
+
* v0.8.1(issue #170 第 2 步)从 service.js 上提为共享定义——sleep 的跨 scope
|
|
111
|
+
* 候选配对必须与去重键用同一把钥匙,NULL 与已标注才不会错配。
|
|
112
|
+
*
|
|
113
|
+
* @param {unknown} v scope 标注原值
|
|
114
|
+
* @returns {string|null} 比较键(null=未标注)
|
|
115
|
+
*/
|
|
116
|
+
export function scopeKeyOf(v) {
|
|
117
|
+
return typeof v === "string" && v.trim() ? v.trim() : null;
|
|
118
|
+
}
|