@modusensus/dsh-mneme 0.8.1 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +14 -3
  2. package/lib/api-standalone.js +12 -0
  3. package/lib/api.js +12 -2
  4. package/lib/client.js +5 -1
  5. package/lib/config.js +18 -0
  6. package/lib/dream/decisions.js +55 -8
  7. package/lib/dream/sleep.js +36 -5
  8. package/lib/dream.js +18 -7
  9. package/lib/entities/extractor.js +2 -2
  10. package/lib/index.js +8 -0
  11. package/lib/inject.js +43 -1
  12. package/lib/lang.js +2 -2
  13. package/lib/local-embedder.js +21 -1
  14. package/lib/reranker.js +47 -8
  15. package/lib/runtime/model-fetch.js +361 -0
  16. package/lib/service.js +37 -5
  17. package/lib/settings.js +1 -0
  18. package/lib/store.js +85 -25
  19. package/lib/summarize.js +2 -2
  20. package/lib/tools.js +6 -19
  21. package/lib/vector-index.js +2 -8
  22. package/package.json +2 -1
  23. package/scripts/sync-test-badge.mjs +61 -0
  24. package/src/api-standalone.js +12 -0
  25. package/src/api.js +12 -2
  26. package/src/config.js +18 -0
  27. package/src/dream/decisions.js +55 -8
  28. package/src/dream/sleep.js +36 -5
  29. package/src/dream.js +18 -7
  30. package/src/entities/extractor.js +2 -2
  31. package/src/index.js +8 -0
  32. package/src/inject.js +43 -1
  33. package/src/lang.js +2 -2
  34. package/src/local-embedder.js +21 -1
  35. package/src/reranker.js +47 -8
  36. package/src/runtime/model-fetch.js +361 -0
  37. package/src/service.js +37 -5
  38. package/src/settings.js +1 -0
  39. package/src/store.js +85 -25
  40. package/src/summarize.js +2 -2
  41. package/src/tools.js +6 -19
  42. package/src/vector-index.js +2 -8
  43. package/test/api.test.js +10 -4
  44. package/test/dream-archive-guardrail.test.js +116 -0
  45. package/test/dream-coverage-degrade.test.js +86 -0
  46. package/test/dream.test.js +15 -11
  47. package/test/inject-first-round-bm25.test.js +89 -0
  48. package/test/inject-rotation.test.js +161 -0
  49. package/test/llm-message-source.test.js +177 -0
  50. package/test/local-embedder.test.js +20 -0
  51. package/test/reranker-single-logit.test.js +73 -0
  52. package/test/runtime-model-fetch.test.js +348 -0
  53. package/test/skip-invalid-restore.test.js +7 -5
  54. package/test/sleep-scheduler-rearm.test.js +89 -0
  55. package/test/sleep.test.js +2 -2
  56. package/test/standalone-api.test.js +34 -1
  57. package/test/store-vector-cache.test.js +98 -0
  58. package/test/tools-memory-get-schema.test.js +65 -0
package/src/dream.js CHANGED
@@ -910,8 +910,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
910
910
  maxTokens: config.dreamMaxTokens ?? 4096,
911
911
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
912
912
  messages: [
913
- { role: "system", content: [{ type: "text", text: consolidationPrompt }] },
914
- { role: "user", content: [{ type: "text", text: listText }] }
913
+ { role: "system", content: [{ type: "text", text: consolidationPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
914
+ { role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
915
915
  ]
916
916
  }, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
917
917
  };
@@ -933,8 +933,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
933
933
  logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0}; head: ${head})`);
934
934
  return finish({ ok: false, error: "no json array in llm output", summary: false });
935
935
  }
936
- const { ok, errors, skipped, resolvedShortIds } = validateDecisions(decisions, snapshot, {
936
+ const { ok, errors, skipped, resolvedShortIds, coverageShortfall } = validateDecisions(decisions, snapshot, {
937
937
  maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
938
+ // Issue #104 方向 2:archive 批量上限(全局闸门,超限整单拒绝)。
939
+ maxArchivePerRun: config.dreamMaxArchivePerRun,
938
940
  minAgeHours: config.reflectionUpdateMinAgeHours,
939
941
  // v0.4.4 fix:显式透传,用户配 dreamImplicitKeep:false 时严格模式必须
940
942
  // 真正生效,dreamMinExplicitCoverage 决定隐式 keep 下的覆盖率下限。
@@ -972,6 +974,12 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
972
974
  if (skippedInvalid) {
973
975
  logger?.warn?.(`dsh-mneme dream: ${skipped.length} invalid decision(s) skipped (run degrades): ${skipped.map((s) => s.error).join("; ")}`);
974
976
  }
977
+ // Issue #104 方向 1:覆盖率不足降级为 degraded(不再整单拒绝)——合法子集
978
+ // 照常应用、未提及条目隐式 keep,理由随 outcome.degradations 落库供离线定位。
979
+ const coverageDegraded = typeof coverageShortfall === "string" && coverageShortfall.length > 0;
980
+ if (coverageDegraded) {
981
+ logger?.warn?.(`dsh-mneme dream: ${coverageShortfall} (run degrades: valid subset applied, unclaimed ids kept implicitly)`);
982
+ }
975
983
 
976
984
  // Capture pre-update snapshots so the audit records what each update changed.
977
985
  const updateSnapshots = {};
@@ -1087,6 +1095,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1087
1095
  // claim "merge-archived" (item ②). Conflicts/failures ride along so the
1088
1096
  // audit row records why the run diverged.
1089
1097
  const outcome = { ...buildOutcome(committed), conflicts, failures };
1098
+ if (coverageDegraded) outcome.degradations = [coverageShortfall];
1090
1099
  // Frozen conflicts were not adjudicated: mark both sides pending in the
1091
1100
  // per-id outcome so the audit row shows they were parked, not skipped.
1092
1101
  if (frozenIds.length) {
@@ -1130,8 +1139,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1130
1139
  maxTokens: config.dreamMaxTokens ?? 2048,
1131
1140
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
1132
1141
  messages: [
1133
- { role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }] },
1134
- { role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
1142
+ { role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
1143
+ { role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin", plugin: "dsh-mneme" } }
1135
1144
  ]
1136
1145
  }, reportUsage, (reason) => { summaryStreamFailure = describeStreamFailure(reason); }));
1137
1146
  };
@@ -1171,7 +1180,9 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1171
1180
  // degraded — real consolidation landed but the run did not produce its
1172
1181
  // full output: the summary came back empty/missing, or
1173
1182
  // skipInvalid dropped individually-invalid decisions
1174
- // (Issue #89 — marked, not faked). The valid subset was
1183
+ // (Issue #89 — marked, not faked), or the explicit-decision
1184
+ // coverage fell below dreamMinExplicitCoverage (Issue #104
1185
+ // 方向 1 — degrade, don't reject). The valid subset was
1175
1186
  // absorbed (ok for the baseline).
1176
1187
  let status;
1177
1188
  let okResult;
@@ -1182,7 +1193,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1182
1193
  status = summaryStored ? "ok" : "noop";
1183
1194
  okResult = summaryStored;
1184
1195
  } else {
1185
- status = summaryStored && !skippedInvalid ? "ok" : "degraded";
1196
+ status = summaryStored && !skippedInvalid && !coverageDegraded ? "ok" : "degraded";
1186
1197
  okResult = true;
1187
1198
  }
1188
1199
  return finish({
@@ -150,8 +150,8 @@ export async function extractEntities(memory, { store, config, callLLM, logger }
150
150
  const userText = buildUserMessage(memory.content);
151
151
 
152
152
  const messages = [
153
- { role: "system", content: [{ type: "text", text: systemPrompt }] },
154
- { role: "user", content: [{ type: "text", text: userText }] }
153
+ { role: "system", content: [{ type: "text", text: systemPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
154
+ { role: "user", content: [{ type: "text", text: userText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
155
155
  ];
156
156
 
157
157
  // Issue #109: optional provider/model override + reasoning effort are
package/src/index.js CHANGED
@@ -275,6 +275,9 @@ export const apply = (ctx, config) => {
275
275
  batchSize: cfg.localEmbedBatchSize,
276
276
  cacheDir: cfg.embedModelCacheDir,
277
277
  runtimeDir: cfg.runtimeDir,
278
+ // #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
279
+ remoteHost: cfg.embedModelMirror,
280
+ resilientModelDownload: cfg.resilientModelDownload,
278
281
  baseUrl: cfg.ollamaBaseUrl,
279
282
  logger: ctx.logger
280
283
  });
@@ -327,6 +330,11 @@ export const apply = (ctx, config) => {
327
330
  device: cfg.localEmbedDevice,
328
331
  cacheDir: cfg.embedModelCacheDir,
329
332
  runtimeDir: cfg.runtimeDir,
333
+ // #188:量化档默认 q8(此前不传 dtype 会去要 1GB 级 fp32 模型);
334
+ // embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
335
+ useDtype: cfg.rerankDtype,
336
+ remoteHost: cfg.embedModelMirror,
337
+ resilientModelDownload: cfg.resilientModelDownload,
330
338
  logger: ctx.logger
331
339
  });
332
340
  service.setReranker(reranker);
package/src/inject.js CHANGED
@@ -94,6 +94,41 @@ export function createInjector(ctx, service, settings, config) {
94
94
  const language = langOf(config);
95
95
  const maxItems = config.maxInjectedItems ?? 5;
96
96
  const threshold = config.importanceThreshold ?? 3;
97
+ // Issue #205:注入位跨轮轮换。rotationTurns = 最近 N 个「不同用户查询」轮次
98
+ // 注入过的记忆本轮不再优先(0 = 关闭,保持既有行为)。历史按会话维护——
99
+ // 新会话从零开始;同一查询的多次渲染(工具调用轮)视为同一轮,不推进窗口。
100
+ const rotationTurns = Math.max(0, Math.floor(config.injectRotationTurns ?? 0));
101
+ const rotationHistory = new Map(); // sessionId -> [{ query, ids: Set }]
102
+ const ROTATION_HISTORY_MAX = 32; // 覆盖 max(20) 配置档仍有余量
103
+
104
+ function recentInjectedIds(sessionId, query) {
105
+ if (rotationTurns <= 0) return null;
106
+ const deque = rotationHistory.get(sessionId);
107
+ if (!deque || deque.length === 0) return null;
108
+ // 只看「之前的轮次」:排除与当前查询相同的条目——同一查询的重复渲染
109
+ // (工具调用轮)不该拿本轮自己的集合来转自己。
110
+ const prior = query ? deque.filter((e) => e.query !== query) : deque;
111
+ if (prior.length === 0) return null;
112
+ const recent = new Set();
113
+ for (const entry of prior.slice(-rotationTurns)) {
114
+ for (const id of entry.ids) recent.add(id);
115
+ }
116
+ return recent.size > 0 ? recent : null;
117
+ }
118
+
119
+ function recordInjection(sessionId, query, candidates) {
120
+ if (rotationTurns <= 0 || !query) return;
121
+ const deque = rotationHistory.get(sessionId) ?? [];
122
+ const ids = new Set(candidates.map((c) => c.id));
123
+ const last = deque[deque.length - 1];
124
+ if (last && last.query === query) {
125
+ last.ids = ids; // 同一查询的重复渲染:覆盖本轮内容,不推进窗口
126
+ } else {
127
+ deque.push({ query, ids });
128
+ if (deque.length > ROTATION_HISTORY_MAX) deque.shift();
129
+ rotationHistory.set(sessionId, deque);
130
+ }
131
+ }
97
132
  // v0.8.0 A3(issue #17):注入路径的会话 scope 解析——strictScope 硬过滤
98
133
  // 需要。渲染 ctx 与工具 exec 同形(agent.session),解析器直接复用。
99
134
  // logger 透传:registry 反查失败时 warnOnce 才有出口。
@@ -226,7 +261,13 @@ export function createInjector(ctx, service, settings, config) {
226
261
  // v0.8.0 A3:scope 随请求解析(strict 开启时 injectCandidates 内硬过滤;
227
262
  // flag 关闭时解析器返回 null,注入行为不变)。
228
263
  const scope = resolveSessionScope(ctx);
229
- const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold, scope });
264
+ // Issue #205:跨轮轮换——最近 N 个查询轮次注入过的 id 本轮不再优先。
265
+ const sessionId = ctx?.agent?.session?.id ?? "_";
266
+ const rotate = recentInjectedIds(sessionId, query);
267
+ // rotateWindow 随集合一起传入:候选池按 maxItems×(N+1) 扩容——窗口要的
268
+ // 牌比既有 maxItems×2 池多时,轮换才有新鲜牌可换(#205 补测)。
269
+ const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold, scope, rotate, rotateWindow: rotationTurns });
270
+ recordInjection(sessionId, query, candidates);
230
271
  // Hot memory (v0.5.0 1.3) leads the single memory block: the agent
231
272
  // sees the short-term rounds first, then the cross-session recall —
232
273
  // the documented injection order 1→2. Folding it here (instead of a
@@ -246,6 +287,7 @@ export function createInjector(ctx, service, settings, config) {
246
287
 
247
288
  return () => {
248
289
  queryVectorCache.clear();
290
+ rotationHistory.clear();
249
291
  for (const dispose of disposers) {
250
292
  if (typeof dispose === "function") dispose();
251
293
  }
package/src/lang.js CHANGED
@@ -23,8 +23,8 @@ const PROMPTS = {
23
23
  "en": "You are the memory overview assistant. From the consolidated memories below, produce a 150-200 word memory-library overview covering: user preferences, active projects, key decisions. This will be injected as conversation context. Output only the summary text, nothing else."
24
24
  },
25
25
  "consolidation": {
26
- "zh": "你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。\n请执行记忆巩固(consolidation),输出一个决策 JSON 数组。\n\n【决策格式(必须严格遵守)】\n每个决策必须是对象,字段固定:\n- \"action\":必填。取值只能是 \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" 之一(字段名必须是 action,严禁写成 type)\n- \"ids\":必填,数组,本决策涉及的记忆 id 列表\n- \"reason\":可选,字符串,决策理由\n- \"importance\":可选,整数 1-5\n- merge 额外字段:\"keepSource\"(单个 id 字符串,必须是 ids 之一)+ 合并后的 \"title\"、\"content\"\n- conflict 额外字段:\"winner\" 与 \"loser\",都是【单个 id 字符串,不是数组】\n- update 额外字段:修正后的 \"title\" 和/或 \"content\";\"ids\" 只能包含一个 id\n\n【决策 JSON 示例】\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"合并标题\", \"content\": \"合并后的摘要内容\", \"importance\": 4, \"reason\": \"主题相近\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"内容矛盾,保留更新的信息\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"修正后的内容\", \"reason\": \"信息过时\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"重复或过时\" }\n]\n\n【任务】\n1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)\n2. 识别重复/过时信息 → archive\n3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)\n4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)\n - update 的 ids 只能包含一个 id\n - 必须提供修正后的 title 和/或 content\n - 仅当内容确实需要修正时才使用,不要滥用\n - 每次整理最多输出 2 个 update\n - 24 小时内新建的记忆不可 update\n5. 无问题的条目无需输出(未提及的条目将自动保留 keep)\n\n【硬性规则】\n- 字段名必须精确为 \"action\",严禁写成 \"type\";字段名统一用双引号\n- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】\n- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)\n- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep\n- merge 的 keepSource 必须是 ids 之一\n- 仅合并同类型条目(type 相同)\n- 不要编造 ids;只使用提供的 id\n- 重要性 1-5,合并后取最高\n- 只输出 JSON 数组,不要其他文字",
27
- "en": "You are the memory consolidation assistant. Below are all memory entries (id, type, title, content, importance, updated time).\nPerform memory consolidation and output a JSON array of decisions.\n\n[Decision format (must be followed strictly)]\nEach decision must be an object with fixed fields:\n- \"action\": required. Value must be one of \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" (the field name must be action, never type)\n- \"ids\": required, array, the list of memory ids involved in this decision\n- \"reason\": optional, string, rationale for the decision\n- \"importance\": optional, integer 1-5\n- merge extra fields: \"keepSource\" (a single id string, must be one of ids) plus the merged \"title\" and \"content\"\n- conflict extra fields: \"winner\" and \"loser\", each a [single id string, never an array]\n- update extra fields: the corrected \"title\" and/or \"content\"; \"ids\" may contain only one id\n\n[Decision JSON examples]\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"Merged title\", \"content\": \"Merged summary content\", \"importance\": 4, \"reason\": \"similar topics\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"content contradicts; keep the newer information\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"corrected content\", \"reason\": \"information outdated\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"duplicate or outdated\" }\n]\n\n[Task]\n1. Identify entries with similar topics -> merge (merge into a more refined summary; keep the id with the most complete information as keepSource)\n2. Identify duplicate/outdated information -> archive\n3. Identify entries with contradictory content -> conflict (judge winner/loser by recency, source completeness, and information specificity)\n4. Find outdated, wrong, or missing information in a single memory -> update (correct the content directly)\n - update's ids may contain only one id\n - must provide the corrected title and/or content\n - use only when the content truly needs correction; do not abuse\n - output at most 2 updates per consolidation run\n - memories created within the last 24 hours must not be updated\n5. Entries without issues need no output (unmentioned entries are automatically kept)\n\n[Hard rules]\n- Field names must be exactly \"action\", never \"type\"; use double quotes for all field names\n- conflict winner/loser and merge keepSource are [single id strings, never arrays]\n- Each memory may be claimed at most once: the same id must not appear in multiple decisions (an id must not be claimed by both merge and conflict/archive, etc.)\n- Memories not mentioned in any decision are automatically kept (keep); no need to output keep for every memory\n- merge keepSource must be one of ids\n- Only merge entries of the same type (identical type)\n- Never invent ids; use only the provided ids\n- Importance is 1-5; after a merge take the highest\n- Write every title, content, and reason in English\n- Output only the JSON array, nothing else"
26
+ "zh": "你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。\n请执行记忆巩固(consolidation),输出一个决策 JSON 数组。\n\n【决策格式(必须严格遵守)】\n每个决策必须是对象,字段固定:\n- \"action\":必填。取值只能是 \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" 之一(字段名必须是 action,严禁写成 type)\n- \"ids\":必填,数组,本决策涉及的记忆 id 列表\n- \"reason\":可选,字符串,决策理由\n- \"importance\":可选,整数 1-5\n- merge 额外字段:\"keepSource\"(单个 id 字符串,必须是 ids 之一)+ 合并后的 \"title\"、\"content\"\n- conflict 额外字段:\"winner\" 与 \"loser\",都是【单个 id 字符串,不是数组】\n- update 额外字段:修正后的 \"title\" 和/或 \"content\";\"ids\" 只能包含一个 id\n\n【决策 JSON 示例】\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"合并标题\", \"content\": \"合并后的摘要内容\", \"importance\": 4, \"reason\": \"主题相近\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"内容矛盾,保留更新的信息\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"修正后的内容\", \"reason\": \"信息过时\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"重复或过时\" }\n]\n\n【任务】\n1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)\n2. 识别重复/过时信息 → archive\n3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)\n4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)\n - update 的 ids 只能包含一个 id\n - 必须提供修正后的 title 和/或 content\n - 仅当内容确实需要修正时才使用,不要滥用\n - 每次整理最多输出 2 个 update\n - 24 小时内新建的记忆不可 update\n5. 无问题的条目无需输出(未提及的条目将自动保留 keep)\n\n【硬性规则】\n- 字段名必须精确为 \"action\",严禁写成 \"type\";字段名统一用双引号\n- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】\n- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)\n- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep\n- merge 的 keepSource 必须是 ids 之一\n- 仅合并同类型条目(type 相同)\n- 不要编造 ids;只使用提供的 id\n- 重要性 1-5,合并后取最高\n- archive 仅用于确属「重复或已过时」的条目;rejected_solution / constraint / pitfall / preference / pattern 属长保留类型,除非确属重复或过时,否则不要归档\n- 每次整理最多输出 8 个 archive\n- 只输出 JSON 数组,不要其他文字",
27
+ "en": "You are the memory consolidation assistant. Below are all memory entries (id, type, title, content, importance, updated time).\nPerform memory consolidation and output a JSON array of decisions.\n\n[Decision format (must be followed strictly)]\nEach decision must be an object with fixed fields:\n- \"action\": required. Value must be one of \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" (the field name must be action, never type)\n- \"ids\": required, array, the list of memory ids involved in this decision\n- \"reason\": optional, string, rationale for the decision\n- \"importance\": optional, integer 1-5\n- merge extra fields: \"keepSource\" (a single id string, must be one of ids) plus the merged \"title\" and \"content\"\n- conflict extra fields: \"winner\" and \"loser\", each a [single id string, never an array]\n- update extra fields: the corrected \"title\" and/or \"content\"; \"ids\" may contain only one id\n\n[Decision JSON examples]\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"Merged title\", \"content\": \"Merged summary content\", \"importance\": 4, \"reason\": \"similar topics\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"content contradicts; keep the newer information\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"corrected content\", \"reason\": \"information outdated\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"duplicate or outdated\" }\n]\n\n[Task]\n1. Identify entries with similar topics -> merge (merge into a more refined summary; keep the id with the most complete information as keepSource)\n2. Identify duplicate/outdated information -> archive\n3. Identify entries with contradictory content -> conflict (judge winner/loser by recency, source completeness, and information specificity)\n4. Find outdated, wrong, or missing information in a single memory -> update (correct the content directly)\n - update's ids may contain only one id\n - must provide the corrected title and/or content\n - use only when the content truly needs correction; do not abuse\n - output at most 2 updates per consolidation run\n - memories created within the last 24 hours must not be updated\n5. Entries without issues need no output (unmentioned entries are automatically kept)\n\n[Hard rules]\n- Field names must be exactly \"action\", never \"type\"; use double quotes for all field names\n- conflict winner/loser and merge keepSource are [single id strings, never arrays]\n- Each memory may be claimed at most once: the same id must not appear in multiple decisions (an id must not be claimed by both merge and conflict/archive, etc.)\n- Memories not mentioned in any decision are automatically kept (keep); no need to output keep for every memory\n- merge keepSource must be one of ids\n- Only merge entries of the same type (identical type)\n- Never invent ids; use only the provided ids\n- Importance is 1-5; after a merge take the highest\n- Write every title, content, and reason in English\n- archive is only for entries that are genuinely \"duplicate or outdated\"; rejected_solution / constraint / pitfall / preference / pattern are long-retention types — do not archive them unless they are truly duplicate or outdated\n- output at most 8 archives per consolidation run\n- Output only the JSON array, nothing else"
28
28
  },
29
29
  "conflict": {
30
30
  "zh": "你是记忆库冲突仲裁助手。下面是检测到的高相似度记忆对,可能内容矛盾或重复。\n对每一对输出一个 decision 对象:\n- 两条确实矛盾/重复 → { \"action\": \"conflict\", \"winner\": <保留的id>, \"loser\": <归档的id>, \"reason\": \"理由\" }\n- 两条只是主题相近、并无矛盾 → { \"action\": \"keep\", \"ids\": [<两个id>] }\n规则:\n- winner 应为信息更完整、更新或更可信的一条\n- 只使用提供的 id,不要编造\n- 每对必须输出一个 decision\n- 只输出 JSON 数组,不要其他文字",
@@ -4,6 +4,7 @@
4
4
  // can pick a backend by provider name and degrade gracefully on failure.
5
5
  // Methods throw on error — the caller decides the fallback chain.
6
6
  import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
7
+ import { installResilientFetch } from "./runtime/model-fetch.js";
7
8
  import { loadTransformers } from "./runtime/loader.js";
8
9
 
9
10
  const DEFAULT_TIMEOUT_MS = 15000;
@@ -30,7 +31,7 @@ function modelHash(model) {
30
31
  * transformers 的选项里。
31
32
  */
32
33
  async function defaultPipelineLoader(task, model, options) {
33
- const { runtimeDir, ...pipelineOptions } = options ?? {};
34
+ const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
34
35
  const { module } = await loadTransformers({
35
36
  runtimeDir: runtimeDir || defaultRuntimeDir()
36
37
  });
@@ -41,6 +42,17 @@ async function defaultPipelineLoader(task, model, options) {
41
42
  // model is fully cached locally. Mirroring the cache_dir onto env.cacheDir
42
43
  // makes that pre-check resolve locally too — fully offline loading.
43
44
  if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
45
+ // issue #188 补充:embedModelMirror 此前是死配置(定义了但全仓无人读取)。
46
+ // transformers.js 不认 HF_ENDPOINT(那是 Python huggingface_hub 的变量),
47
+ // 模型下载源由 env.remoteHost 决定——把镜像接到真正的开关上,否则
48
+ // huggingface.co 不可达的网络连模型都下不下来。
49
+ if (remoteHost) env.remoteHost = remoteHost;
50
+ // issue #194:模型文件(几百 MB~1GB)下载中断后从断点续传。开关关 = env.fetch 一字不动。
51
+ installResilientFetch(env, {
52
+ enabled: resilientModelDownload === true,
53
+ cacheDir: pipelineOptions.cache_dir,
54
+ logger
55
+ });
44
56
  return pipeline(task, model, pipelineOptions);
45
57
  }
46
58
 
@@ -71,7 +83,12 @@ export class LocalEmbedder {
71
83
  this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
72
84
  // 自管运行时根目录(issue #131):空表示用默认位置,具体解析交给 loader。
73
85
  this.runtimeDir = String(opts.runtimeDir ?? "").trim();
86
+ // issue #194:模型文件下载断点续传。只在显式传 true 时启用——直连构造的旧调用方
87
+ // (含测试)保持裸 fetch 的现状,配置默认值由 index.js 传进来。
88
+ this.resilientModelDownload = opts.resilientModelDownload === true;
74
89
  this.useDtype = opts.useDtype || "q8";
90
+ // 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
91
+ this.remoteHost = String(opts.remoteHost ?? "").trim();
75
92
  this.logger = opts.logger ?? null;
76
93
  // Test hook: replace the pipeline factory without touching modules.
77
94
  this.engineFactory = opts.engineFactory || defaultPipelineLoader;
@@ -91,6 +108,9 @@ export class LocalEmbedder {
91
108
  if (this.cacheDir) options.cache_dir = this.cacheDir;
92
109
  // 传给注入的 engineFactory,由 defaultPipelineLoader 取走后交给三层解析。
93
110
  if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
111
+ if (this.remoteHost) options.remoteHost = this.remoteHost;
112
+ options.resilientModelDownload = this.resilientModelDownload;
113
+ options.logger = this.logger;
94
114
  this.extractor = await this.engineFactory("feature-extraction", this.model, options);
95
115
  this.ready = true; // service reads this to flush queued re-embeds
96
116
  this.logger?.info?.(
package/src/reranker.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import { defaultModelCacheDir, defaultRuntimeDir } from "./runtime/layout.js";
2
+ import { installResilientFetch } from "./runtime/model-fetch.js";
2
3
  import { loadTransformers } from "./runtime/loader.js";
3
4
 
4
5
  // Cross-encoder re-ranker for dsh-mneme recall candidates. Uses
@@ -27,7 +28,7 @@ function modelHash(model) {
27
28
  * runtimeDir 取走后不混进 transformers 的选项。
28
29
  */
29
30
  async function defaultPipelineLoader(task, model, options) {
30
- const { runtimeDir, ...pipelineOptions } = options ?? {};
31
+ const { runtimeDir, remoteHost, resilientModelDownload, logger, ...pipelineOptions } = options ?? {};
31
32
  const { module } = await loadTransformers({
32
33
  runtimeDir: runtimeDir || defaultRuntimeDir()
33
34
  });
@@ -36,6 +37,16 @@ async function defaultPipelineLoader(task, model, options) {
36
37
  // 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
37
38
  // embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
38
39
  if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
40
+ // issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
41
+ // HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
42
+ // 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
43
+ if (remoteHost) env.remoteHost = remoteHost;
44
+ // issue #194:与 embedder 同款断点续传。开关关 = env.fetch 一字不动。
45
+ installResilientFetch(env, {
46
+ enabled: resilientModelDownload === true,
47
+ cacheDir: pipelineOptions.cache_dir,
48
+ logger
49
+ });
39
50
  return pipeline(task, model, pipelineOptions);
40
51
  }
41
52
 
@@ -85,6 +96,13 @@ export class LocalReranker {
85
96
  this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
86
97
  // 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
87
98
  this.runtimeDir = String(opts.runtimeDir ?? "").trim();
99
+ // 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
100
+ // 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
101
+ this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
102
+ // 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
103
+ this.remoteHost = String(opts.remoteHost ?? "").trim();
104
+ // issue #194:模型文件下载断点续传。只在显式传 true 时启用,与 embedder 同规。
105
+ this.resilientModelDownload = opts.resilientModelDownload === true;
88
106
  this.logger = opts.logger ?? null;
89
107
  this.engineFactory = opts.engineFactory || defaultPipelineLoader;
90
108
  // Injectable seam: async (query, passage) => number. When set, init()
@@ -94,6 +112,7 @@ export class LocalReranker {
94
112
  this._batchScorer = null;
95
113
  this._queryVec = null;
96
114
  this._queryKey = null;
115
+ this._constantScoreWarned = false;
97
116
  }
98
117
 
99
118
  /** Load the model; throws when no strategy can be bound. */
@@ -119,9 +138,10 @@ export class LocalReranker {
119
138
  }
120
139
 
121
140
  _engineOptions() {
122
- const options = { device: this.device };
141
+ const options = { device: this.device, dtype: this.useDtype, resilientModelDownload: this.resilientModelDownload, logger: this.logger };
123
142
  if (this.cacheDir) options.cache_dir = this.cacheDir;
124
143
  if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
144
+ if (this.remoteHost) options.remoteHost = this.remoteHost;
125
145
  return options;
126
146
  }
127
147
 
@@ -136,8 +156,14 @@ export class LocalReranker {
136
156
  return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
137
157
  };
138
158
  } else if (strategy === "tc") {
139
- // Classification head: tokenize (query, passage) pairs, score with
140
- // sigmoid(l1 - l0) so relevance lands in [0, 1].
159
+ // Classification head: score so relevance lands in [0, 1]. Branch on the
160
+ // output width (issue #188): a dual-logit head (num_labels=2 exports,
161
+ // e.g. bge-reranker-v2-m3) gives the positive class as the softmax
162
+ // probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
163
+ // Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
164
+ // the positive class is simply sigmoid(logit) — reading l1 as l0 there
165
+ // collapses every score to sigmoid(0) = 0.5 and silently disables
166
+ // reranking (no error, status still "ready").
141
167
  this._batchScorer = async (query, passages) => {
142
168
  const { tokenizer, model } = this.pipeline;
143
169
  const inputs = tokenizer(passages.map(() => query), {
@@ -151,10 +177,10 @@ export class LocalReranker {
151
177
  const rows = [];
152
178
  for (let i = 0; i < dims[0]; i++) {
153
179
  const base = i * cols;
154
- const l0 = logits.data[base];
155
- const l1 = cols > 1 ? logits.data[base + 1] : l0;
156
- // sigmoid(l1 - l0) == softmax probability of the positive class.
157
- rows.push(clamp01(1 / (1 + Math.exp(l0 - l1))));
180
+ const score = cols > 1
181
+ ? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
182
+ : 1 / (1 + Math.exp(-logits.data[base]));
183
+ rows.push(clamp01(score));
158
184
  }
159
185
  return rows;
160
186
  };
@@ -208,6 +234,19 @@ export class LocalReranker {
208
234
  results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
209
235
  }
210
236
  }
237
+ // Silent-failure tripwire (issue #188): a reranker whose every candidate
238
+ // scores identically is not ranking anything — e.g. a single-logit head
239
+ // read through the dual-logit formula. Warn once so the degradation is
240
+ // observable instead of a constant 0.5 column hiding behind a "ready"
241
+ // status forever.
242
+ if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
243
+ if (!this._constantScoreWarned) {
244
+ this._constantScoreWarned = true;
245
+ this.logger?.warn?.(
246
+ `[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
247
+ );
248
+ }
249
+ }
211
250
  return results
212
251
  .filter((r) => r.score >= this.scoreThreshold)
213
252
  .sort((a, b) => b.score - a.score);