@modusensus/dsh-mneme 0.8.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/lib/api-standalone.js +12 -0
- package/lib/api.js +12 -2
- package/lib/config.js +8 -0
- package/lib/dream/decisions.js +55 -8
- package/lib/dream/sleep.js +36 -5
- package/lib/dream.js +18 -7
- package/lib/entities/extractor.js +2 -2
- package/lib/index.js +6 -0
- package/lib/lang.js +2 -2
- package/lib/local-embedder.js +9 -1
- package/lib/reranker.js +38 -8
- package/lib/service.js +16 -0
- package/lib/store.js +85 -25
- package/lib/summarize.js +2 -2
- package/lib/tools.js +6 -19
- package/lib/vector-index.js +2 -8
- package/package.json +2 -1
- package/scripts/sync-test-badge.mjs +61 -0
- package/src/api-standalone.js +12 -0
- package/src/api.js +12 -2
- package/src/config.js +8 -0
- package/src/dream/decisions.js +55 -8
- package/src/dream/sleep.js +36 -5
- package/src/dream.js +18 -7
- package/src/entities/extractor.js +2 -2
- package/src/index.js +6 -0
- package/src/lang.js +2 -2
- package/src/local-embedder.js +9 -1
- package/src/reranker.js +38 -8
- package/src/service.js +16 -0
- package/src/store.js +85 -25
- package/src/summarize.js +2 -2
- package/src/tools.js +6 -19
- package/src/vector-index.js +2 -8
- package/test/api.test.js +5 -0
- package/test/dream-archive-guardrail.test.js +116 -0
- package/test/dream-coverage-degrade.test.js +86 -0
- package/test/dream.test.js +15 -11
- package/test/inject-first-round-bm25.test.js +89 -0
- package/test/llm-message-source.test.js +177 -0
- package/test/reranker-single-logit.test.js +73 -0
- package/test/skip-invalid-restore.test.js +7 -5
- package/test/sleep-scheduler-rearm.test.js +89 -0
- package/test/sleep.test.js +2 -2
- package/test/standalone-api.test.js +34 -1
- package/test/store-vector-cache.test.js +98 -0
- package/test/tools-memory-get-schema.test.js +65 -0
package/README.md
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
[](https://www.npmjs.com/package/@modusensus/dsh-mneme)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
[](https://github.com/awesome-dsh-plugin/awesome-dsh-plugin)
|
|
8
|
-
[](https://github.com/modusensus/dsh-mneme)
|
|
9
9
|
[](https://github.com/modusensus/dsh-mneme/actions)
|
|
10
|
-
[](https://nodejs.org)
|
|
11
11
|
[](https://www.npmjs.com/package/@modusensus/dsh-mneme)
|
|
12
12
|
[](https://codecov.io/gh/modusensus/dsh-mneme)
|
|
13
13
|
|
|
@@ -229,6 +229,7 @@ v0.3.0 起新增**记忆基因**层:从记忆里抽取**命名实体**、**带
|
|
|
229
229
|
| 版本 | 亮点 |
|
|
230
230
|
|------|------|
|
|
231
231
|
| **v0.8.1** | 归属显式声明与人工纠偏 + 插件版本自检:记忆归属三段式可控——per-dim 来源列 + save/update scope 参数 + 面板可编辑(#170 第 1 步,#171);sleep 跨 scope 相似对停车进冲突队列人工裁决(第 2 步,#172);`strictScope` 硬过滤只认显式声明、自动标注降为纯软加权(第 3 步,#173,4.3 已确认);`GET /api/dsh-mneme/version-check` 只读路由 + 设置页偏差横幅(仅 outdated 渲染,registry 精确 host 白名单 + TTL 缓存 + 全失败静默,pnpm 钉子警示 + 市场收录延迟说明,#174 后续);1026 测试全绿 |
|
|
232
|
+
| **v0.8.2** | 生态与性能批次(社区贡献集中合入):LLM 消息补 source 契约——严格 provider 下 dream/summarize/sleep/实体抽取不再序列化抛错(#189/#190);memory_get 输出 schema 与共享 DTO 同源(#184/#186);sleep 计时器撞 CD 重排 + 构造挂表(#187/#192);单 logit 重排器恒 0.5 修复 + 镜像/量化接线(#188/#193);连通性探测改发 user 消息(#197);首轮注入 BM25 同步兜底——老偏好不再占满槽位(#198/#199);autoDream 宽容路径闭环——coverage 不足降级 degraded + archive 类型护栏/批量上限 `dreamMaxArchivePerRun`(#104/#200/#201);检索路径性能第一批——updated_at 索引 + all() 剪列 + 向量解析缓存(#202/#203,all() 231→51ms、searchVector ~140→16ms);standalone API 补 profile/rules 路由(#180/#185);1061 测试全绿 |
|
|
232
233
|
| **v0.8.0** | 作用域隔离(issue #17 批次 A)+ 冲突集中处理 + 斜杠命令提交 Agent + 注入转义回归修复:记忆按 agent 与工作区双维标注(agent_scope / workspace_scope / sensitivity / occurred_at 四列可空),检索按当前会话作用域加权(命中 ×1.25、他 scope ×0.5 保留可见),opt-in `strictScope` 硬过滤贯通检索/注入/列表/单取(fail-closed);`scopeEnabled` / `strictScope` 默认关、面板「作用域隔离」组可启停;去重键扩展含作用域三元——跨作用域同标题不再物理合并;`occurred_at` 事件发生时间 + `occurred_from/to` 过滤贯通搜索与列表;冲突冻结(conflictFreezeEnabled)新增人工出口——状态页冲突队列并排对比 + 保留 A/B/仅标记(#166);斜杠命令经 agent.followup 真正提交模型(#152);恢复 v0.7.4 的注入花括号转义 + hot memory 跳过 reasoning——`{{.Server.Version}}` 类文本不再卡死会话(#165);988 测试全绿 |
|
|
233
234
|
| **v0.7.32** | 冲突动作集扩展 + 运行时自管化 + 记忆/提示语言 + 候选集 hybrid 档 + 一批修复:`sleepActionSet` 新增 `full` 档(supersede/differentiate/update/merge/conflict/keep 六分支,默认 `conflict` 零行为变化)——supersede 赢家正文干净、输家归档附「已被取代」注记,differentiate 双留+差异注记;sleep 校验接入 `dreamSkipInvalid` 宽容策略(默认 true,一票否决变宽容,严格可关);运行时三档取件(收编/本地 .tgz/registry)+ transformers 转可选 peer(#131);`memory.language`(zh/en)提示/注入/镜像语言可选(#124);summarize 节流 + 产出上限 + 同会话去重(#127);inject 正文读取修复(#129)、#135 五连修(前缀解析/effort 最低档/importance 豁免/ready 语义/覆盖度降级告警)、boot 回填指纹短路修复(#128)、degraded 轮跳过明细落库(#104);916 测试全绿、总覆盖 92.7%(runtime 96%、embedding 100%) |
|
|
234
235
|
| **v0.7.30** | 状态页向量卡修复(issue #118):改读开放的 `/semantic`——不再吃 `/vector-config` 的 401「加载失败」,ollama/local 模式不再恒显「未启用」;新增「初始化中(embedder 不可达,正在重试)」与「已索引 N / M 条」回填进度显示;legacy OpenAI 兼容 embedder「Object · 0D」显示修复(embedder 显式 `name: "OpenAI"` + 维度回退 `index.dimension`);「已索引 N / M」分子分母同口径(`embeddedCount` 剔除归档/遗忘,与 `count()` 默认过滤对齐);embedder init 失败有界重试(共 5 次,不再一次性永久降级,unload 清理定时器)+ `OllamaEmbedder.ready` 生命周期位 + `/semantic` 新增 `ready` 字段;745 测试全绿 |
|
|
@@ -337,6 +338,7 @@ v0.3.0 起新增**记忆基因**层:从记忆里抽取**命名实体**、**带
|
|
|
337
338
|
| **v0.7.32** | ✅ 完成 | sleep 冲突动作集扩展 + 运行时自管化 + 候选集 hybrid 档 + 一批修复 | `sleepActionSet` 新增 `full` 档(supersede/differentiate/update/merge/conflict/keep 六分支,默认 `conflict` 零行为变化)+ sleep 校验接入 `dreamSkipInvalid` 宽容策略 + 被跳决策带 phase 落库 `dream_runs.skipped`;运行时三档取件 + transformers 转可选 peer(#131);`memory.language` 提示/注入/镜像语言可选(#124);候选集向量驱动 hybrid 档(#125,PR #147);summarize 节流 + 产出上限 + 同会话去重(#127);inject 正文读取修复(#129)+ #135 五连修 + boot 回填指纹短路修复(#128)+ degraded 轮跳过明细落库(#104);runtime 覆盖统计归一 + 补齐(总覆盖 92.7%);916 测试全绿 |
|
|
338
339
|
| **v0.8.0** | ✅ 完成 | 作用域隔离 + 冲突队列 | 按 agent 与工作区双维隔离记忆(标注 / 检索加权 / opt-in strictScope 四路硬过滤,issue #17 批次 A);冻结冲突新增人工确认队列(状态页并排对比,#166);斜杠命令真正提交 Agent(#152);注入转义回归恢复 + hot memory 跳过 reasoning(#162)。图谱增强(兴趣漂移可视化)、跨 workspace 记忆共享仍在规划 |
|
|
339
340
|
| **v0.8.1** | ✅ 完成 | 归属显式声明与人工纠偏 + 插件版本自检 | 归属三段式可控:per-dim 来源列 + save/update scope 参数 + 面板可编辑(#170 第 1 步,#171);sleep 跨 scope 相似对停车人工裁决接冲突队列(第 2 步,#172);`strictScope` 硬过滤只认显式声明、自动标注降纯软加权(第 3 步,#173)。插件自报新版本:/version-check 只读路由 + 设置页 outdated 横幅(registry 精确白名单 + TTL 缓存 + 全失败静默,pnpm 钉子警示,#174 后续,#176);1026 测试全绿 |
|
|
341
|
+
| **v0.8.2** | ✅ 完成 | 生态与性能批次:宽容路径闭环 + 检索提速 + 社区贡献集中合入 | autoDream 宽容路径三方向闭环:coverage 不足降级 degraded、archive 类型护栏 + `dreamMaxArchivePerRun` 批量上限、失败审计明细贯通(#104/#200/#201);检索路径三处固定成本:updated_at 索引 + all() 剪列 + 向量解析缓存,all() 231→51ms、searchVector ~140→16ms(#202/#203);LLM 消息补 source 契约(#189/#190);首轮注入 BM25 同步兜底(#198/#199);memory_get schema 同源(#184/#186);sleep CD 重排(#187/#192);单 logit 重排器修复(#188/#193);standalone profile/rules 路由(#180/#185);1061 测试全绿 |
|
|
340
342
|
|
|
341
343
|
> 新能力一律做成**可开关的功能**(配置启用/关闭),默认保守开启、不破坏现有行为。`failure_memories` 表与 autoDream 决策引擎已为后续反思性成长铺好路。
|
|
342
344
|
|
|
@@ -413,6 +415,7 @@ dsh web
|
|
|
413
415
|
| `dreamCandidateMode` | `window` | dream 候选集构造:`window`(只取最近 `dreamMaxSnapshotSize` 条)/ `hybrid`(在此基础上并入向量翻出的高相似组)。纯时间窗口下,实测 45 对「双方活跃且 sim≥0.85」里 0 对能同时进窗口——该合并的一对几乎永远碰不到面。**已知边界**:dream 侧动作集仍是五分支(keep / merge / archive / update / conflict),hybrid 翻出的互补型 / 演进型对在 dream 里没有 `differentiate` / `supersede` 出口;正确出口在 sleep 侧(`sleepActionSet: full`)|
|
|
414
416
|
| `dreamCandidateMax` | `0` | hybrid 的候选总量上限;`0` = 复用 `dreamMaxSnapshotSize`。候选总量与库总量解耦,输入成本不随库增长 |
|
|
415
417
|
| `dreamCandidateMinSim` | `0.85` | hybrid 判「高相似」的阈值(与 sleep 的 normal 档对齐,两个模块共用同一个「高相似」定义) |
|
|
418
|
+
| `dreamMaxArchivePerRun` | `8` | 单轮 archive 决策上限(超限整单拒绝,防一次性大扫除;正常清理可调高) |
|
|
416
419
|
| `apiToken` | 空 | 可选 API 鉴权 token;设置后写操作与密钥接口要求 `Authorization: Bearer <apiToken>` |
|
|
417
420
|
| `embedProvider` | `openai` | 语义后端:`openai`(默认,兼容 v0.1)/ `local`(ONNX 离线)/ `ollama` |
|
|
418
421
|
| `localEmbedModel` | `Xenova/bge-small-zh-v1.5` | 本地 ONNX embedding 模型 |
|
|
@@ -433,6 +436,7 @@ dsh web
|
|
|
433
436
|
| `rerankBatchSize` | `8` | Rerank 批大小 |
|
|
434
437
|
| `rerankMaxCandidates` | `30` | Rerank 最大候选数 |
|
|
435
438
|
| `rerankScoreThreshold` | `0.1` | Rerank 分数阈值(低于丢弃) |
|
|
439
|
+
| `rerankDtype` | `q8` | Rerank 模型量化档(`q8` ≈ fp32 体积的 1/4,`fp32` 可关) |
|
|
436
440
|
| `reflectionUpdateEnabled` | `true` | update 决策总开关 |
|
|
437
441
|
| `reflectionFailureTracking` | `true` | 失败追踪总开关 |
|
|
438
442
|
| `reflectionUpdateMaxPerRun` | `2` | 每次整理最多 update 数 |
|
|
@@ -470,6 +474,8 @@ dsh web
|
|
|
470
474
|
> 🔐 **API 安全**:DSH 无内置鉴权且默认仅监听 `127.0.0.1`。插件 API 默认开放(便于 Web 面板即装即用)。如需防护(如局域网暴露),在配置中设置 `apiToken`:写操作(画像/规则/命令)与密钥端点(`vector-config`、`vector-reindex`)需携带 `Authorization: Bearer <token>`(前端设置面板可填入同一 token),只读的 `list` / `search` / `semantic` 保持开放。`/api/dsh-mneme/vector-config` 返回的 `apiKey` 已掩码(`sk-***…`),存储仍保留明文供调用;前端回传空或掩码值表示"不改 key"。
|
|
471
475
|
|
|
472
476
|
|
|
477
|
+
> ⚠️ **已知边界:极简模式(minimal agent preset)下不注入**(宿主设计,非插件缺陷):minimal 预设的组合文件显式 `complete: true` + `includeRuntimeContext: false`——整条系统提示词被 persona 钉死、全部 runtime context 快照被压制,**记忆注入 / 用户画像 / hot memory 在极简模式会话中一律不送达模型**(宿主内置 context 同样消失)。判断与解法:会话头部显示「极简模式」即为该形态;需要记忆注入请改用标准模式(会话级切换,或 `~/.dsh/settings.yaml` 设 `agent-presets.default: standard`——注意 `--dump-config` 显示的是 bundle 默认值,会被用户层覆盖,不能反映实际生效值);极简模式下的过渡方案是把画像/规则写入 `AGENTS.md`(agent-instructions 走 section 路径,不受该压制影响)。
|
|
478
|
+
|
|
473
479
|
## 外部 API 与 CLI
|
|
474
480
|
|
|
475
481
|
除 DSH 内部端口外,插件还可以开启一个**独立的 HTTP 外部 API**(默认 `http://127.0.0.1:8790`,Bearer token 鉴权),供其他插件、CLI 脚本或桌面工具读写记忆,不依赖 DSH 内部端口。
|
|
@@ -486,6 +492,8 @@ dsh web
|
|
|
486
492
|
|------|------|------|
|
|
487
493
|
| `GET` | `/health` | 健康检查(免鉴权),返回 `{ok:true}` |
|
|
488
494
|
| `GET` | `/status` | 版本、记忆统计、实体数、运行时长 |
|
|
495
|
+
| `GET` | `/profile` | 用户画像,返回 `{profile:"..."}`;未设置时为空字符串 |
|
|
496
|
+
| `GET` | `/rules` | 行为规则列表,返回 `{rules:[...]}`;未设置时为空数组 |
|
|
489
497
|
| `GET` | `/memories?limit&offset&type&minImportance&source&order=chrono` | 分页列出记忆 |
|
|
490
498
|
| `GET` | `/memories/:id` | 单条记忆 |
|
|
491
499
|
| `POST` | `/memories` | 新增记忆 `{type,title,content,importance?,tags?,source?}` |
|
|
@@ -592,7 +600,7 @@ scripts/ # e2e-dsh.js 端到端演示 · stress-dsh.js 三轴线压
|
|
|
592
600
|
```bash
|
|
593
601
|
cd dsh-mneme
|
|
594
602
|
npm install # 安装 peer 依赖(以 devDependencies 形式,用于本地测试)
|
|
595
|
-
npm test # 运行
|
|
603
|
+
npm test # 运行 1026 个测试
|
|
596
604
|
npm run stress # 三轴线压测:长会话检索 / 冲突仲裁 / 多 Agent 并发(离线 mock LLM)
|
|
597
605
|
npm run sync # 把 src/ 同步到 lib/(发布时由 prepack 钩子自动执行)
|
|
598
606
|
```
|
package/lib/api-standalone.js
CHANGED
|
@@ -152,6 +152,18 @@ export function createStandaloneApi({ service, store, config = {}, logger, setti
|
|
|
152
152
|
return;
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
// --- GET /profile: user self-description -------------------------------
|
|
156
|
+
if (req.method === "GET" && pathname === "/profile") {
|
|
157
|
+
sendJson(res, 200, { profile: settings?.getProfile?.() ?? "" });
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// --- GET /rules: agent behavior rules -----------------------------------
|
|
162
|
+
if (req.method === "GET" && pathname === "/rules") {
|
|
163
|
+
sendJson(res, 200, { rules: settings?.getRules?.() ?? [] });
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
|
|
155
167
|
// --- GET /memories: paged + filtered list (same semantics as the
|
|
156
168
|
// internal /api/dsh-mneme/list) ------------------------------------
|
|
157
169
|
if (req.method === "GET" && pathname === "/memories") {
|
package/lib/api.js
CHANGED
|
@@ -330,7 +330,17 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
330
330
|
// 按钮无放大风险。
|
|
331
331
|
maxTokens: 1024,
|
|
332
332
|
...(reasoningEffort && reasoningEffort !== "none" ? { reasoningEffort } : {}),
|
|
333
|
-
|
|
333
|
+
// role 用 user 而非 system(#189 后续):0.1.6-alpha.1 起适配器把 system
|
|
334
|
+
// 抽成顶层参数,system-only 对话会被官方 API 以
|
|
335
|
+
// 「messages: at least one message is required」整单拒绝——探测按钮 502。
|
|
336
|
+
// user 消息在任何适配器下都是一条合法对话,探测从此不怕 host 演进。
|
|
337
|
+
messages: [
|
|
338
|
+
{
|
|
339
|
+
role: "user",
|
|
340
|
+
content: [{ type: "text", text: "Reply with exactly one word: ok" }],
|
|
341
|
+
source: { kind: "plugin", plugin: "dsh-mneme" }
|
|
342
|
+
}
|
|
343
|
+
]
|
|
334
344
|
})) {
|
|
335
345
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") reply += chunk.text;
|
|
336
346
|
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
@@ -1298,7 +1308,7 @@ export function createApi(ctx, service, settings, commands, embedder, semantic =
|
|
|
1298
1308
|
});
|
|
1299
1309
|
|
|
1300
1310
|
return {
|
|
1301
|
-
routes:
|
|
1311
|
+
routes: 30,
|
|
1302
1312
|
dispose: () => {
|
|
1303
1313
|
for (const dispose of disposers) dispose();
|
|
1304
1314
|
}
|
package/lib/config.js
CHANGED
|
@@ -111,6 +111,10 @@ export const Config = z.object({
|
|
|
111
111
|
// 记忆做 consolidation。大记忆量下全量快照会把 LLM 输入撑爆(636 记忆 →
|
|
112
112
|
// 677 "missing" errors、applied=0),窗口外的旧记忆不进 snapshot。
|
|
113
113
|
dreamMaxSnapshotSize: z.natural().min(1).max(1000).default(200),
|
|
114
|
+
// Issue #104 方向 2:单轮 archive 决策上限。一次性大扫除(12 条互不相关主题
|
|
115
|
+
// 被批量归档,其中 8 条疑似误伤长保留类型)是失控信号,与 update 上限同类的
|
|
116
|
+
// 全局闸门:skipInvalid 也不豁免,超限整单拒绝。正常清理可调高。
|
|
117
|
+
dreamMaxArchivePerRun: z.natural().min(1).max(200).default(8),
|
|
114
118
|
// Issue #125:候选集构造方式。"window"(默认,等同现状)只取最近
|
|
115
119
|
// dreamMaxSnapshotSize 条;"hybrid" 在此基础上并入**向量翻出的高相似组**——
|
|
116
120
|
// 本机实测 45 对「双方活跃且 sim≥0.85」里 0 对能同时进窗口(窗口覆盖率 11.7%),
|
|
@@ -257,6 +261,10 @@ export const Config = z.object({
|
|
|
257
261
|
rerankBatchSize: z.natural().min(1).max(64).default(8),
|
|
258
262
|
rerankMaxCandidates: z.natural().min(5).max(100).default(30),
|
|
259
263
|
rerankScoreThreshold: z.number().min(0).max(1).default(0.1),
|
|
264
|
+
// #188:量化档,与嵌入层的 useDtype 同语义(q8 = model_quantized.onnx,约为
|
|
265
|
+
// fp32 体积的 1/4)。此前重排层不传 dtype,即使缓存里已有量化文件也会去下载
|
|
266
|
+
// 1GB 级的 model.onnx。非法值由 transformers 在 init 时抛出 → 重排降级并告警。
|
|
267
|
+
rerankDtype: z.string().default("q8"),
|
|
260
268
|
|
|
261
269
|
// --- reflection: update decision + failure tracking (v0.2.1) ------------
|
|
262
270
|
reflectionUpdateEnabled: z.boolean().default(true),
|
package/lib/dream/decisions.js
CHANGED
|
@@ -11,6 +11,15 @@ export const ACTIONS = new Set(["keep", "merge", "archive", "conflict", "update"
|
|
|
11
11
|
// = preferred. observation (measured) > inferred (derived) > subjective (guess).
|
|
12
12
|
const EPISTEMIC_PRIORITY = { observation: 3, inferred: 2, subjective: 1 };
|
|
13
13
|
|
|
14
|
+
// Issue #104 方向 2:archive 类型护栏的长保留类型集合——这些类型的存在意义
|
|
15
|
+
// 就是长期保存(踩坑记录、约束、用户偏好、挖掘出的模式),LLM 以「价值判断」
|
|
16
|
+
// 类理由批量归档它们时误伤损失最大。conflict 裁决的败者归档不经过 archive
|
|
17
|
+
// action(走 applyDecisions 的 conflict 分支),不受此护栏影响。
|
|
18
|
+
const ARCHIVE_GUARDED_TYPES = new Set(["preference", "pattern", "rejected_solution", "constraint", "pitfall"]);
|
|
19
|
+
// 「重复 / 过时」类理由的关键词(中英双语)。命中才允许归档长保留类型;
|
|
20
|
+
// 词表宁可放宽也不误拦——护栏只跳过单条并落明细,不整单拒绝。
|
|
21
|
+
const ARCHIVE_RATIONALE_RE = /重复|过时|陈旧|过期|已被取代|被取代|冗余|不再使用|旧版|废弃|stale|outdated|obsolete|dup(?:licate|licat)?|superseded|redundant|deprecated|expired|replaced/i;
|
|
22
|
+
|
|
14
23
|
/**
|
|
15
24
|
* Validate a dream decision list against a snapshot of eligible memories.
|
|
16
25
|
* @param decisions - LLM-produced decision list. In skipInvalid mode, invalid
|
|
@@ -128,6 +137,21 @@ export function validateDecisions(decisions, snapshot, options = {}) {
|
|
|
128
137
|
// update-specific field validation runs BEFORE claiming ids, so a failing
|
|
129
138
|
// update never pollutes the claimed set (which drives the "every id must
|
|
130
139
|
// appear in a decision" check below).
|
|
140
|
+
// Issue #104 方向 2:archive 类型护栏。长保留类型的 archive 决策必须携带
|
|
141
|
+
// 「重复 / 过时」类理由(提示词对 archive 的定义本就如此),「价值判断」类
|
|
142
|
+
// 理由(保持整洁、参考价值低……)不放行——单条跳过记 skipped(run 记
|
|
143
|
+
// degraded,明细进 dream_runs.skipped),绝不整单拒绝。未知 id 在下方通用
|
|
144
|
+
// 校验里另行报错,这里 snapshot.get 为 undefined 时跳过护栏判断。
|
|
145
|
+
if (local.length === 0 && d?.action === "archive") {
|
|
146
|
+
const guardedTypes = new Set();
|
|
147
|
+
for (const id of d.ids) {
|
|
148
|
+
const mem = snapshot.get(id);
|
|
149
|
+
if (mem && ARCHIVE_GUARDED_TYPES.has(mem.type)) guardedTypes.add(mem.type);
|
|
150
|
+
}
|
|
151
|
+
if (guardedTypes.size > 0 && !ARCHIVE_RATIONALE_RE.test(String(d.reason ?? ""))) {
|
|
152
|
+
local.push(`${at}: archive of long-retention type(s) ${[...guardedTypes].join(", ")} needs a duplicate/outdated rationale (重复/过时), got: ${JSON.stringify(String(d.reason ?? "")).slice(0, 60)}`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
131
155
|
if (local.length === 0 && d?.action === "update") {
|
|
132
156
|
// 只能更新单条
|
|
133
157
|
if (!Array.isArray(d.ids) || d.ids.length !== 1) {
|
|
@@ -216,26 +240,49 @@ export function validateDecisions(decisions, snapshot, options = {}) {
|
|
|
216
240
|
if (createCount > maxCreatePerRun) {
|
|
217
241
|
errors.push(`too many create decisions: ${createCount} > ${maxCreatePerRun}`);
|
|
218
242
|
}
|
|
243
|
+
// Issue #104 方向 2:archive 批量上限——一次性大扫除(12 条互不相关主题被
|
|
244
|
+
// 批量归档)是失控信号,与 update 上限同一类全局闸门:skipInvalid 也不豁免,
|
|
245
|
+
// 超限整单拒绝;正常的大规模清理由用户调高 dreamMaxArchivePerRun。
|
|
246
|
+
const archiveCount = survivors.filter((d) => d.action === "archive").length;
|
|
247
|
+
const maxArchivePerRun = options.maxArchivePerRun ?? 8;
|
|
248
|
+
if (archiveCount > maxArchivePerRun) {
|
|
249
|
+
errors.push(`too many archive decisions: ${archiveCount} > ${maxArchivePerRun}`);
|
|
250
|
+
}
|
|
251
|
+
// Issue #89 防洗白核心(#104 方向 1 重审后保留):全部决策都被跳过、合法
|
|
252
|
+
// 子集为空时仍整单拒绝——宽容的前提是「有合法子集可应用」;模型输出整体
|
|
253
|
+
// 非法(如全部幻觉 id)是真故障,不能被隐式 keep 洗成 ok。#104 只豁免
|
|
254
|
+
// 「有合法子集但覆盖率不足」的良性挑重点(见下方 coverageShortfall)。
|
|
255
|
+
if (options.skipInvalid === true && skipped.length > 0 && survivors.length === 0 && errors.length === 0) {
|
|
256
|
+
errors.push(`all ${skipped.length} decision(s) invalid — nothing valid survived validation`);
|
|
257
|
+
return { ok: false, errors, skipped, resolvedShortIds };
|
|
258
|
+
}
|
|
219
259
|
// v0.4.4: 隐式 keep。默认(dreamImplicitKeep !== false)下,未 claim 的
|
|
220
260
|
// snapshot 记忆自动补 {action:"keep"},而不是整体拒绝——大记忆量下 LLM 漏报
|
|
221
261
|
// 一两条就全拒(636 记忆 → 677 errors)会白白浪费整轮 run。设 false 则保留
|
|
222
262
|
// 旧的严格"全量覆盖"校验。补齐的 keep 直接 append 到 decisions,调用方
|
|
223
263
|
// (runDream/applyDecisions/audit)复用同一数组即可覆盖全部 snapshot 记忆。
|
|
224
264
|
//
|
|
225
|
-
// v0.4.4 fix
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
//
|
|
265
|
+
// v0.4.4 fix(残缺输出防洗白)→ Issue #104 方向 1 重审:显式覆盖率护栏的
|
|
266
|
+
// 初衷是拦「输出截断 → 只 claim 少量 → 其余被隐式 keep 洗白」,但它无法区分
|
|
267
|
+
// 恶性截断与良性挑重点——而提示词硬性规则本身要求「无问题的条目无需输出」,
|
|
268
|
+
// 挑重点正是要求的行为。官方路由 + 强模型的实测(heptaspirit:81 轮 failed
|
|
269
|
+
// 全是 coverage,LLM 调用本身成功、applied=0)证明这是常规路径而非边角。
|
|
270
|
+
// 处置:覆盖率不足不再 ok:false 整单拒绝,改为返回 coverageShortfall 理由、
|
|
271
|
+
// 照常补齐隐式 keep——调用方把 run 记 degraded(合法子集已应用),护栏以
|
|
272
|
+
// 状态暴露而非一票否决,与 skipInvalid 同一宽容哲学。明确异常信号(create
|
|
273
|
+
// 上限等硬错误)保持整单拒绝。要恢复旧行为可设 dreamImplicitKeep:false
|
|
274
|
+
// (严格模式,缺失即 error)或 dreamMinExplicitCoverage: 0(关护栏)。
|
|
275
|
+
// 非覆盖类硬错误(create 上限溢出、严格模式缺失 id)仍然先整单拒绝——
|
|
276
|
+
// #104 方向 1 只豁免 coverage 这一个信号,防洗白语义对它们原样保留。
|
|
230
277
|
if (errors.length > 0) {
|
|
231
278
|
return { ok: false, errors, skipped, resolvedShortIds };
|
|
232
279
|
}
|
|
233
280
|
const minCoverage = options.dreamMinExplicitCoverage ?? 0.5;
|
|
281
|
+
let coverageShortfall = null;
|
|
234
282
|
if (options.dreamImplicitKeep !== false) {
|
|
235
283
|
const coverage = snapshot.size > 0 ? claimed.size / snapshot.size : 1;
|
|
236
284
|
if (coverage < minCoverage) {
|
|
237
|
-
|
|
238
|
-
return { ok: false, errors, skipped, resolvedShortIds };
|
|
285
|
+
coverageShortfall = `explicit decision coverage ${Math.round(coverage * 100)}% < minimum ${Math.round(minCoverage * 100)}%`;
|
|
239
286
|
}
|
|
240
287
|
for (const id of snapshot.keys()) {
|
|
241
288
|
if (!claimed.has(id)) survivors.push({ action: "keep", ids: [id] });
|
|
@@ -252,7 +299,7 @@ export function validateDecisions(decisions, snapshot, options = {}) {
|
|
|
252
299
|
// 当"被跳过的非法决策数 == 隐式补齐的 keep 数"时长度回到相等但内容已变,
|
|
253
300
|
// 被跳过的决策会残留进 apply/audit。一律无条件 splice 最安全。
|
|
254
301
|
decisions.splice(0, decisions.length, ...survivors);
|
|
255
|
-
return { ok: true, errors, skipped, resolvedShortIds };
|
|
302
|
+
return { ok: true, errors, skipped, resolvedShortIds, coverageShortfall };
|
|
256
303
|
}
|
|
257
304
|
|
|
258
305
|
/** Marker thrown when a decision target changed since the run snapshot. */
|
package/lib/dream/sleep.js
CHANGED
|
@@ -239,8 +239,8 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
|
|
|
239
239
|
maxTokens: 2048,
|
|
240
240
|
...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
|
|
241
241
|
messages: [
|
|
242
|
-
{ role: "system", content: [{ type: "text", text: conflictPrompt }] },
|
|
243
|
-
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
242
|
+
{ role: "system", content: [{ type: "text", text: conflictPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
243
|
+
{ role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
244
244
|
]
|
|
245
245
|
}, (reason) => { conflictStreamFailure = describeStreamFailure(reason); });
|
|
246
246
|
};
|
|
@@ -390,8 +390,8 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
|
|
|
390
390
|
maxTokens: 2048,
|
|
391
391
|
...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
|
|
392
392
|
messages: [
|
|
393
|
-
{ role: "system", content: [{ type: "text", text: STR.prompts.pattern[language].replace("N", String(maxPatterns)) }] },
|
|
394
|
-
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
393
|
+
{ role: "system", content: [{ type: "text", text: STR.prompts.pattern[language].replace("N", String(maxPatterns)) }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
394
|
+
{ role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
395
395
|
]
|
|
396
396
|
}, (reason) => { patternStreamFailure = describeStreamFailure(reason); });
|
|
397
397
|
};
|
|
@@ -596,7 +596,33 @@ export function createSleepScheduler({
|
|
|
596
596
|
const delay = Math.max(0, idleMs - (now() - lastWriteAt)) + 1000;
|
|
597
597
|
idleTimer = setTimeoutFn(async () => {
|
|
598
598
|
idleTimer = null;
|
|
599
|
-
|
|
599
|
+
// Issue #187:false = 撞上 CD(或 run 失败)——此前这里直接丢弃,"CD 到期"
|
|
600
|
+
// 成了没有监听者的时刻,要等下一次写入才救回来。改为按剩余窗口重排。
|
|
601
|
+
const ran = await maybeSchedule();
|
|
602
|
+
if (!ran) armNextWindow();
|
|
603
|
+
}, delay);
|
|
604
|
+
idleTimer.unref?.();
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
// Issue #187:撞 CD / run 失败后的补挂表——按「剩余 CD / 剩余静默」里更晚的
|
|
608
|
+
// 到点时刻重排一次,回调里重复同样的判断所以不会空转。跑成了一轮就不重排:
|
|
609
|
+
// 安静的库没有要处理的新东西,触发权交还给下一次写入的 noteWrite。
|
|
610
|
+
function armNextWindow() {
|
|
611
|
+
if (disposed || idleTimer) return;
|
|
612
|
+
if (config.sleepModeEnabled !== true) return;
|
|
613
|
+
// lastRunAt === 0 表示从未跑过(此路径不该出现,防御性返回)。
|
|
614
|
+
if (lastRunAt <= 0) return;
|
|
615
|
+
const idleMs = (config.sleepIdleMinutes ?? 5) * 60000;
|
|
616
|
+
const cdMs = (config.sleepMinIntervalHours ?? 8) * 3600000;
|
|
617
|
+
const delay = Math.max(
|
|
618
|
+
lastRunAt + cdMs - now(),
|
|
619
|
+
lastWriteAt + idleMs - now(),
|
|
620
|
+
0
|
|
621
|
+
) + 1000;
|
|
622
|
+
idleTimer = setTimeoutFn(async () => {
|
|
623
|
+
idleTimer = null;
|
|
624
|
+
const ran = await maybeSchedule();
|
|
625
|
+
if (!ran) armNextWindow();
|
|
600
626
|
}, delay);
|
|
601
627
|
idleTimer.unref?.();
|
|
602
628
|
}
|
|
@@ -665,5 +691,10 @@ export function createSleepScheduler({
|
|
|
665
691
|
}
|
|
666
692
|
}
|
|
667
693
|
|
|
694
|
+
// Issue #187(同族缺陷):构造即挂表——重启后只要没发生过写入,此前永远没有
|
|
695
|
+
// 闹钟,安静多久都不会触发。lastWriteAt 已在构造时设为当前时刻,shouldRun 的
|
|
696
|
+
// 静默条件自然生效:重启后静默满 idleMinutes 才可能起跑。
|
|
697
|
+
armIdleTimer();
|
|
698
|
+
|
|
668
699
|
return { noteWrite, maybeSchedule, shouldRun, dispose };
|
|
669
700
|
}
|
package/lib/dream.js
CHANGED
|
@@ -910,8 +910,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
910
910
|
maxTokens: config.dreamMaxTokens ?? 4096,
|
|
911
911
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
912
912
|
messages: [
|
|
913
|
-
{ role: "system", content: [{ type: "text", text: consolidationPrompt }] },
|
|
914
|
-
{ role: "user", content: [{ type: "text", text: listText }] }
|
|
913
|
+
{ role: "system", content: [{ type: "text", text: consolidationPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
914
|
+
{ role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
915
915
|
]
|
|
916
916
|
}, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
|
|
917
917
|
};
|
|
@@ -933,8 +933,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
933
933
|
logger?.warn?.(`dsh-mneme dream: no json array in llm output (raw length ${decisionText?.length ?? 0}; head: ${head})`);
|
|
934
934
|
return finish({ ok: false, error: "no json array in llm output", summary: false });
|
|
935
935
|
}
|
|
936
|
-
const { ok, errors, skipped, resolvedShortIds } = validateDecisions(decisions, snapshot, {
|
|
936
|
+
const { ok, errors, skipped, resolvedShortIds, coverageShortfall } = validateDecisions(decisions, snapshot, {
|
|
937
937
|
maxUpdatePerRun: config.reflectionUpdateMaxPerRun,
|
|
938
|
+
// Issue #104 方向 2:archive 批量上限(全局闸门,超限整单拒绝)。
|
|
939
|
+
maxArchivePerRun: config.dreamMaxArchivePerRun,
|
|
938
940
|
minAgeHours: config.reflectionUpdateMinAgeHours,
|
|
939
941
|
// v0.4.4 fix:显式透传,用户配 dreamImplicitKeep:false 时严格模式必须
|
|
940
942
|
// 真正生效,dreamMinExplicitCoverage 决定隐式 keep 下的覆盖率下限。
|
|
@@ -972,6 +974,12 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
972
974
|
if (skippedInvalid) {
|
|
973
975
|
logger?.warn?.(`dsh-mneme dream: ${skipped.length} invalid decision(s) skipped (run degrades): ${skipped.map((s) => s.error).join("; ")}`);
|
|
974
976
|
}
|
|
977
|
+
// Issue #104 方向 1:覆盖率不足降级为 degraded(不再整单拒绝)——合法子集
|
|
978
|
+
// 照常应用、未提及条目隐式 keep,理由随 outcome.degradations 落库供离线定位。
|
|
979
|
+
const coverageDegraded = typeof coverageShortfall === "string" && coverageShortfall.length > 0;
|
|
980
|
+
if (coverageDegraded) {
|
|
981
|
+
logger?.warn?.(`dsh-mneme dream: ${coverageShortfall} (run degrades: valid subset applied, unclaimed ids kept implicitly)`);
|
|
982
|
+
}
|
|
975
983
|
|
|
976
984
|
// Capture pre-update snapshots so the audit records what each update changed.
|
|
977
985
|
const updateSnapshots = {};
|
|
@@ -1087,6 +1095,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1087
1095
|
// claim "merge-archived" (item ②). Conflicts/failures ride along so the
|
|
1088
1096
|
// audit row records why the run diverged.
|
|
1089
1097
|
const outcome = { ...buildOutcome(committed), conflicts, failures };
|
|
1098
|
+
if (coverageDegraded) outcome.degradations = [coverageShortfall];
|
|
1090
1099
|
// Frozen conflicts were not adjudicated: mark both sides pending in the
|
|
1091
1100
|
// per-id outcome so the audit row shows they were parked, not skipped.
|
|
1092
1101
|
if (frozenIds.length) {
|
|
@@ -1130,8 +1139,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1130
1139
|
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
1131
1140
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
1132
1141
|
messages: [
|
|
1133
|
-
{ role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }] },
|
|
1134
|
-
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }] }
|
|
1142
|
+
{ role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
1143
|
+
{ role: "user", content: [{ type: "text", text: service.all().filter((m) => !m.archived && m.type !== "summary").map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
1135
1144
|
]
|
|
1136
1145
|
}, reportUsage, (reason) => { summaryStreamFailure = describeStreamFailure(reason); }));
|
|
1137
1146
|
};
|
|
@@ -1171,7 +1180,9 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1171
1180
|
// degraded — real consolidation landed but the run did not produce its
|
|
1172
1181
|
// full output: the summary came back empty/missing, or
|
|
1173
1182
|
// skipInvalid dropped individually-invalid decisions
|
|
1174
|
-
// (Issue #89 — marked, not faked)
|
|
1183
|
+
// (Issue #89 — marked, not faked), or the explicit-decision
|
|
1184
|
+
// coverage fell below dreamMinExplicitCoverage (Issue #104
|
|
1185
|
+
// 方向 1 — degrade, don't reject). The valid subset was
|
|
1175
1186
|
// absorbed (ok for the baseline).
|
|
1176
1187
|
let status;
|
|
1177
1188
|
let okResult;
|
|
@@ -1182,7 +1193,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1182
1193
|
status = summaryStored ? "ok" : "noop";
|
|
1183
1194
|
okResult = summaryStored;
|
|
1184
1195
|
} else {
|
|
1185
|
-
status = summaryStored && !skippedInvalid ? "ok" : "degraded";
|
|
1196
|
+
status = summaryStored && !skippedInvalid && !coverageDegraded ? "ok" : "degraded";
|
|
1186
1197
|
okResult = true;
|
|
1187
1198
|
}
|
|
1188
1199
|
return finish({
|
|
@@ -150,8 +150,8 @@ export async function extractEntities(memory, { store, config, callLLM, logger }
|
|
|
150
150
|
const userText = buildUserMessage(memory.content);
|
|
151
151
|
|
|
152
152
|
const messages = [
|
|
153
|
-
{ role: "system", content: [{ type: "text", text: systemPrompt }] },
|
|
154
|
-
{ role: "user", content: [{ type: "text", text: userText }] }
|
|
153
|
+
{ role: "system", content: [{ type: "text", text: systemPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
154
|
+
{ role: "user", content: [{ type: "text", text: userText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
155
155
|
];
|
|
156
156
|
|
|
157
157
|
// Issue #109: optional provider/model override + reasoning effort are
|
package/lib/index.js
CHANGED
|
@@ -275,6 +275,8 @@ export const apply = (ctx, config) => {
|
|
|
275
275
|
batchSize: cfg.localEmbedBatchSize,
|
|
276
276
|
cacheDir: cfg.embedModelCacheDir,
|
|
277
277
|
runtimeDir: cfg.runtimeDir,
|
|
278
|
+
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
279
|
+
remoteHost: cfg.embedModelMirror,
|
|
278
280
|
baseUrl: cfg.ollamaBaseUrl,
|
|
279
281
|
logger: ctx.logger
|
|
280
282
|
});
|
|
@@ -327,6 +329,10 @@ export const apply = (ctx, config) => {
|
|
|
327
329
|
device: cfg.localEmbedDevice,
|
|
328
330
|
cacheDir: cfg.embedModelCacheDir,
|
|
329
331
|
runtimeDir: cfg.runtimeDir,
|
|
332
|
+
// #188:量化档默认 q8(此前不传 dtype 会去要 1GB 级 fp32 模型);
|
|
333
|
+
// embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
|
|
334
|
+
useDtype: cfg.rerankDtype,
|
|
335
|
+
remoteHost: cfg.embedModelMirror,
|
|
330
336
|
logger: ctx.logger
|
|
331
337
|
});
|
|
332
338
|
service.setReranker(reranker);
|
package/lib/lang.js
CHANGED
|
@@ -23,8 +23,8 @@ const PROMPTS = {
|
|
|
23
23
|
"en": "You are the memory overview assistant. From the consolidated memories below, produce a 150-200 word memory-library overview covering: user preferences, active projects, key decisions. This will be injected as conversation context. Output only the summary text, nothing else."
|
|
24
24
|
},
|
|
25
25
|
"consolidation": {
|
|
26
|
-
"zh": "你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。\n请执行记忆巩固(consolidation),输出一个决策 JSON 数组。\n\n【决策格式(必须严格遵守)】\n每个决策必须是对象,字段固定:\n- \"action\":必填。取值只能是 \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" 之一(字段名必须是 action,严禁写成 type)\n- \"ids\":必填,数组,本决策涉及的记忆 id 列表\n- \"reason\":可选,字符串,决策理由\n- \"importance\":可选,整数 1-5\n- merge 额外字段:\"keepSource\"(单个 id 字符串,必须是 ids 之一)+ 合并后的 \"title\"、\"content\"\n- conflict 额外字段:\"winner\" 与 \"loser\",都是【单个 id 字符串,不是数组】\n- update 额外字段:修正后的 \"title\" 和/或 \"content\";\"ids\" 只能包含一个 id\n\n【决策 JSON 示例】\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"合并标题\", \"content\": \"合并后的摘要内容\", \"importance\": 4, \"reason\": \"主题相近\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"内容矛盾,保留更新的信息\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"修正后的内容\", \"reason\": \"信息过时\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"重复或过时\" }\n]\n\n【任务】\n1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)\n2. 识别重复/过时信息 → archive\n3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)\n4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)\n - update 的 ids 只能包含一个 id\n - 必须提供修正后的 title 和/或 content\n - 仅当内容确实需要修正时才使用,不要滥用\n - 每次整理最多输出 2 个 update\n - 24 小时内新建的记忆不可 update\n5. 无问题的条目无需输出(未提及的条目将自动保留 keep)\n\n【硬性规则】\n- 字段名必须精确为 \"action\",严禁写成 \"type\";字段名统一用双引号\n- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】\n- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)\n- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep\n- merge 的 keepSource 必须是 ids 之一\n- 仅合并同类型条目(type 相同)\n- 不要编造 ids;只使用提供的 id\n- 重要性 1-5,合并后取最高\n- 只输出 JSON 数组,不要其他文字",
|
|
27
|
-
"en": "You are the memory consolidation assistant. Below are all memory entries (id, type, title, content, importance, updated time).\nPerform memory consolidation and output a JSON array of decisions.\n\n[Decision format (must be followed strictly)]\nEach decision must be an object with fixed fields:\n- \"action\": required. Value must be one of \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" (the field name must be action, never type)\n- \"ids\": required, array, the list of memory ids involved in this decision\n- \"reason\": optional, string, rationale for the decision\n- \"importance\": optional, integer 1-5\n- merge extra fields: \"keepSource\" (a single id string, must be one of ids) plus the merged \"title\" and \"content\"\n- conflict extra fields: \"winner\" and \"loser\", each a [single id string, never an array]\n- update extra fields: the corrected \"title\" and/or \"content\"; \"ids\" may contain only one id\n\n[Decision JSON examples]\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"Merged title\", \"content\": \"Merged summary content\", \"importance\": 4, \"reason\": \"similar topics\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"content contradicts; keep the newer information\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"corrected content\", \"reason\": \"information outdated\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"duplicate or outdated\" }\n]\n\n[Task]\n1. Identify entries with similar topics -> merge (merge into a more refined summary; keep the id with the most complete information as keepSource)\n2. Identify duplicate/outdated information -> archive\n3. Identify entries with contradictory content -> conflict (judge winner/loser by recency, source completeness, and information specificity)\n4. Find outdated, wrong, or missing information in a single memory -> update (correct the content directly)\n - update's ids may contain only one id\n - must provide the corrected title and/or content\n - use only when the content truly needs correction; do not abuse\n - output at most 2 updates per consolidation run\n - memories created within the last 24 hours must not be updated\n5. Entries without issues need no output (unmentioned entries are automatically kept)\n\n[Hard rules]\n- Field names must be exactly \"action\", never \"type\"; use double quotes for all field names\n- conflict winner/loser and merge keepSource are [single id strings, never arrays]\n- Each memory may be claimed at most once: the same id must not appear in multiple decisions (an id must not be claimed by both merge and conflict/archive, etc.)\n- Memories not mentioned in any decision are automatically kept (keep); no need to output keep for every memory\n- merge keepSource must be one of ids\n- Only merge entries of the same type (identical type)\n- Never invent ids; use only the provided ids\n- Importance is 1-5; after a merge take the highest\n- Write every title, content, and reason in English\n- Output only the JSON array, nothing else"
|
|
26
|
+
"zh": "你是记忆库整理助手。下面是全部记忆条目(id、类型、标题、内容、重要性、更新时间)。\n请执行记忆巩固(consolidation),输出一个决策 JSON 数组。\n\n【决策格式(必须严格遵守)】\n每个决策必须是对象,字段固定:\n- \"action\":必填。取值只能是 \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" 之一(字段名必须是 action,严禁写成 type)\n- \"ids\":必填,数组,本决策涉及的记忆 id 列表\n- \"reason\":可选,字符串,决策理由\n- \"importance\":可选,整数 1-5\n- merge 额外字段:\"keepSource\"(单个 id 字符串,必须是 ids 之一)+ 合并后的 \"title\"、\"content\"\n- conflict 额外字段:\"winner\" 与 \"loser\",都是【单个 id 字符串,不是数组】\n- update 额外字段:修正后的 \"title\" 和/或 \"content\";\"ids\" 只能包含一个 id\n\n【决策 JSON 示例】\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"合并标题\", \"content\": \"合并后的摘要内容\", \"importance\": 4, \"reason\": \"主题相近\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"内容矛盾,保留更新的信息\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"修正后的内容\", \"reason\": \"信息过时\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"重复或过时\" }\n]\n\n【任务】\n1. 识别主题相近的条目 → merge(合并为更精炼的摘要,保留信息最完整的 id 作为 keepSource)\n2. 识别重复/过时信息 → archive\n3. 识别内容矛盾的条目 → conflict(按时间新旧、来源完整性、信息具体程度判断 winner/loser)\n4. 发现单条记忆中的信息过时、错误或遗漏 → update(直接修正内容)\n - update 的 ids 只能包含一个 id\n - 必须提供修正后的 title 和/或 content\n - 仅当内容确实需要修正时才使用,不要滥用\n - 每次整理最多输出 2 个 update\n - 24 小时内新建的记忆不可 update\n5. 无问题的条目无需输出(未提及的条目将自动保留 keep)\n\n【硬性规则】\n- 字段名必须精确为 \"action\",严禁写成 \"type\";字段名统一用双引号\n- conflict 的 winner/loser、merge 的 keepSource 都是【单个 id 字符串,绝不是数组】\n- 每条记忆最多被 claim 一次:同一个 id 不能出现在多个决策中(同一 id 不能被 merge 和 conflict/archive 等重复占用)\n- 未在决策中提及的记忆将自动保留(keep),无需为每条记忆输出 keep\n- merge 的 keepSource 必须是 ids 之一\n- 仅合并同类型条目(type 相同)\n- 不要编造 ids;只使用提供的 id\n- 重要性 1-5,合并后取最高\n- archive 仅用于确属「重复或已过时」的条目;rejected_solution / constraint / pitfall / preference / pattern 属长保留类型,除非确属重复或过时,否则不要归档\n- 每次整理最多输出 8 个 archive\n- 只输出 JSON 数组,不要其他文字",
|
|
27
|
+
"en": "You are the memory consolidation assistant. Below are all memory entries (id, type, title, content, importance, updated time).\nPerform memory consolidation and output a JSON array of decisions.\n\n[Decision format (must be followed strictly)]\nEach decision must be an object with fixed fields:\n- \"action\": required. Value must be one of \"keep\" / \"merge\" / \"archive\" / \"update\" / \"conflict\" (the field name must be action, never type)\n- \"ids\": required, array, the list of memory ids involved in this decision\n- \"reason\": optional, string, rationale for the decision\n- \"importance\": optional, integer 1-5\n- merge extra fields: \"keepSource\" (a single id string, must be one of ids) plus the merged \"title\" and \"content\"\n- conflict extra fields: \"winner\" and \"loser\", each a [single id string, never an array]\n- update extra fields: the corrected \"title\" and/or \"content\"; \"ids\" may contain only one id\n\n[Decision JSON examples]\n[\n { \"action\": \"merge\", \"ids\": [\"m1\", \"m2\"], \"keepSource\": \"m1\", \"title\": \"Merged title\", \"content\": \"Merged summary content\", \"importance\": 4, \"reason\": \"similar topics\" },\n { \"action\": \"conflict\", \"winner\": \"m3\", \"loser\": \"m4\", \"reason\": \"content contradicts; keep the newer information\" },\n { \"action\": \"update\", \"ids\": [\"m5\"], \"content\": \"corrected content\", \"reason\": \"information outdated\" },\n { \"action\": \"archive\", \"ids\": [\"m6\"], \"reason\": \"duplicate or outdated\" }\n]\n\n[Task]\n1. Identify entries with similar topics -> merge (merge into a more refined summary; keep the id with the most complete information as keepSource)\n2. Identify duplicate/outdated information -> archive\n3. Identify entries with contradictory content -> conflict (judge winner/loser by recency, source completeness, and information specificity)\n4. Find outdated, wrong, or missing information in a single memory -> update (correct the content directly)\n - update's ids may contain only one id\n - must provide the corrected title and/or content\n - use only when the content truly needs correction; do not abuse\n - output at most 2 updates per consolidation run\n - memories created within the last 24 hours must not be updated\n5. Entries without issues need no output (unmentioned entries are automatically kept)\n\n[Hard rules]\n- Field names must be exactly \"action\", never \"type\"; use double quotes for all field names\n- conflict winner/loser and merge keepSource are [single id strings, never arrays]\n- Each memory may be claimed at most once: the same id must not appear in multiple decisions (an id must not be claimed by both merge and conflict/archive, etc.)\n- Memories not mentioned in any decision are automatically kept (keep); no need to output keep for every memory\n- merge keepSource must be one of ids\n- Only merge entries of the same type (identical type)\n- Never invent ids; use only the provided ids\n- Importance is 1-5; after a merge take the highest\n- Write every title, content, and reason in English\n- archive is only for entries that are genuinely \"duplicate or outdated\"; rejected_solution / constraint / pitfall / preference / pattern are long-retention types — do not archive them unless they are truly duplicate or outdated\n- output at most 8 archives per consolidation run\n- Output only the JSON array, nothing else"
|
|
28
28
|
},
|
|
29
29
|
"conflict": {
|
|
30
30
|
"zh": "你是记忆库冲突仲裁助手。下面是检测到的高相似度记忆对,可能内容矛盾或重复。\n对每一对输出一个 decision 对象:\n- 两条确实矛盾/重复 → { \"action\": \"conflict\", \"winner\": <保留的id>, \"loser\": <归档的id>, \"reason\": \"理由\" }\n- 两条只是主题相近、并无矛盾 → { \"action\": \"keep\", \"ids\": [<两个id>] }\n规则:\n- winner 应为信息更完整、更新或更可信的一条\n- 只使用提供的 id,不要编造\n- 每对必须输出一个 decision\n- 只输出 JSON 数组,不要其他文字",
|
package/lib/local-embedder.js
CHANGED
|
@@ -30,7 +30,7 @@ function modelHash(model) {
|
|
|
30
30
|
* transformers 的选项里。
|
|
31
31
|
*/
|
|
32
32
|
async function defaultPipelineLoader(task, model, options) {
|
|
33
|
-
const { runtimeDir, ...pipelineOptions } = options ?? {};
|
|
33
|
+
const { runtimeDir, remoteHost, ...pipelineOptions } = options ?? {};
|
|
34
34
|
const { module } = await loadTransformers({
|
|
35
35
|
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
36
36
|
});
|
|
@@ -41,6 +41,11 @@ async function defaultPipelineLoader(task, model, options) {
|
|
|
41
41
|
// model is fully cached locally. Mirroring the cache_dir onto env.cacheDir
|
|
42
42
|
// makes that pre-check resolve locally too — fully offline loading.
|
|
43
43
|
if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
|
|
44
|
+
// issue #188 补充:embedModelMirror 此前是死配置(定义了但全仓无人读取)。
|
|
45
|
+
// transformers.js 不认 HF_ENDPOINT(那是 Python huggingface_hub 的变量),
|
|
46
|
+
// 模型下载源由 env.remoteHost 决定——把镜像接到真正的开关上,否则
|
|
47
|
+
// huggingface.co 不可达的网络连模型都下不下来。
|
|
48
|
+
if (remoteHost) env.remoteHost = remoteHost;
|
|
44
49
|
return pipeline(task, model, pipelineOptions);
|
|
45
50
|
}
|
|
46
51
|
|
|
@@ -72,6 +77,8 @@ export class LocalEmbedder {
|
|
|
72
77
|
// 自管运行时根目录(issue #131):空表示用默认位置,具体解析交给 loader。
|
|
73
78
|
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
74
79
|
this.useDtype = opts.useDtype || "q8";
|
|
80
|
+
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
81
|
+
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
75
82
|
this.logger = opts.logger ?? null;
|
|
76
83
|
// Test hook: replace the pipeline factory without touching modules.
|
|
77
84
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
@@ -91,6 +98,7 @@ export class LocalEmbedder {
|
|
|
91
98
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
92
99
|
// 传给注入的 engineFactory,由 defaultPipelineLoader 取走后交给三层解析。
|
|
93
100
|
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
101
|
+
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
94
102
|
this.extractor = await this.engineFactory("feature-extraction", this.model, options);
|
|
95
103
|
this.ready = true; // service reads this to flush queued re-embeds
|
|
96
104
|
this.logger?.info?.(
|
package/lib/reranker.js
CHANGED
|
@@ -27,7 +27,7 @@ function modelHash(model) {
|
|
|
27
27
|
* runtimeDir 取走后不混进 transformers 的选项。
|
|
28
28
|
*/
|
|
29
29
|
async function defaultPipelineLoader(task, model, options) {
|
|
30
|
-
const { runtimeDir, ...pipelineOptions } = options ?? {};
|
|
30
|
+
const { runtimeDir, remoteHost, ...pipelineOptions } = options ?? {};
|
|
31
31
|
const { module } = await loadTransformers({
|
|
32
32
|
runtimeDir: runtimeDir || defaultRuntimeDir()
|
|
33
33
|
});
|
|
@@ -36,6 +36,10 @@ async function defaultPipelineLoader(task, model, options) {
|
|
|
36
36
|
// 的 cache_dir,回退到 env.cacheDir —— 即使模型已完全缓存在本地也会去请求网络。
|
|
37
37
|
// embedder 侧早就镜像了,reranker 侧一直漏着,于是「已缓存 + 离线」时重排初始化会失败。
|
|
38
38
|
if (pipelineOptions.cache_dir) env.cacheDir = pipelineOptions.cache_dir;
|
|
39
|
+
// issue #188 补充:embedModelMirror 此前是死配置——transformers.js 不认
|
|
40
|
+
// HF_ENDPOINT(那是 Python huggingface_hub 的变量),下载源由 env.remoteHost
|
|
41
|
+
// 决定。把镜像接到真正的开关上,huggingface.co 不可达的网络才能自动下载。
|
|
42
|
+
if (remoteHost) env.remoteHost = remoteHost;
|
|
39
43
|
return pipeline(task, model, pipelineOptions);
|
|
40
44
|
}
|
|
41
45
|
|
|
@@ -85,6 +89,11 @@ export class LocalReranker {
|
|
|
85
89
|
this.cacheDir = String(opts.cacheDir ?? "").trim() || defaultModelCacheDir();
|
|
86
90
|
// 自管运行时根目录(issue #131):空表示用默认位置,解析交给 loader。
|
|
87
91
|
this.runtimeDir = String(opts.runtimeDir ?? "").trim();
|
|
92
|
+
// 与嵌入层同约定:默认 q8(4 倍小的 model_quantized.onnx),"fp32" 可关。
|
|
93
|
+
// 此前不传 dtype,即使缓存里有量化文件也会去要 1GB 级的 model.onnx(#188)。
|
|
94
|
+
this.useDtype = String(opts.useDtype ?? "q8").trim() || "q8";
|
|
95
|
+
// 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
|
|
96
|
+
this.remoteHost = String(opts.remoteHost ?? "").trim();
|
|
88
97
|
this.logger = opts.logger ?? null;
|
|
89
98
|
this.engineFactory = opts.engineFactory || defaultPipelineLoader;
|
|
90
99
|
// Injectable seam: async (query, passage) => number. When set, init()
|
|
@@ -94,6 +103,7 @@ export class LocalReranker {
|
|
|
94
103
|
this._batchScorer = null;
|
|
95
104
|
this._queryVec = null;
|
|
96
105
|
this._queryKey = null;
|
|
106
|
+
this._constantScoreWarned = false;
|
|
97
107
|
}
|
|
98
108
|
|
|
99
109
|
/** Load the model; throws when no strategy can be bound. */
|
|
@@ -119,9 +129,10 @@ export class LocalReranker {
|
|
|
119
129
|
}
|
|
120
130
|
|
|
121
131
|
_engineOptions() {
|
|
122
|
-
const options = { device: this.device };
|
|
132
|
+
const options = { device: this.device, dtype: this.useDtype };
|
|
123
133
|
if (this.cacheDir) options.cache_dir = this.cacheDir;
|
|
124
134
|
if (this.runtimeDir) options.runtimeDir = this.runtimeDir;
|
|
135
|
+
if (this.remoteHost) options.remoteHost = this.remoteHost;
|
|
125
136
|
return options;
|
|
126
137
|
}
|
|
127
138
|
|
|
@@ -136,8 +147,14 @@ export class LocalReranker {
|
|
|
136
147
|
return out.map((r) => clamp01(typeof r?.score === "number" ? r.score : 0));
|
|
137
148
|
};
|
|
138
149
|
} else if (strategy === "tc") {
|
|
139
|
-
// Classification head:
|
|
140
|
-
//
|
|
150
|
+
// Classification head: score so relevance lands in [0, 1]. Branch on the
|
|
151
|
+
// output width (issue #188): a dual-logit head (num_labels=2 exports,
|
|
152
|
+
// e.g. bge-reranker-v2-m3) gives the positive class as the softmax
|
|
153
|
+
// probability sigmoid(l1 - l0); a SINGLE-logit head (e.g.
|
|
154
|
+
// Xenova/bge-reranker-base) emits one raw relevance logit per pair, and
|
|
155
|
+
// the positive class is simply sigmoid(logit) — reading l1 as l0 there
|
|
156
|
+
// collapses every score to sigmoid(0) = 0.5 and silently disables
|
|
157
|
+
// reranking (no error, status still "ready").
|
|
141
158
|
this._batchScorer = async (query, passages) => {
|
|
142
159
|
const { tokenizer, model } = this.pipeline;
|
|
143
160
|
const inputs = tokenizer(passages.map(() => query), {
|
|
@@ -151,10 +168,10 @@ export class LocalReranker {
|
|
|
151
168
|
const rows = [];
|
|
152
169
|
for (let i = 0; i < dims[0]; i++) {
|
|
153
170
|
const base = i * cols;
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
rows.push(clamp01(
|
|
171
|
+
const score = cols > 1
|
|
172
|
+
? 1 / (1 + Math.exp(logits.data[base] - logits.data[base + 1]))
|
|
173
|
+
: 1 / (1 + Math.exp(-logits.data[base]));
|
|
174
|
+
rows.push(clamp01(score));
|
|
158
175
|
}
|
|
159
176
|
return rows;
|
|
160
177
|
};
|
|
@@ -208,6 +225,19 @@ export class LocalReranker {
|
|
|
208
225
|
results.push({ id: chunk[j].id, score: clamp01(Number(scores[j]) || 0) });
|
|
209
226
|
}
|
|
210
227
|
}
|
|
228
|
+
// Silent-failure tripwire (issue #188): a reranker whose every candidate
|
|
229
|
+
// scores identically is not ranking anything — e.g. a single-logit head
|
|
230
|
+
// read through the dual-logit formula. Warn once so the degradation is
|
|
231
|
+
// observable instead of a constant 0.5 column hiding behind a "ready"
|
|
232
|
+
// status forever.
|
|
233
|
+
if (results.length > 1 && results.every((r) => r.score === results[0].score)) {
|
|
234
|
+
if (!this._constantScoreWarned) {
|
|
235
|
+
this._constantScoreWarned = true;
|
|
236
|
+
this.logger?.warn?.(
|
|
237
|
+
`[dsh-mneme] reranker scores degenerated to a constant (${results[0].score}) — check the model's output dims (single vs dual logit head)`
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
211
241
|
return results
|
|
212
242
|
.filter((r) => r.score >= this.scoreThreshold)
|
|
213
243
|
.sort((a, b) => b.score - a.score);
|