dsh-dot-life 0.11.3 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/docs/ROADMAP.md +2 -1
- package/docs/memory.md +42 -0
- package/package.json +1 -1
- package/src/dot/core.js +12 -3
- package/src/dot/panel.js +1 -0
- package/src/memory/memory.js +80 -4
- package/src/memory/recall.js +443 -0
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ One plugin, six components (living next to other dsh plugins: see 「与其他
|
|
|
11
11
|
| **`dsh-dot/phone`** | Her phone: an Android emulator the plugin downloads (emulator + platform-tools + one system image, ~2 GB, after you accept the SDK license), boots, keeps alive and exposes as `mcp__dot-phone__*` ([mobile-mcp](https://github.com/mobile-next/mobile-mcp) over adb). The emulator window is the live view; her apps and logins live in the AVD and survive restarts. `DSH_DOT_PHONE=off` turns it off. [Details](docs/phone.md) | hardware virtualization (Apple silicon or Intel Mac, Linux with KVM, Windows with WHPX) |
|
|
12
12
|
| **`dsh-dot/interlude`** | 幕间, after [HDS-Interlude](https://github.com/REcklessNOLOve117/HDS-Interlude): the Canon (persona; HDSI presets by share code) as a prompt section; a life between conversations — script, continuity, intents; going to look at things on her phone or in her browser (read-only, no approvals); optional proactive contact. `DSH_DOT_INTERLUDE=off` turns it off. [Details](docs/interlude.md) | — |
|
|
13
13
|
| **`dsh-dot/qq`** | Her QQ on her computer: [NapCat](https://github.com/NapNeko/NapCatQQ) or [SnowLuma](https://github.com/SnowLuma/SnowLuma) (OneBot 11, `/qq backend`) inside the computer VM, the same account as the QQ app on her phone; messages wake her the moment they arrive, she answers with `qq_send`. `DSH_DOT_QQ=off` turns it off. [Details](docs/qq.md) | the computer component |
|
|
14
|
-
| **`dsh-dot/memory`** | What she remembers: you, her own notes, the people she meets, what happened; hands her conversation over to a fresh session past 150k tokens. `DSH_DOT_MEMORY=off` turns it off. [Details](docs/memory.md) | — |
|
|
14
|
+
| **`dsh-dot/memory`** | What she remembers: you, her own notes, the people she meets, what happened; recalls the old turns most like the message that woke her (local embedding by default, cloud optional); hands her conversation over to a fresh session past 150k tokens. `DSH_DOT_MEMORY=off` turns it off. [Details](docs/memory.md) | — |
|
|
15
15
|
| **`dsh-dot/dot`** | The contact layer: scheduled results → IM unless the reply is `DOT_SILENT`, approval and device notices on IM, working rules as a prompt section. `DSH_DOT_CONTACT=off` turns it off | [dsh-im](https://github.com/xmanrui/dsh-im) for IM; dsh's built-in schedule service |
|
|
16
16
|
| **`dsh-dot/computer`** (optional) | A bundled microVM (no Docker needed) with shell, browser, desktop, live view, approval gates and an audit log — the isolation boundary for work. `DSH_DOT_COMPUTER=off` turns it off. [Details](docs/computer.md) | [microsandbox](https://github.com/zerocore-ai/microsandbox) (bundled) or Docker |
|
|
17
17
|
|
|
@@ -38,7 +38,7 @@ dsh-dot 让 dsh 里住进一个**在自己手机上过日子的角色**。她有
|
|
|
38
38
|
| **`dsh-dot/phone`** 她的手机 | Android 模拟器。插件自己从 Google 仓库下载模拟器、platform-tools 和一个系统镜像(约 2 GB,先要你接受 SDK 许可),自己写 AVD(不需要 Java / Android Studio),开机、保活,以 `mcp__dot-phone__*` 暴露给模型([mobile-mcp](https://github.com/mobile-next/mobile-mcp) 走 adb)。模拟器窗口就是实时画面;App 和登录态在 AVD 里,重启不丢。`DSH_DOT_PHONE=off` 关掉。详见 [docs/phone.md](docs/phone.md) | 硬件虚拟化:Apple silicon / Intel Mac,Linux 要 KVM,Windows 要 WHPX |
|
|
39
39
|
| **`dsh-dot/interlude`** 幕间 | 按 [HDS-Interlude](https://github.com/REcklessNOLOve117/HDS-Interlude):Canon(人设,HDSI 分享码导入)作为系统提示的一段;两次对话之间的生活——剧本、连续性、打算;想看什么就拿手机(或电脑浏览器)去看,只读、无审批;可选的主动联系。`DSH_DOT_INTERLUDE=off` 关掉。详见 [docs/interlude.md](docs/interlude.md) | — |
|
|
40
40
|
| **`dsh-dot/qq`** 她电脑上的 QQ | [NapCat](https://github.com/NapNeko/NapCatQQ) 或 [SnowLuma](https://github.com/SnowLuma/SnowLuma)(OneBot 11,`/qq backend` 切换)跑在她的电脑 VM 里,和手机上的 QQ 同一个账号;消息到的瞬间叫醒她,她用 `qq_send` 秒回;dot 的提醒和她的主动联系也走这里,官方机器人退为后备。`DSH_DOT_QQ=off` 关掉(电脑关了它也关)。详见 [docs/qq.md](docs/qq.md) | 电脑组件 |
|
|
41
|
-
| **`dsh-dot/memory`** 她记得的 | 关于你的画像和她自己的笔记(按 Hermes 的做法,有字数上限,她自己能改),她在各个 App 里遇到的人(每人一张卡:互动次数由代码根据真实事件记,印象和事由模型写;只在帖子里互动过的人只记次数和一句印象;关于别人的事先进待核实区),发生过的事(按 dsh-whale-inurqq:碎片带重要度、被说起几次、会老化)。她的对话到 15 万 token 就交班换一个新会话,远早于 dsh 自己在 80%
|
|
41
|
+
| **`dsh-dot/memory`** 她记得的 | 关于你的画像和她自己的笔记(按 Hermes 的做法,有字数上限,她自己能改),她在各个 App 里遇到的人(每人一张卡:互动次数由代码根据真实事件记,印象和事由模型写;只在帖子里互动过的人只记次数和一句印象;关于别人的事先进待核实区),发生过的事(按 dsh-whale-inurqq:碎片带重要度、被说起几次、会老化)。她的对话到 15 万 token 就交班换一个新会话,远早于 dsh 自己在 80% 窗口时的压缩。0.12 起每次她被叫醒,还会用叫醒她的那句话从以前的聊天里自动翻出最相关的原话放在她面前(默认本机 bge-small-zh,不要 key,第一次用时下载;也可以切 SiliconFlow / DashScope 或关掉,一条 `/dot memory recall …` 命令,不用重启)。公开记忆基准 108 题上从 74 题提到 93 题。`DSH_DOT_MEMORY=off` 关掉。详见 [docs/memory.md](docs/memory.md) | — |
|
|
42
42
|
| **`dsh-dot/dot`** 联系人层 | 定时任务的结果推到 IM(回复只是 `DOT_SILENT` 就不发)、审批和设备状况的提醒、工作规则作为系统提示的一段。`DSH_DOT_CONTACT=off` 关掉 | IM 用 [dsh-im](https://github.com/xmanrui/dsh-im);定时用 dsh 内置的定时服务 |
|
|
43
43
|
| **`dsh-dot/computer`**(可选)她的电脑 | 自带 microVM(不需要 Docker),里面有 shell、浏览器、桌面;实时画面、审批闸门、审计日志。干活时的隔离边界。`DSH_DOT_COMPUTER=off` 关掉。详见 [docs/computer.md](docs/computer.md) | [microsandbox](https://github.com/zerocore-ai/microsandbox)(自带)或 Docker |
|
|
44
44
|
|
|
@@ -89,10 +89,10 @@ Web 版:`dsh plugin --profile web add dsh-dot-life`。从源码装:把包名
|
|
|
89
89
|
| | |
|
|
90
90
|
|---|---|
|
|
91
91
|
| `~/dot/interlude/` | 人设 `canon.json`、剧本 `script.md`、连续性 `state.json`、打算 `intents.json`、看到的东西 `web-context.jsonl`、每轮原始回复和 token 用量 `calls.jsonl` |
|
|
92
|
-
| `~/dot/memory/` | `memory.md`(给你看的:关于你、她的笔记、认识的人、待核实、碎片)、`diary.md`(她每天 23:30 写的日记)、`memory.json`、真实发生过的对话 `transcript.jsonl
|
|
92
|
+
| `~/dot/memory/` | `memory.md`(给你看的:关于你、她的笔记、认识的人、待核实、碎片)、`diary.md`(她每天 23:30 写的日记)、`memory.json`、真实发生过的对话 `transcript.jsonl`、自动召回的索引 `recall/`、交班记录 `handoffs.jsonl`、每次整理的原始回复 `calls.jsonl` |
|
|
93
93
|
| `~/.dsh/phone/` | 模拟器、platform-tools、系统镜像、AVD(`avd/dot.avd/`);`~/.dsh/dsh-phone.log` 日志、`dsh-phone-audit.jsonl` 审计、`dsh-phone.json` 设置 |
|
|
94
94
|
| `~/.dsh/dsh-computer*` | 电脑的日志、审计、状态 |
|
|
95
|
-
| `/phone [accept\|restart\|screenshot]` · `/interlude [now\|advance\|seed\|preset <code>\|canon]` · `/dot [test\|check]` · `/dot memory [now\|diary\|handoff]` · `/dot people [approve <id>\|drop <id>\|forget <名字>]` | 各组件的状态和操作 |
|
|
95
|
+
| `/phone [accept\|restart\|screenshot]` · `/interlude [now\|advance\|seed\|preset <code>\|canon]` · `/dot [test\|check]` · `/dot memory [now\|diary\|handoff\|recall …]` · `/dot people [approve <id>\|drop <id>\|forget <名字>]` | 各组件的状态和操作 |
|
|
96
96
|
| 右侧栏 **dot** 标签页(会话顶部 ◎ 按钮) | 一页看全:手机 / 电脑 QQ / 电脑 状态、她的对话交班进度、她此刻在做什么和打算、今天的轮数 / 玩 / 去看 / 叫醒 / token、记忆用量、日记、最近两段剧本。只读,3 秒刷新;页面由 `dsh-dot/dot` 挂在 dsh 自己的 web server 上(`/dsh-dot/panel/`),不开新端口 |
|
|
97
97
|
| 安全边界 | 手机实时画面与控制(`/dsh-dot/phone/`)和 dot 面板都挂在 dsh 的 web server 上,每个请求先过 dsh 的 `connection` 服务(dsh 登录态 + Host/Origin 围栏,不信任任何转发头)——插件不自带鉴权、不开自己的端口。电脑的实时桌面同样挂在 dsh 的 web server 上(`/dsh-dot/computer/`,含 VNC 的 WebSocket),由插件带密钥中继给虚拟机网关;网关端口(`127.0.0.1:18080`)只绑本机回环且要求密钥(`SANDBOX_API_KEY`,每台机器自动生成在 `~/.dsh/dsh-computer.key`),只有插件内部持有。若要从外面看,请暴露 dsh 本身而不是这个端口 |
|
|
98
98
|
|
package/docs/ROADMAP.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# dsh-dot 总体规划(2026-10-23;「现在在哪」更新于 2026-11-
|
|
1
|
+
# dsh-dot 总体规划(2026-10-23;「现在在哪」更新于 2026-11-03,v0.12.0)
|
|
2
2
|
|
|
3
3
|
## 一句话
|
|
4
4
|
|
|
@@ -16,6 +16,7 @@ dot 是一个**在自己手机上过日子的角色**:有自己的手机、账
|
|
|
16
16
|
| 电脑(`computer`) | ✅ 可选;网关加密钥、端口转发封死、实时画面一次性票据、按操作类型审批、审计 | `DSH_DOT_COMPUTER=off` |
|
|
17
17
|
| 她电脑上的 QQ(`qq`) | ✅ NapCat 或 SnowLuma(`/qq backend`),她用手机摄像头扫码登录(`/qq qr`);只认她的号 | 10-30 真机:SnowLuma 连上、`mcp__dot-qq__*` 挂载,她自己扫码登录 |
|
|
18
18
|
| IM 通道 | ✅ QQ 连上时 dot 的提醒、幕间主动联系、她对你 QQ 消息的回答都走她的 QQ;没连上时退回 dsh-im 官方机器人。SnowLuma 下收发已实测(10-29,22 条进 38 条出);`qq_send` 对陌生人的审批只有单元测试,没有陌生人来过 | 真机 10-29 |
|
|
19
|
+
| 自动召回(`memory` 的 RAG 层) | ✅ 0.12:每次叫醒前按叫醒她的话从 transcript 取原话注入;本机 bge-small-zh 默认(首次用时装库、走 hf-mirror 下模型),SiliconFlow / DashScope 可切,`/dot memory recall …` 一条命令切换 | 公开基准 108 题:74 → 93(真实代码路径,本机后端);真机未跑 |
|
|
19
20
|
| "玩"(点赞/评论/关注) | ✅ 代码在(playIntent → 正常轮次,Moltbook 守则),11-02 起默认开、内置 App 表按手机上装的收窄;发帖/私信不做 | 真机 0 次(之前默认关) |
|
|
20
21
|
|
|
21
22
|
## 架构(不再变的部分)
|
package/docs/memory.md
CHANGED
|
@@ -52,16 +52,58 @@
|
|
|
52
52
|
|
|
53
53
|
dsh 自己的压缩要到窗口的 80%(DeepSeek 是约 84 万 token)才发生,而且摘要模板是给编程助手写的;交班让这件事不会发生在她身上。dsh-im 机器人开的会话不交班(绑定在 dsh-im 手里)。
|
|
54
54
|
|
|
55
|
+
## 自动召回(0.12 起)
|
|
56
|
+
|
|
57
|
+
画像和碎片是"整理过的"记忆;问到"三周前他到底怎么说的",整理时没写下来的细节就没了。自动召回补这一块:她每次被叫醒(你的消息、QQ 消息、手机通知、定时任务)之前,用叫醒她的那句话去她自己的 `transcript.jsonl` 里找最像的旧对话,按时间顺序作为「你记得的相关原话」放在她面前,每条带日期。画像 / 碎片 / 日记 / 交班照旧,`recall` 工具也还在(她自己按关键词翻)。
|
|
58
|
+
|
|
59
|
+
- **一轮一块**:叫醒她的话 + 她回的话 + 日期。每轮结束增量建索引,向量存 `~/dot/memory/recall/<后端>-<模型>/`(`vectors.f32` + `meta.jsonl`,不用向量库)。每个后端 + 模型各存一份,来回切换不互相覆盖、不重算。
|
|
60
|
+
- **不重复**:她当前会话里的轮次本来就在上下文里,不再取(`recallWindowTurns`);取 top 8(`recallTopK`),最多 1500 token(`recallMaxTokens`,超了先丢最不像的)。
|
|
61
|
+
- **只给她的对话**:作为动态 context 注入(dsh 放在历史之后,不破坏前缀缓存);跑腿会话看不到。
|
|
62
|
+
- **密码 / key 不进**:和碎片同一个过滤,含密码、API key 的那行不入索引、不发给云端、不注入;问话本身像密钥时不查。
|
|
63
|
+
- **后端**:
|
|
64
|
+
|
|
65
|
+
| 后端 | 模型 | 要什么 | 说明 |
|
|
66
|
+
|---|---|---|---|
|
|
67
|
+
| `local`(默认) | `BAAI/bge-small-zh-v1.5`(ONNX q8,即 `Xenova/bge-small-zh-v1.5`),不重排 | 不要 key | transformers.js 在 CPU 上跑;模型 23 MB,常驻约 0.15 GB 内存,一次查询毫秒级。推理库 `@huggingface/transformers` 有 300–480 MB,不打进插件包:第一次用时装到 `$DSH_HOME/dsh-dot-recall/`(像 QQ / 手机按需 `npx` 拉 MCP 一样),模型从 `https://hf-mirror.com` 下(`recallMirror`,也认 `HF_ENDPOINT`),缓存也在那里。装好、下好之前按关键词找,状态里写着 |
|
|
68
|
+
| `siliconflow` | `BAAI/bge-m3` + `BAAI/bge-reranker-v2-m3` 重排 | key(免费档够用) | `SILICONFLOW_API_KEY` 或 `/dot memory recall key siliconflow <key>` |
|
|
69
|
+
| `dashscope` | `qwen3.7-text-embedding` + `qwen3.7-text-rerank` 重排 | key | `DASHSCOPE_API_KEY` 或 `/dot memory recall key dashscope <key>`;工作空间域名填 `recallDashscopeUrl` |
|
|
70
|
+
| `off` | — | — | 不自动召回 |
|
|
71
|
+
|
|
72
|
+
- **命令**(不用改文件、不用重启;写进 `$DSH_HOME/dsh-memory.json`,权限 0600,优先于插件配置):
|
|
73
|
+
- `/dot memory recall`:后端、模型、是否就绪、索引了几轮、上次查询耗时和条数、镜像、key 来源(打码)
|
|
74
|
+
- `/dot memory recall backend <local|siliconflow|dashscope|off>`
|
|
75
|
+
- `/dot memory recall key <siliconflow|dashscope> <key>`(`-` 删掉)
|
|
76
|
+
- `/dot memory recall model <名字|default>`、`/dot memory recall rerank <名字|none|default>`(云端)
|
|
77
|
+
- `/dot memory recall mirror <url|default>`
|
|
78
|
+
- `/dot memory recall rebuild`:清掉当前后端的索引重建
|
|
79
|
+
- `/dot memory recall test <一句话>`:看这句话会翻出什么(带分数)
|
|
80
|
+
- 右侧 dot 面板「记忆」卡片里有一行「自动召回」。
|
|
81
|
+
|
|
82
|
+
**数据**([Ornn8/memory-system-benchmark](https://github.com/Ornn8/memory-system-benchmark) 公开示例 PUB01,108 题;答题和判分都是 deepseek-flash、关思考,判分是适配器自己的 LLM 判卷,不是基准作者的打分;适配器在 dsh-dot 仓库外):
|
|
83
|
+
|
|
84
|
+
| 配置 | 通过 |
|
|
85
|
+
|---|---|
|
|
86
|
+
| 0.11 现状(画像 + 碎片 + 交班 + `recall` 工具) | 74 / 108 |
|
|
87
|
+
| 纯检索:本机 bge-small-zh,不重排 | 85 / 108 |
|
|
88
|
+
| 纯检索:云端 bge-m3 + bge-reranker-v2-m3(SiliconFlow) | 90 / 108 |
|
|
89
|
+
| 纯检索:云端 qwen3.7 embedding + rerank(DashScope) | 89 / 108 |
|
|
90
|
+
| **0.12:现状 + 自动召回(本机 bge-small-zh,真实代码路径)** | **93 / 108**(`recallMinScore` 0)· **95 / 108**(0.45,默认) |
|
|
91
|
+
|
|
92
|
+
现状赢在"约定、改口、生日"这类题(整理过的事实),检索赢在原话细节;所以是加一层,不是换掉。
|
|
93
|
+
|
|
55
94
|
## 文件和命令
|
|
56
95
|
|
|
57
96
|
- `~/dot/memory/memory.md`:给你看的版本;`diary.md`:她的日记;`memory.json`:数据;`handoffs.jsonl`:每次交班;`calls.jsonl`:每次整理的原始回复和 token 用量。
|
|
58
97
|
- `/dot memory`:状态(第几段、当前 token、今天整理花了多少 token、距离下次整理还有几条、上次日记);`/dot memory now`:立刻整理;`/dot memory diary`:立刻整理并写日记;`/dot memory handoff`:立刻交班。
|
|
98
|
+
- `/dot memory recall …`:自动召回,见上一节。
|
|
59
99
|
- `/dot people`:认识的人、待核实、待确认的合并;`approve <p…|m…>` / `drop <p…>`:处理待核实和合并;`merge <a> <b>`:手动合并两张卡;`forget <名字>`:删掉一个人。
|
|
60
100
|
|
|
61
101
|
## 配置(`dot-memory`)
|
|
62
102
|
|
|
63
103
|
`handoffTokens` 150000 · `minTokensBetweenHandoffs` 40000 · `consolidateEveryLines` 20 · `consolidateIdleMinutes` 30 · `dailyAt` 23:30 · `fragmentBudgetTokens` 600 · `agingEveryGens` 5 · `maxFragments` 300 · `handoffTailLines` 12 · `maxTokens` 16384 · `limits: { owner: 1375, self: 2200, person: 400, impression: 40 }` · `coexist` auto · `otherMemoryServices` [hermesMemory, autoMemory, dshMemory, memoryBank]
|
|
64
104
|
|
|
105
|
+
自动召回:`recallBackend` local · `recallModel` ''(后端默认)· `recallRerankModel` ''(云端默认;`none` 不重排)· `recallMirror` ''(`HF_ENDPOINT`,否则 hf-mirror.com)· `recallInstall` true · `recallTransformersVersion` 4.3.2 · `recallTopK` 8 · `recallMaxTokens` 1500 · `recallMinScore` 0.45(本机余弦 / 云端重排分;寒暄不触发注入)· `recallCandidates` 40 · `recallWindowTurns` 0(整个当前会话不取)· `recallChunkChars` 1200 · `recallWaitMs` 3000 · `recallSiliconflowUrl` · `recallDashscopeUrl`。命令改的值存在 `$DSH_HOME/dsh-memory.json`,优先于这里。
|
|
106
|
+
|
|
65
107
|
`DSH_DOT_MEMORY=off` 整个组件不加载。
|
|
66
108
|
|
|
67
109
|
**和别的记忆插件共存**(`coexist`,环境变量 `DSH_DOT_MEMORY_COEXIST`):`auto` 时,`otherMemoryServices` 里任何一个服务存在(每次用到时用 `ctx.get` 查,所以晚启动的插件也算;cordis 没有公开的服务枚举,只能按名单)就打一行 info 日志,进入 `people-only`:人物卡、碎片、日记、交班照旧,画像的 `owner` / `self` 两栏不再进系统提示,整理时模型给的这两栏改动也丢掉,交给那个插件。`full` 强制全开;`people-only` 强制只留人物。
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-dot-life",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.2",
|
|
4
4
|
"description": "dot — a character who lives on her own phone, for DeepSeek Harness (dsh): an Android phone the plugin installs and runs, her own QQ, memory and a diary from real events, a life between conversations (幕间, after HDS-Interlude), an optional microVM computer with approval gates. One plugin, six components.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "AGPL-3.0-only",
|
package/src/dot/core.js
CHANGED
|
@@ -235,7 +235,11 @@ export function apply(ctx, config = {}) {
|
|
|
235
235
|
// restarts, archived and replaced once it grows past errandMaxTokens.
|
|
236
236
|
let errandId = cfg.errands ? (store.get('errandSession') || '') : '';
|
|
237
237
|
let errandCreating = null;
|
|
238
|
-
|
|
238
|
+
// Errands that were rotated away stay errands: their last turn/end must not make one of them her
|
|
239
|
+
// "latest session" (his QQ message then goes into an archived errand and he never gets an answer).
|
|
240
|
+
const pastErrands = new Set(store.get('pastErrands') ?? []);
|
|
241
|
+
const isErrand = (id) => Boolean(id) && (id === errandId || pastErrands.has(id));
|
|
242
|
+
if (isErrand(lastSession)) { lastSession = ''; store.set('lastSession', ''); }
|
|
239
243
|
async function ensureErrand() {
|
|
240
244
|
if (!cfg.errands) return '';
|
|
241
245
|
if (errandId) return errandId;
|
|
@@ -261,6 +265,8 @@ export function apply(ctx, config = {}) {
|
|
|
261
265
|
try { tokens = Number(meter.measure(agent.session)?.totalTokens ?? 0); } catch (error) { void error; return; }
|
|
262
266
|
if (tokens < cfg.errandMaxTokens) return;
|
|
263
267
|
if (errandId === id) { errandId = ''; store.set('errandSession', ''); }
|
|
268
|
+
pastErrands.add(id); store.set('pastErrands', [...pastErrands].slice(-20));
|
|
269
|
+
if (lastSession === id) { lastSession = ''; store.set('lastSession', ''); }
|
|
264
270
|
try { await ctx.get('workspaceRegistry')?.archiveSession?.(id); } catch (err) { logger.warn(`errand archive failed: ${err.message}`); }
|
|
265
271
|
audit('errand-rotated', { session: id, tokens });
|
|
266
272
|
}
|
|
@@ -456,7 +462,8 @@ export function apply(ctx, config = {}) {
|
|
|
456
462
|
send(lastSession, items.filter(isConversation));
|
|
457
463
|
send(e, errands);
|
|
458
464
|
};
|
|
459
|
-
|
|
465
|
+
// Items held for an errand that was rotated meanwhile go to the current errand, not the archived one.
|
|
466
|
+
const releaseHeld = async (id) => { const items = held.get(id); if (!items?.length) return; held.delete(id); deliverTracked(items, true, pastErrands.has(id) ? (await ensureErrand()) || lastSession : id); };
|
|
460
467
|
const conversationPlatform = { qq: 'qq', 'com.tencent.mobileqq': 'qq', 'com.tencent.mm': 'wechat' };
|
|
461
468
|
const wake = (n) => {
|
|
462
469
|
// Her QQ on the computer delivers the same messages as events; the phone's QQ notifications would double them.
|
|
@@ -532,7 +539,7 @@ export function apply(ctx, config = {}) {
|
|
|
532
539
|
commandCtx.effect(() => commandCtx.commands.register({
|
|
533
540
|
name: 'dot',
|
|
534
541
|
description: 'dot status; "test" sends a test message to your IM; "check" makes her pick up her phone now; "memory" / "people" show what she remembers',
|
|
535
|
-
input: { hint: '[test | check | memory [now|handoff] | people [approve|drop <id> | forget <name>]]' },
|
|
542
|
+
input: { hint: '[test | check | memory [now|diary|handoff|recall [backend|key|model|rerank|mirror|rebuild|test]] | people [approve|drop <id> | forget <name>]]' },
|
|
536
543
|
handler: async ({ rawInput }) => {
|
|
537
544
|
const arg = String(rawInput ?? '').trim();
|
|
538
545
|
// /dot memory … and /dot people … belong to dsh-dot/memory.
|
|
@@ -550,6 +557,8 @@ export function apply(ctx, config = {}) {
|
|
|
550
557
|
`computer: ${vm ? `${vm.state}${vm.publicLiveViewUrl ? ` · ${vm.publicLiveViewUrl}` : ''}` : 'not installed (optional)'} · phone: ${ph ? `${ph.state}${ph.error ? ` (${ph.error})` : ''}` : 'not installed (optional)'}`,
|
|
551
558
|
`IM: ${t.error ? `not ready (${t.error})` : `${t.name ?? t.target} via ${t.bot}`}`,
|
|
552
559
|
`scope: ${cfg.scope} · home: ${home} · im sessions seen: ${imSessions.size}`,
|
|
560
|
+
// Where his next message goes: when she "does not answer", look here first.
|
|
561
|
+
`her conversation: ${lastSession ? `${lastSession}${turns.has(lastSession) ? ' (in a turn)' : ''}` : '(none yet: say something to her in a session under home)'} · errand: ${errandId || '(none)'}`,
|
|
553
562
|
`budget: ${ctx.get('budget') ? (blocked.length ? `BLOCKED (${blocked.join(', ')})` : 'ok') : 'not installed (optional)'}`,
|
|
554
563
|
`schedule: ${ctx.get('schedule') ? 'ok' : 'not installed'}`,
|
|
555
564
|
];
|
package/src/dot/panel.js
CHANGED
|
@@ -105,6 +105,7 @@ function render(s){
|
|
|
105
105
|
h+='<span>她自己</span><span>'+m.self.entries.length+' 条 · '+m.self.chars+'/'+m.self.limit+' 字'+bar(m.self.chars,m.self.limit)+'</span>';
|
|
106
106
|
h+='<span>认识的人</span><span>'+m.people+(m.contacts?' <span class="mute">(+'+m.contacts+' 好友没说过话)</span>':'')+(m.pending?' · 待核实 '+m.pending:'')+(m.merges?' · 待确认合并 '+m.merges:'')+'</span>';
|
|
107
107
|
h+='<span>碎片</span><span>'+m.fragments+' · 距下次整理还有 '+m.sinceCursor+' 条 · 整理过 '+m.consolidations+' 次</span>';
|
|
108
|
+
if(m.recall){const r=m.recall;h+='<span>自动召回</span><span>'+esc(r.backend)+(r.model?' · '+esc(r.model):'')+(r.rerank&&r.rerank!=='none'?' + '+esc(r.rerank):'')+' · <span class="'+(r.ready||r.backend==='off'?'':'mute')+'">'+esc(r.state)+'</span> · 索引 '+fmt(r.turns)+' 轮'+(r.lastQuery?' · 上次 '+r.lastQuery.ms+' ms / '+r.lastQuery.hits+' 条':'')+'</span>';}
|
|
108
109
|
h+='</div>';
|
|
109
110
|
if(m.self.entries.length) h+='<details><summary>她自己记下的</summary><ul>'+m.self.entries.map(e=>'<li>'+esc(e)+'</li>').join('')+'</ul></details>';
|
|
110
111
|
$('memory').innerHTML=h;
|
package/src/memory/memory.js
CHANGED
|
@@ -9,19 +9,23 @@
|
|
|
9
9
|
// consolidate one model call over the transcript since the last one: bank edits, people,
|
|
10
10
|
// fragments. Every `consolidateEveryLines` lines, or after `consolidateIdleMinutes`
|
|
11
11
|
// of quiet, or at a handoff.
|
|
12
|
+
// recall before each turn, the old turns most like the message that woke her (RAG over the
|
|
13
|
+
// transcript, recall.js): local bge-small-zh by default, cloud or off by one command.
|
|
12
14
|
// handoff after dsh-whale-inurqq's: when her conversation passes `handoffTokens`, consolidate,
|
|
13
15
|
// archive it, open a new one in <home>, and inject a handoff note (what she is doing,
|
|
14
16
|
// what she promised, the last lines verbatim). Deliveries wait meanwhile.
|
|
15
17
|
// She edits her memory herself too: `memory` (Hermes-style add/replace/remove) and `recall`.
|
|
16
|
-
import { appendFileSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
18
|
+
import { appendFileSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
17
19
|
import { homedir } from 'node:os';
|
|
18
|
-
import { join, relative, isAbsolute } from 'node:path';
|
|
20
|
+
import { join, relative, isAbsolute, sep } from 'node:path';
|
|
21
|
+
import { fileURLToPath } from 'node:url';
|
|
19
22
|
import { createUserMessage } from '@deepseek-ai/dsh-llm';
|
|
20
23
|
import {
|
|
21
24
|
DEFAULT_LIMITS, approvePending, ageFragments, bankOp, bankUsage, emptyMemory, findPerson, normalizeMemory,
|
|
22
25
|
observeFragment, handleOf, renderPerson, renderStanding, search, touchPerson, updatePerson, compareFragments, rejectReason,
|
|
23
26
|
resolvePerson, setOwner, ownerOf, proposeMerge, approveMerge, mergePeople, syncFriends,
|
|
24
27
|
} from './store.js';
|
|
28
|
+
import { createRecall } from './recall.js';
|
|
25
29
|
|
|
26
30
|
export const name = 'dot-memory';
|
|
27
31
|
export const inject = ['llm', 'agentDefaultModel'];
|
|
@@ -47,9 +51,34 @@ export const defaults = {
|
|
|
47
51
|
coexist: 'auto',
|
|
48
52
|
// Names other memory plugins provide. A bare 'memory' is deliberately not here: too generic a name
|
|
49
53
|
// to risk silencing her own banks because something unrelated provides it. Add names as you meet them.
|
|
54
|
+
// Most memory plugins provide no service at all (dsh-auto-memory and friends), so the packages
|
|
55
|
+
// installed next to this one are looked at too: any whose name says "memory" counts.
|
|
50
56
|
otherMemoryServices: ['hermesMemory', 'autoMemory', 'dshMemory', 'memoryBank'],
|
|
57
|
+
// Automatic recall (recall.js). Everything here can also be changed at runtime with
|
|
58
|
+
// /dot memory recall … (saved to $DSH_HOME/dsh-memory.json, which wins over this config).
|
|
59
|
+
recallBackend: 'local', // local (bge-small-zh on this machine, no key) | siliconflow | dashscope | off
|
|
60
|
+
recallModel: '', // embedding model; '' = the backend's own (local Xenova/bge-small-zh-v1.5 · siliconflow BAAI/bge-m3 · dashscope qwen3.7-text-embedding)
|
|
61
|
+
recallRerankModel: '', // cloud only; '' = BAAI/bge-reranker-v2-m3 · qwen3.7-text-rerank; 'none' = cosine only
|
|
62
|
+
recallMirror: '', // where local models download from; '' = $HF_ENDPOINT, else https://hf-mirror.com
|
|
63
|
+
recallInstall: true, // install @huggingface/transformers into $DSH_HOME/dsh-dot-recall on first use (300–480 MB; not a package dependency)
|
|
64
|
+
recallTransformersVersion: '4.3.2',
|
|
65
|
+
recallTopK: 8, // turns injected per wake
|
|
66
|
+
recallMaxTokens: 1500, // …and at most this many tokens of them (the weakest are dropped first)
|
|
67
|
+
// Drop hits below this similarity (local: cosine; cloud: rerank score). bge-small-zh on the public
|
|
68
|
+
// sample: real questions score 0.60–0.75 at the top, small talk (在吗 / 好的 / 哈哈) stays under 0.45,
|
|
69
|
+
// so 0.45 keeps the answers and stops greetings from pulling in 8 turns. 95/108 vs 93 at 0.
|
|
70
|
+
recallMinScore: 0.45,
|
|
71
|
+
recallCandidates: 40, // cosine candidates handed to a reranker
|
|
72
|
+
recallWindowTurns: 0, // 0: skip her whole current session (dsh keeps it in the window); N: skip only the last N turns (hosts that show a window)
|
|
73
|
+
recallChunkChars: 1200, // one turn's text is clipped to this before embedding
|
|
74
|
+
recallWaitMs: 3000, // prompt assembly waits this long for a query still running, then goes on without it
|
|
75
|
+
recallSiliconflowUrl: 'https://api.siliconflow.cn/v1',
|
|
76
|
+
recallDashscopeUrl: 'https://dashscope.aliyuncs.com', // a workspace host (https://<ws>.cn-beijing.maas.aliyuncs.com) works too
|
|
51
77
|
};
|
|
52
78
|
|
|
79
|
+
// Messages that wake her and are worth remembering against: his, QQ / phone notifications, schedules.
|
|
80
|
+
const RECALL_WAKES = (src) => src.kind === 'user' || src.kind === 'schedule' || (src.kind === 'dsh-dot' && src.form === 'phone-notification');
|
|
81
|
+
|
|
53
82
|
export const HANDOFF_SOURCE = Object.freeze({ kind: 'handoff' });
|
|
54
83
|
|
|
55
84
|
const inside = (root, p) => { const r = relative(root, p); return r === '' || (!r.startsWith('..') && !isAbsolute(r)); };
|
|
@@ -206,11 +235,26 @@ export function apply(ctx, config = {}) {
|
|
|
206
235
|
const peopleOnly = () => {
|
|
207
236
|
if (cfg.coexist === 'people-only') return true;
|
|
208
237
|
if (cfg.coexist !== 'auto') return false;
|
|
209
|
-
const other = [].concat(cfg.otherMemoryServices ?? []).find((n) => { try { return ctx.get(n) != null; } catch { return false; } });
|
|
238
|
+
const other = [].concat(cfg.otherMemoryServices ?? []).find((n) => { try { return ctx.get(n) != null; } catch { return false; } }) || installedMemoryPlugin();
|
|
210
239
|
if (other && !coexistLogged) { coexistLogged = true; logger.info(`another memory service (${other}) is present: people-only mode (owner/self banks left to it)`); }
|
|
211
240
|
return Boolean(other);
|
|
212
241
|
};
|
|
213
242
|
const audit = (event, data = {}) => (ctx.get('dshComputer')?.audit ?? ctx.get('dshPhone')?.audit)?.({ kind: 'memory', event, ...data });
|
|
243
|
+
/** A memory plugin installed in the same profile (a package next to ours whose name says "memory"), or ''. Looked up once. */
|
|
244
|
+
let installedMemory;
|
|
245
|
+
function installedMemoryPlugin() {
|
|
246
|
+
if (installedMemory !== undefined) return installedMemory;
|
|
247
|
+
installedMemory = '';
|
|
248
|
+
try {
|
|
249
|
+
const here = fileURLToPath(import.meta.url);
|
|
250
|
+
const nm = here.slice(0, here.lastIndexOf(`${sep}node_modules${sep}`) + 14);
|
|
251
|
+
if (!nm.endsWith('node_modules')) return installedMemory;
|
|
252
|
+
const names = [];
|
|
253
|
+
for (const n of readdirSync(nm)) { if (n.startsWith('@')) { try { for (const m of readdirSync(join(nm, n))) names.push(`${n}/${m}`); } catch {} } else names.push(n); }
|
|
254
|
+
installedMemory = names.find((n) => /memory/i.test(n) && !/^dsh-dot(-|$)/.test(n) && !n.includes('/dsh-dot')) ?? '';
|
|
255
|
+
} catch {}
|
|
256
|
+
return installedMemory;
|
|
257
|
+
}
|
|
214
258
|
|
|
215
259
|
let memory;
|
|
216
260
|
try { memory = normalizeMemory(JSON.parse(readFileSync(files.memory, 'utf8'))); } catch (error) { void error; memory = emptyMemory(); }
|
|
@@ -248,6 +292,9 @@ export function apply(ctx, config = {}) {
|
|
|
248
292
|
try { appendFileSync(files.transcript, `${JSON.stringify(line)}\n`); } catch (err) { logger.warn(`transcript write failed: ${err.message}`); }
|
|
249
293
|
};
|
|
250
294
|
const sinceCursor = () => lines.filter((l) => l.at > memory.cursor);
|
|
295
|
+
const recall = createRecall({ cfg, dir, lines: () => lines, transcriptFile: files.transcript, logger, audit: (event, data) => audit(event, data), embedder: config.recallEmbedder });
|
|
296
|
+
// Catch the index up with what happened while dsh was down (the local backend installs on first use).
|
|
297
|
+
ctx.effect(() => { const t = setTimeout(() => { void recall.sync(); }, 20000); t.unref?.(); return () => clearTimeout(t); }, 'dsh-dot/memory: recall index');
|
|
251
298
|
|
|
252
299
|
// First run: memory starts after a conversation may already be long. Read that conversation's log
|
|
253
300
|
// once so the first consolidation (and the first handoff) has it.
|
|
@@ -305,6 +352,8 @@ export function apply(ctx, config = {}) {
|
|
|
305
352
|
else if (src.kind === 'schedule') record({ who: 'schedule', session: id, text });
|
|
306
353
|
else if (src.kind === 'dsh-dot' && src.form === 'phone-notification') record({ who: 'phone', session: id, text: text.split('\n(')[0] });
|
|
307
354
|
else if (src.kind === 'dsh-dot' && src.form === 'errand-note') record({ who: 'note', session: id, text: text.split('\n(')[0] });
|
|
355
|
+
// What woke her is the query; the result is in front of her before the model is called.
|
|
356
|
+
if (RECALL_WAKES(src) && !dot()?.isErrand?.(id)) void recall.prepare(id, src.kind === 'dsh-dot' ? text.split('\n(')[0] : text);
|
|
308
357
|
return;
|
|
309
358
|
}
|
|
310
359
|
case 'tool/call': {
|
|
@@ -326,6 +375,7 @@ export function apply(ctx, config = {}) {
|
|
|
326
375
|
turns.delete(id);
|
|
327
376
|
// Her final words count as said only when they were not a QQ turn (desktop / IM bot replies).
|
|
328
377
|
if (!t.sentViaQq && t.last && !isSilent(t.last)) record({ who: 'her', session: id, text: t.last });
|
|
378
|
+
recall.clear(id); recall.turnEnded();
|
|
329
379
|
void afterTurn(id);
|
|
330
380
|
return;
|
|
331
381
|
}
|
|
@@ -492,6 +542,28 @@ export function apply(ctx, config = {}) {
|
|
|
492
542
|
return renderStanding(memory, { budgetTokens: cfg.fragmentBudgetTokens, banks: !peopleOnly() });
|
|
493
543
|
},
|
|
494
544
|
});
|
|
545
|
+
// The turns recall found for this wake. A dynamic context: dsh puts it after the history, so the
|
|
546
|
+
// cached prefix holds. Her conversation only — errands are not her remembering.
|
|
547
|
+
scope.systemPrompt.context({
|
|
548
|
+
name: 'dot:recall',
|
|
549
|
+
order: 905,
|
|
550
|
+
text: (c) => { const s = c.agent?.session; return s && !dot()?.isErrand?.(s.id) && isConversation(s.id, s.header?.cwd) ? recall.textFor(s.id) : ''; },
|
|
551
|
+
});
|
|
552
|
+
// Contexts are read synchronously; a query still running (a cloud call, a cold model) is waited
|
|
553
|
+
// for here, bounded by recallWaitMs, and its text put into the assembly. Registered on the
|
|
554
|
+
// systemPrompt scope like the interlude's prompt trim (proven on a real dsh); without it the
|
|
555
|
+
// result shows up from the next step on.
|
|
556
|
+
scope.effect(() => scope.on('system-prompt/assemble', async (assembly, c, next) => {
|
|
557
|
+
const id = c?.agent?.session?.id;
|
|
558
|
+
const p = id && recall.pending(id);
|
|
559
|
+
if (p && Array.isArray(assembly?.contexts)) {
|
|
560
|
+
await Promise.race([p, new Promise((r) => { setTimeout(r, cfg.recallWaitMs).unref?.(); })]);
|
|
561
|
+
const entry = assembly.contexts.find((x) => x.name === 'dot:recall');
|
|
562
|
+
const text = recall.textFor(id);
|
|
563
|
+
if (entry) entry.text = text; else if (text) assembly.contexts.push({ name: 'dot:recall', text });
|
|
564
|
+
}
|
|
565
|
+
return next();
|
|
566
|
+
}), 'dsh-dot/memory: recall waits for its query');
|
|
495
567
|
});
|
|
496
568
|
|
|
497
569
|
// ── her own hands on it ──
|
|
@@ -554,6 +626,7 @@ export function apply(ctx, config = {}) {
|
|
|
554
626
|
forInterlude() { return [...memory.owner.map((e) => `关于他:${e}`), ...[...memory.fragments].sort(compareFragments).slice(0, 12).map((f) => f.text)]; },
|
|
555
627
|
/** For the interlude: her own notes — what she does these days and has learned. Her life is built around these. */
|
|
556
628
|
herLife() { return [...memory.self]; },
|
|
629
|
+
recall,
|
|
557
630
|
consolidate, handoff, status: () => ({ gen: memory.gen, owner: memory.owner.length, self: memory.self.length, people: Object.keys(memory.people).length, pending: memory.pending.length, fragments: memory.fragments.length, sinceCursor: sinceCursor().length }),
|
|
558
631
|
/** For the panel: banks with their usage, counts, today's ledger, the diary (newest first), her conversation's tokens. */
|
|
559
632
|
async panel() {
|
|
@@ -570,6 +643,7 @@ export function apply(ctx, config = {}) {
|
|
|
570
643
|
ledger: memory.ledger?.day === d ? memory.ledger : { day: d, in: 0, out: 0, calls: 0 },
|
|
571
644
|
conversation: { session: id || '', tokens: id ? await measure(id) : 0, handoffAt: cfg.handoffTokens },
|
|
572
645
|
diary, lastDiary: memory.lastDiary ?? null, dailyAt: cfg.dailyAt,
|
|
646
|
+
recall: recall.status(),
|
|
573
647
|
};
|
|
574
648
|
},
|
|
575
649
|
/** `/dot memory …` and `/dot people …` */
|
|
@@ -584,6 +658,7 @@ export function apply(ctx, config = {}) {
|
|
|
584
658
|
const people = all.filter((p) => p.tier !== 'contact').sort((a, b) => (b.lastSeen ?? 0) - (a.lastSeen ?? 0));
|
|
585
659
|
return [`people: ${people.length} (+${all.length - people.length} friends never talked to) · pending: ${memory.pending.length} · merges to confirm: ${memory.merges.length}`, ...people.slice(0, 30).map((p) => renderPerson(p)), ...(memory.merges.length ? ['merges to confirm (/dot people approve <m…>):', ...memory.merges.map((m) => ` ${m.id} · ${renderPerson(memory.people[m.a]).split('\n')[0]} + ${renderPerson(memory.people[m.b]).split('\n')[0]}:${m.why}`)] : []), ...(memory.pending.length ? ['pending (/dot people approve <id> | drop <id>):', ...memory.pending.slice(-20).map((x) => ` ${x.id} · ${x.person}:${x.text}`)] : [])].join('\n');
|
|
586
660
|
}
|
|
661
|
+
if (sub === 'recall') return recall.command(rest);
|
|
587
662
|
if (sub === 'now') { const r = await consolidate({ reason: 'manual' }); return r.ok ? `consolidated: ${JSON.stringify(r.report)}` : `not run: ${r.skipped ?? r.error}`; }
|
|
588
663
|
if (sub === 'diary') { const r = await consolidate({ reason: 'manual-diary', withDiary: true }); return r.ok ? `consolidated: ${JSON.stringify(r.report)} → ${files.diary}` : `not run: ${r.skipped ?? r.error}`; }
|
|
589
664
|
if (sub === 'handoff') { const id = dot()?.session?.(); if (!id) return 'no conversation yet'; const r = await handoff(id, await measure(id)); return r.ok ? `handed over → ${r.sessionId}` : `not run: ${r.skipped ?? r.error}`; }
|
|
@@ -593,7 +668,8 @@ export function apply(ctx, config = {}) {
|
|
|
593
668
|
const led = memory.ledger?.day === dayLabel(new Date()).slice(0, 10) ? memory.ledger : { in: 0, out: 0, calls: 0 };
|
|
594
669
|
return [`memory: 第 ${s.gen} 段 · owner ${bankUsage(memory.owner, cfg.limits.owner)} · self ${bankUsage(memory.self, cfg.limits.self)} · people ${s.people} (+${s.pending} pending) · fragments ${s.fragments} · today ${led.calls} call(s), ${led.in} in / ${led.out} out tokens · diary ${memory.lastDiary ?? 'never'} (daily at ${cfg.dailyAt})`,
|
|
595
670
|
`conversation: ${id || '(none)'} · ${tokens} tokens (handoff at ${cfg.handoffTokens}) · ${s.sinceCursor} line(s) since the last consolidation (every ${cfg.consolidateEveryLines}, or ${cfg.consolidateIdleMinutes} min quiet)`,
|
|
596
|
-
|
|
671
|
+
(await recall.command([])).split('\n').slice(0, 2).join('\n'),
|
|
672
|
+
`files: ${files.md} · ${files.diary}`, '/dot memory now | diary | handoff | recall … · /dot people [approve <id> | drop <id> | merge <a> <b> | forget <name>]'].join('\n');
|
|
597
673
|
},
|
|
598
674
|
};
|
|
599
675
|
// Her QQ (dsh-dot/qq): the owner's number makes his card the owner's, and the friend list — the
|
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
// dsh-dot/memory · recall: before each turn of hers, the old conversation turns that look most like
|
|
2
|
+
// the message that woke her, as "原话 you remember" — RAG over her own transcript.
|
|
3
|
+
//
|
|
4
|
+
// It sits next to the portrait / fragments / diary / handoff, not instead of them: on the public
|
|
5
|
+
// memory benchmark those win the "what did we agree / what did he correct / when is his birthday"
|
|
6
|
+
// questions, and verbatim recall wins "what exactly did he say about X three weeks ago". The lexical
|
|
7
|
+
// `recall` tool stays as her own way to look things up.
|
|
8
|
+
//
|
|
9
|
+
// index one chunk per turn (what woke her + what she said back, with the date), embedded once,
|
|
10
|
+
// appended to <home>/memory/recall/<backend>-<model>/{vectors.f32, meta.jsonl}. One index
|
|
11
|
+
// per backend+model, so switching back and forth never mixes vector spaces or re-embeds.
|
|
12
|
+
// query the waking message (owner, QQ, notification, schedule) → top-k by cosine (cloud backends
|
|
13
|
+
// rerank the top candidates), minus what is already in her context window, in time order.
|
|
14
|
+
// backends local — bge-small-zh-v1.5 (ONNX q8) via transformers.js on the CPU, no key. The library
|
|
15
|
+
// is 300–480 MB, so it is not a dependency: it is installed into $DSH_HOME on first
|
|
16
|
+
// use (the way qq/phone pull their MCP servers with npx), models come from an HF
|
|
17
|
+
// mirror. Until it is there, queries fall back to lexical search (said in status).
|
|
18
|
+
// siliconflow / dashscope — cloud embedding + rerank, key from /dot memory recall key or env.
|
|
19
|
+
// off.
|
|
20
|
+
// Passwords and keys never reach the index, the cloud, or her prompt (the same filter as fragments).
|
|
21
|
+
import { spawn } from 'node:child_process';
|
|
22
|
+
import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
23
|
+
import { createRequire } from 'node:module';
|
|
24
|
+
import { homedir } from 'node:os';
|
|
25
|
+
import { dirname, join } from 'node:path';
|
|
26
|
+
import { pathToFileURL } from 'node:url';
|
|
27
|
+
import { estimateTokens, rejectReason, search } from './store.js';
|
|
28
|
+
|
|
29
|
+
export const BACKENDS = ['local', 'siliconflow', 'dashscope', 'off'];
|
|
30
|
+
const MODELS = {
|
|
31
|
+
local: { embed: 'Xenova/bge-small-zh-v1.5', rerank: 'none' },
|
|
32
|
+
siliconflow: { embed: 'BAAI/bge-m3', rerank: 'BAAI/bge-reranker-v2-m3' },
|
|
33
|
+
dashscope: { embed: 'qwen3.7-text-embedding', rerank: 'qwen3.7-text-rerank' },
|
|
34
|
+
};
|
|
35
|
+
const KEY_ENV = { siliconflow: 'SILICONFLOW_API_KEY', dashscope: 'DASHSCOPE_API_KEY' };
|
|
36
|
+
// transformers.js needs the ONNX export; people will type the BAAI name from the model card.
|
|
37
|
+
const ALIASES = { 'BAAI/bge-small-zh-v1.5': 'Xenova/bge-small-zh-v1.5', 'BAAI/bge-m3': 'Xenova/bge-m3', 'intfloat/multilingual-e5-small': 'Xenova/multilingual-e5-small' };
|
|
38
|
+
// Per-model conventions from each model card (pooling, query/passage prefixes).
|
|
39
|
+
const SPECS = {
|
|
40
|
+
'Xenova/bge-small-zh-v1.5': { pooling: 'cls', query: '为这个句子生成表示以用于检索相关文章:', passage: '' },
|
|
41
|
+
'Xenova/bge-m3': { pooling: 'cls', query: '', passage: '' },
|
|
42
|
+
'Xenova/multilingual-e5-small': { pooling: 'mean', query: 'query: ', passage: 'passage: ' },
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
const dshHome = () => process.env.DSH_HOME || join(homedir(), '.dsh');
|
|
46
|
+
export const settingsFile = () => join(dshHome(), 'dsh-memory.json');
|
|
47
|
+
const readSettings = () => { try { return existsSync(settingsFile()) ? JSON.parse(readFileSync(settingsFile(), 'utf8')) : {}; } catch { return {}; } };
|
|
48
|
+
// Holds API keys: owner-only, like dsh-qq.json.
|
|
49
|
+
function writeRecallSettings(patch) {
|
|
50
|
+
const all = readSettings();
|
|
51
|
+
const next = { ...all, recall: { ...(all.recall ?? {}), ...patch } };
|
|
52
|
+
mkdirSync(dirname(settingsFile()), { recursive: true });
|
|
53
|
+
writeFileSync(settingsFile(), JSON.stringify(next, null, 2), { mode: 0o600 });
|
|
54
|
+
try { chmodSync(settingsFile(), 0o600); } catch {}
|
|
55
|
+
return next.recall;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const WHO = { owner: '他', her: '我', phone: '[手机]', note: '[我刚做的]', schedule: '[定时任务]' };
|
|
59
|
+
const p2 = (n) => String(n).padStart(2, '0');
|
|
60
|
+
const dateStamp = (ms) => { const d = new Date(ms); return `${d.getFullYear()}-${p2(d.getMonth() + 1)}-${p2(d.getDate())} ${p2(d.getHours())}:${p2(d.getMinutes())}`; };
|
|
61
|
+
const clip = (s, n) => (s.length > n ? `${s.slice(0, n)}…` : s);
|
|
62
|
+
const slug = (s) => String(s).replace(/[^A-Za-z0-9._-]+/g, '_');
|
|
63
|
+
const mask = (k) => (k ? `${k.slice(0, 3)}…${k.slice(-4)}` : '');
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Transcript lines → one chunk per turn: whatever woke her, then what she said back. A new chunk
|
|
67
|
+
* starts when something comes in after she has spoken, or the session changes. Secret lines are
|
|
68
|
+
* dropped here, so they are neither embedded (nor sent to a cloud API) nor shown.
|
|
69
|
+
*/
|
|
70
|
+
export function chunkTurns(lines, maxChars = 1200) {
|
|
71
|
+
const out = [];
|
|
72
|
+
let cur = null;
|
|
73
|
+
for (const l of lines) {
|
|
74
|
+
const text = String(l?.text ?? '').trim();
|
|
75
|
+
const incoming = l?.who !== 'her';
|
|
76
|
+
if (!cur || cur.session !== l.session || (incoming && cur.replied)) { cur = { at: l.at, end: l.at, session: l.session ?? '', lines: [], replied: false }; out.push(cur); }
|
|
77
|
+
cur.end = l.at;
|
|
78
|
+
if (!incoming) cur.replied = true;
|
|
79
|
+
if (!text || rejectReason(text) === 'secret') continue;
|
|
80
|
+
const to = l.who === 'her' && l.to && l.to !== 'owner' ? `(发给 ${l.to})` : '';
|
|
81
|
+
cur.lines.push(`${WHO[l.who] ?? l.who}${to}:${text.replace(/\s*\n+\s*/g, ' / ')}`);
|
|
82
|
+
}
|
|
83
|
+
return out.filter((c) => c.lines.length).map((c) => ({ at: c.at, end: c.end, session: c.session, text: clip(`[${dateStamp(c.at)}] ${c.lines.join('\n')}`, maxChars) }));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const dot = (a, b) => { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s; };
|
|
87
|
+
const norm = (v) => { const n = Math.sqrt(dot(v, v)) || 1; const o = new Float32Array(v.length); for (let i = 0; i < v.length; i++) o[i] = v[i] / n; return o; };
|
|
88
|
+
|
|
89
|
+
/** One index on disk: vectors.f32 (n × dim, normalized) + meta.jsonl (one line per chunk). */
|
|
90
|
+
export function openIndex(dir) {
|
|
91
|
+
const f = { vec: join(dir, 'vectors.f32'), meta: join(dir, 'meta.jsonl') };
|
|
92
|
+
let meta = [], vecs = [], dim = 0;
|
|
93
|
+
try {
|
|
94
|
+
meta = readFileSync(f.meta, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
|
|
95
|
+
dim = meta[0]?.dim ?? 0;
|
|
96
|
+
const buf = dim ? readFileSync(f.vec) : Buffer.alloc(0);
|
|
97
|
+
const n = Math.min(meta.length, Math.floor(buf.length / (dim * 4)));
|
|
98
|
+
const all = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + n * dim * 4));
|
|
99
|
+
for (let i = 0; i < n; i++) vecs.push(all.subarray(i * dim, (i + 1) * dim));
|
|
100
|
+
// A crash between the two appends leaves them uneven: keep the common prefix.
|
|
101
|
+
if (n !== meta.length || buf.length !== n * dim * 4) { meta = meta.slice(0, n); rewrite(); }
|
|
102
|
+
} catch { meta = []; vecs = []; }
|
|
103
|
+
function rewrite() {
|
|
104
|
+
mkdirSync(dir, { recursive: true });
|
|
105
|
+
writeFileSync(f.meta, meta.map((m) => JSON.stringify(m)).join('\n') + (meta.length ? '\n' : ''));
|
|
106
|
+
const all = new Float32Array(vecs.length * dim); vecs.forEach((v, i) => all.set(v, i * dim));
|
|
107
|
+
writeFileSync(f.vec, Buffer.from(all.buffer));
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
dir,
|
|
111
|
+
get size() { return meta.length; },
|
|
112
|
+
get end() { return meta.at(-1)?.end ?? 0; },
|
|
113
|
+
meta: () => meta,
|
|
114
|
+
vecs: () => vecs,
|
|
115
|
+
add(chunks, vectors) {
|
|
116
|
+
if (!chunks.length) return;
|
|
117
|
+
mkdirSync(dir, { recursive: true });
|
|
118
|
+
const vs = vectors.map(norm);
|
|
119
|
+
dim ||= vs[0].length;
|
|
120
|
+
if (vs.some((v) => v.length !== dim)) throw new Error(`embedding size changed (${dim} → ${vs.find((v) => v.length !== dim).length}): /dot memory recall rebuild`);
|
|
121
|
+
const rows = chunks.map((c) => ({ at: c.at, end: c.end, session: c.session, text: c.text, dim }));
|
|
122
|
+
const all = new Float32Array(vs.length * dim); vs.forEach((v, i) => all.set(v, i * dim));
|
|
123
|
+
appendFileSync(f.vec, Buffer.from(all.buffer));
|
|
124
|
+
appendFileSync(f.meta, rows.map((r) => JSON.stringify(r)).join('\n') + '\n');
|
|
125
|
+
meta.push(...rows); vecs.push(...vs);
|
|
126
|
+
},
|
|
127
|
+
clear() { meta = []; vecs = []; dim = 0; rmSync(f.meta, { force: true }); rmSync(f.vec, { force: true }); },
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ── backends ──
|
|
132
|
+
|
|
133
|
+
async function post(url, key, body, { tries = 3, timeoutMs = 30000 } = {}) {
|
|
134
|
+
for (let attempt = 0; ; attempt++) {
|
|
135
|
+
const res = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(timeoutMs) });
|
|
136
|
+
if (res.ok) return res.json();
|
|
137
|
+
if (attempt < tries && (res.status === 429 || res.status >= 500)) { await new Promise((r) => setTimeout(r, 1500 * (attempt + 1))); continue; }
|
|
138
|
+
throw new Error(`HTTP ${res.status} from ${new URL(url).host}: ${(await res.text()).slice(0, 200)}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** OpenAI-compatible embeddings + SiliconFlow/Jina-style or DashScope-native rerank. */
|
|
143
|
+
function cloudEmbedder(backend, { key, model, rerank, siliconflowUrl, dashscopeUrl }) {
|
|
144
|
+
const ds = backend === 'dashscope';
|
|
145
|
+
const base = (ds ? `${dashscopeUrl.replace(/\/$/, '')}/compatible-mode/v1` : siliconflowUrl.replace(/\/$/, ''));
|
|
146
|
+
const batch = ds ? 10 : 32;
|
|
147
|
+
return {
|
|
148
|
+
async embed(texts) {
|
|
149
|
+
const out = [];
|
|
150
|
+
for (let i = 0; i < texts.length; i += batch) {
|
|
151
|
+
const j = await post(`${base}/embeddings`, key, { model, input: texts.slice(i, i + batch), encoding_format: 'float' });
|
|
152
|
+
for (const d of [...j.data].sort((a, b) => a.index - b.index)) out.push(Float32Array.from(d.embedding));
|
|
153
|
+
}
|
|
154
|
+
return out;
|
|
155
|
+
},
|
|
156
|
+
rerank: rerank && rerank !== 'none' ? async (query, docs, topN) => {
|
|
157
|
+
if (ds) {
|
|
158
|
+
const j = await post(`${dashscopeUrl.replace(/\/$/, '')}/api/v1/services/rerank/text-rerank/text-rerank`, key, { model: rerank, input: { query, documents: docs }, parameters: { top_n: topN, return_documents: false } });
|
|
159
|
+
return j.output.results.map((r) => ({ index: r.index, score: r.relevance_score }));
|
|
160
|
+
}
|
|
161
|
+
const j = await post(`${base}/rerank`, key, { model: rerank, query, documents: docs, top_n: topN, return_documents: false });
|
|
162
|
+
return j.results.map((r) => ({ index: r.index, score: r.relevance_score }));
|
|
163
|
+
} : null,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* transformers.js, from wherever it already is (installed next to this plugin) or from
|
|
169
|
+
* $DSH_HOME/dsh-dot-recall, installing it there on first use. Never a dependency of the plugin.
|
|
170
|
+
*/
|
|
171
|
+
export const libDir = () => join(dshHome(), 'dsh-dot-recall');
|
|
172
|
+
function resolveTransformers() {
|
|
173
|
+
for (const from of [import.meta.url, pathToFileURL(join(libDir(), 'package.json')).href]) {
|
|
174
|
+
try { return createRequire(from).resolve('@huggingface/transformers'); } catch { /* next */ }
|
|
175
|
+
}
|
|
176
|
+
return '';
|
|
177
|
+
}
|
|
178
|
+
let installing = null;
|
|
179
|
+
function installTransformers(version, log) {
|
|
180
|
+
installing ??= new Promise((resolve, reject) => {
|
|
181
|
+
const dir = libDir();
|
|
182
|
+
mkdirSync(dir, { recursive: true });
|
|
183
|
+
if (!existsSync(join(dir, 'package.json'))) writeFileSync(join(dir, 'package.json'), '{ "private": true, "description": "dsh-dot memory recall: @huggingface/transformers, installed on first use" }\n');
|
|
184
|
+
const win = process.platform === 'win32';
|
|
185
|
+
log(`installing @huggingface/transformers@${version} into ${dir} (300–480 MB, once)`);
|
|
186
|
+
const child = spawn(win ? 'npm.cmd' : 'npm', ['install', '--no-audit', '--no-fund', '--omit=dev', `@huggingface/transformers@${version}`], { cwd: dir, shell: win, stdio: ['ignore', 'pipe', 'pipe'], env: process.env });
|
|
187
|
+
let tail = '';
|
|
188
|
+
const keep = (d) => { tail = (tail + d).slice(-2000); try { appendFileSync(join(dir, 'install.log'), d); } catch {} };
|
|
189
|
+
child.stdout.on('data', keep); child.stderr.on('data', keep);
|
|
190
|
+
child.on('error', (err) => reject(new Error(`npm not runnable: ${err.message}`)));
|
|
191
|
+
child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`npm install exited ${code}: ${tail.trim().split('\n').at(-1) ?? ''}`))));
|
|
192
|
+
}).finally(() => { installing = null; });
|
|
193
|
+
return installing;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
let tf = null;
|
|
197
|
+
async function localEmbedder({ model, mirror, install, version, log, onPhase }) {
|
|
198
|
+
let entry = resolveTransformers();
|
|
199
|
+
if (!entry) {
|
|
200
|
+
if (!install) throw new Error('@huggingface/transformers is not installed (recallInstall is off)');
|
|
201
|
+
onPhase('installing');
|
|
202
|
+
await installTransformers(version, log);
|
|
203
|
+
entry = resolveTransformers();
|
|
204
|
+
if (!entry) throw new Error('installed, but @huggingface/transformers still cannot be resolved');
|
|
205
|
+
}
|
|
206
|
+
tf ??= await import(pathToFileURL(entry).href);
|
|
207
|
+
// Models are fetched once into $DSH_HOME, from the mirror (hf-mirror.com unless set otherwise).
|
|
208
|
+
tf.env.cacheDir = join(libDir(), 'models');
|
|
209
|
+
tf.env.allowLocalModels = false;
|
|
210
|
+
tf.env.remoteHost = `${mirror.replace(/\/+$/, '')}/`;
|
|
211
|
+
onPhase('loading');
|
|
212
|
+
const spec = SPECS[model] ?? { pooling: 'mean', query: '', passage: '' };
|
|
213
|
+
const ex = await tf.pipeline('feature-extraction', model, { dtype: 'q8', device: 'cpu' });
|
|
214
|
+
return {
|
|
215
|
+
async embed(texts, role = 'passage') {
|
|
216
|
+
const out = [];
|
|
217
|
+
for (const t of texts) out.push(Float32Array.from((await ex((spec[role] ?? '') + t, { pooling: spec.pooling, normalize: true, truncation: true })).data));
|
|
218
|
+
return out;
|
|
219
|
+
},
|
|
220
|
+
rerank: null,
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// ── the recall ──
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* opts: { cfg (memory config: recall* keys), dir (<home>/memory), lines () → in-memory transcript
|
|
228
|
+
* lines, transcriptFile, logger, audit, embedder? (backend, spec) → { embed, rerank } — tests/hosts }
|
|
229
|
+
*/
|
|
230
|
+
export function createRecall({ cfg, dir, lines, transcriptFile, logger, audit = () => {}, embedder: makeEmbedder }) {
|
|
231
|
+
const saved = () => readSettings().recall ?? {};
|
|
232
|
+
// Command-set values (dsh-memory.json) win over plugin config, so switching needs no restart or file edit.
|
|
233
|
+
const backend = () => { const b = saved().backend ?? cfg.recallBackend; return BACKENDS.includes(b) ? b : 'local'; };
|
|
234
|
+
const pick = (field, b, def) => saved()[field]?.[b] || (b === cfg.recallBackend && cfg[def]) || MODELS[b]?.[field === 'model' ? 'embed' : 'rerank'] || '';
|
|
235
|
+
const model = (b = backend()) => { const m = pick('model', b, 'recallModel'); return b === 'local' ? (ALIASES[m] ?? m) : m; };
|
|
236
|
+
const rerankModel = (b = backend()) => (b === 'local' ? 'none' : pick('rerank', b, 'recallRerankModel'));
|
|
237
|
+
const keyOf = (b) => { const k = saved().keys?.[b]; if (k) return { key: k, from: 'dsh-memory.json' }; const e = process.env[KEY_ENV[b]]; return e ? { key: e, from: KEY_ENV[b] } : { key: '', from: '' }; };
|
|
238
|
+
const mirror = () => saved().mirror || cfg.recallMirror || process.env.HF_ENDPOINT || 'https://hf-mirror.com';
|
|
239
|
+
|
|
240
|
+
const indexes = new Map(); // `${backend}-${model}` → index
|
|
241
|
+
const index = (b = backend()) => { const id = `${b}-${slug(model(b))}`; if (!indexes.has(id)) indexes.set(id, openIndex(join(dir, 'recall', id))); return indexes.get(id); };
|
|
242
|
+
|
|
243
|
+
// Embedder per backend + model (+ mirror / key), loaded lazily. The local one can take minutes the
|
|
244
|
+
// first time (install, download): nothing waits for that — queries go lexical meanwhile, and
|
|
245
|
+
// indexing resumes once it is ready. A failure is retried after 10 minutes, or after a command.
|
|
246
|
+
const emb = new Map(); // id → { phase, error, failedAt, ready, impl }
|
|
247
|
+
const embId = (b) => `${b}|${model(b)}|${rerankModel(b)}|${b === 'local' ? mirror() : keyOf(b).key}`;
|
|
248
|
+
function current(b = backend()) {
|
|
249
|
+
if (b === 'off') return null;
|
|
250
|
+
const id = embId(b);
|
|
251
|
+
const old = emb.get(id);
|
|
252
|
+
if (old && !(old.phase === 'error' && Date.now() - old.failedAt > 10 * 60000)) return old;
|
|
253
|
+
const e = { phase: 'starting', error: '', failedAt: 0, impl: null };
|
|
254
|
+
const spec = { model: model(b), rerank: rerankModel(b), mirror: mirror(), key: keyOf(b).key, siliconflowUrl: cfg.recallSiliconflowUrl, dashscopeUrl: cfg.recallDashscopeUrl };
|
|
255
|
+
e.ready = (async () => {
|
|
256
|
+
if (makeEmbedder) return makeEmbedder(b, spec);
|
|
257
|
+
if (b === 'local') return localEmbedder({ ...spec, install: cfg.recallInstall, version: cfg.recallTransformersVersion, log: (m) => logger.info(m), onPhase: (p) => { e.phase = p; } });
|
|
258
|
+
if (!spec.key) throw new Error(`no key: /dot memory recall key ${b} <key>, or set ${KEY_ENV[b]}`);
|
|
259
|
+
return cloudEmbedder(b, spec);
|
|
260
|
+
})().then((impl) => { e.impl = impl; e.phase = 'ready'; logger.info(`recall: ${b} ${spec.model} ready`); return impl; }, (err) => {
|
|
261
|
+
Object.assign(e, { phase: 'error', error: err.message, failedAt: Date.now() });
|
|
262
|
+
logger.warn(`recall: ${b} not available (${err.message}); lexical fallback`); audit('recall-unavailable', { backend: b, error: clip(err.message, 200) });
|
|
263
|
+
return null;
|
|
264
|
+
});
|
|
265
|
+
emb.set(id, e);
|
|
266
|
+
return e;
|
|
267
|
+
}
|
|
268
|
+
const forgetFailures = () => { for (const [k, v] of emb) if (v.phase === 'error') emb.delete(k); };
|
|
269
|
+
/** The embedder if it is ready within `waitMs`, else null. */
|
|
270
|
+
async function ready(b = backend(), waitMs = Infinity) {
|
|
271
|
+
const e = current(b);
|
|
272
|
+
if (!e || e.phase === 'error') return null;
|
|
273
|
+
if (e.phase === 'ready') return e.impl;
|
|
274
|
+
return waitMs === Infinity ? e.ready : Promise.race([e.ready, new Promise((r) => { setTimeout(r, waitMs, null).unref?.(); })]);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// ── indexing: up to the end of the last finished turn, incrementally; queued so runs never overlap ──
|
|
278
|
+
let completeThrough = lines().at(-1)?.at ?? 0; // at boot every line on disk belongs to a finished turn
|
|
279
|
+
let chain = Promise.resolve();
|
|
280
|
+
function sync() {
|
|
281
|
+
chain = chain.then(async () => {
|
|
282
|
+
const b = backend();
|
|
283
|
+
const e = current(b);
|
|
284
|
+
if (!e || e.phase === 'error') return;
|
|
285
|
+
if (e.phase !== 'ready') { if (!e.resume) { e.resume = true; void e.ready.then((x) => x && sync()); } return; }
|
|
286
|
+
const ix = index(b);
|
|
287
|
+
let src = lines();
|
|
288
|
+
// The in-memory transcript is trimmed; a new or rebuilt index reads the whole file.
|
|
289
|
+
if (!src.length || ix.end < src[0].at) { try { src = readFileSync(transcriptFile, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); } catch { /* none yet */ } }
|
|
290
|
+
const chunks = chunkTurns(src.filter((l) => l.at > ix.end && l.at <= completeThrough), cfg.recallChunkChars);
|
|
291
|
+
for (let i = 0; i < chunks.length; i += 32) {
|
|
292
|
+
const part = chunks.slice(i, i + 32);
|
|
293
|
+
ix.add(part, await e.impl.embed(part.map((c) => c.text), 'passage'));
|
|
294
|
+
}
|
|
295
|
+
if (chunks.length > 8) logger.info(`recall: indexed ${chunks.length} turn(s) (${b}, ${ix.size} total)`);
|
|
296
|
+
}).catch((err) => { logger.warn(`recall indexing failed: ${err.message}`); audit('recall-index-failed', { error: clip(err.message, 200) }); });
|
|
297
|
+
return chain;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** What to inject for `query` in `session`: { text, hits, mode, ms } (text '' when nothing). */
|
|
301
|
+
let last = null;
|
|
302
|
+
async function query(q, { session = '', record = true } = {}) {
|
|
303
|
+
const t0 = performance.now();
|
|
304
|
+
const b = backend();
|
|
305
|
+
const res = { text: '', hits: [], mode: b, ms: 0 };
|
|
306
|
+
q = clip(String(q ?? '').trim(), 500);
|
|
307
|
+
// Off, nothing to search for, or a secret (which must not go to a cloud API either).
|
|
308
|
+
if (b === 'off' || !q || rejectReason(q) === 'secret') return res;
|
|
309
|
+
const impl = await ready(b, 50);
|
|
310
|
+
if (impl) await sync();
|
|
311
|
+
let picked;
|
|
312
|
+
const N = Number(cfg.recallWindowTurns) || 0;
|
|
313
|
+
// What she already sees: her whole current session (dsh keeps it in the window until the
|
|
314
|
+
// handoff), or — for a host that shows only the last N turns — those N.
|
|
315
|
+
const outside = (all) => (N > 0 ? all.slice(0, Math.max(0, all.length - N)) : all.filter((c) => c.session !== session));
|
|
316
|
+
if (impl) {
|
|
317
|
+
try {
|
|
318
|
+
const ix = index(b);
|
|
319
|
+
const pool = outside(ix.meta().map((m, i) => ({ ...m, i })));
|
|
320
|
+
picked = [];
|
|
321
|
+
if (pool.length) {
|
|
322
|
+
const [qv] = (await impl.embed([q], 'query')).map(norm);
|
|
323
|
+
const vecs = ix.vecs();
|
|
324
|
+
let cand = pool.map((c) => ({ c, s: dot(qv, vecs[c.i]) })).sort((x, y) => y.s - x.s);
|
|
325
|
+
if (impl.rerank) {
|
|
326
|
+
cand = cand.slice(0, cfg.recallCandidates);
|
|
327
|
+
const r = await impl.rerank(q, cand.map((x) => x.c.text), Math.min(cfg.recallTopK, cand.length));
|
|
328
|
+
picked = r.map((x) => ({ c: cand[x.index].c, s: x.score }));
|
|
329
|
+
} else picked = cand.slice(0, cfg.recallTopK);
|
|
330
|
+
picked = picked.filter((x) => x.s >= (Number(cfg.recallMinScore) || -Infinity));
|
|
331
|
+
}
|
|
332
|
+
} catch (err) { logger.warn(`recall query failed (${err.message}); lexical this time`); picked = null; }
|
|
333
|
+
}
|
|
334
|
+
if (!picked) {
|
|
335
|
+
// Not ready (installing, downloading, no key, failed, a cloud hiccup): the same turns, found by words.
|
|
336
|
+
res.mode = `${b}→lexical`;
|
|
337
|
+
const all = outside(chunkTurns(lines(), cfg.recallChunkChars));
|
|
338
|
+
picked = search(all, q, cfg.recallTopK).map((c, i) => ({ c, s: 1 - i / 100 }));
|
|
339
|
+
}
|
|
340
|
+
// Fit the budget by dropping the weakest, then show in time order.
|
|
341
|
+
const budget = Number(cfg.recallMaxTokens) || Infinity;
|
|
342
|
+
picked.sort((x, y) => y.s - x.s);
|
|
343
|
+
let used = 0;
|
|
344
|
+
picked = picked.filter((x) => { const n = estimateTokens(x.c.text); if (used + n > budget) return false; used += n; return true; });
|
|
345
|
+
picked.sort((x, y) => x.c.at - y.c.at);
|
|
346
|
+
res.hits = picked.map((x) => ({ at: x.c.at, score: Number(x.s.toFixed?.(3) ?? x.s), text: x.c.text }));
|
|
347
|
+
res.text = picked.length ? ['你记得的相关原话(从以前的聊天里自动翻出来的,按时间;不是新消息,用得上就用,用不上就当没看见,也别提"翻到"):', ...picked.map((x) => x.c.text)].join('\n\n') : '';
|
|
348
|
+
res.ms = Math.round(performance.now() - t0);
|
|
349
|
+
if (record) last = { at: Date.now(), ms: res.ms, mode: res.mode, hits: res.hits.length, query: clip(q, 40) };
|
|
350
|
+
return res;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// ── per-session result for the prompt: computed when the waking message arrives ──
|
|
354
|
+
const results = new Map(); // session → { promise, text }
|
|
355
|
+
function prepare(session, text) {
|
|
356
|
+
const r = { text: '', done: false };
|
|
357
|
+
r.promise = query(text, { session }).then((x) => { r.text = x.text; return x; }, (err) => { logger.warn(`recall query failed: ${err.message}`); return { text: '' }; }).finally(() => { r.done = true; });
|
|
358
|
+
results.set(session, r);
|
|
359
|
+
return r.promise;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function status() {
|
|
363
|
+
const b = backend();
|
|
364
|
+
const ix = b === 'off' ? null : index(b);
|
|
365
|
+
const e = b === 'off' ? null : emb.get(embId(b));
|
|
366
|
+
const k = KEY_ENV[b] ? keyOf(b) : null;
|
|
367
|
+
const state = b === 'off' ? 'off' : !e ? 'not started' : e.phase === 'ready' ? 'ready' : e.phase === 'error' ? `unavailable: ${e.error} → lexical fallback` : `${e.phase} → lexical fallback meanwhile`;
|
|
368
|
+
return { backend: b, model: b === 'off' ? '' : model(b), rerank: b === 'off' ? '' : rerankModel(b), state, ready: e?.phase === 'ready', turns: ix?.size ?? 0, indexedThrough: ix?.end ?? 0, lastQuery: last, mirror: b === 'local' ? mirror() : '', key: k ? (k.key ? `${mask(k.key)} (${k.from})` : 'none') : '', dir: ix?.dir ?? '' };
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
async function command(args) {
|
|
372
|
+
const [verb, a, ...rest] = args;
|
|
373
|
+
const fmt = () => {
|
|
374
|
+
const s = status();
|
|
375
|
+
const lq = s.lastQuery ? `last query ${s.lastQuery.ms} ms, ${s.lastQuery.hits} hit(s) via ${s.lastQuery.mode}` : 'no query yet';
|
|
376
|
+
return [`recall: ${s.backend}${s.model ? ` · ${s.model}` : ''}${s.rerank && s.rerank !== 'none' ? ` + ${s.rerank}` : ''} · ${s.state}`,
|
|
377
|
+
`index: ${s.turns} turn(s)${s.indexedThrough ? ` through ${dateStamp(s.indexedThrough)}` : ''} · ${lq} · top ${cfg.recallTopK}, ≤${cfg.recallMaxTokens} tokens${s.mirror ? ` · mirror ${s.mirror}` : ''}${s.key ? ` · key ${s.key}` : ''}`,
|
|
378
|
+
'/dot memory recall backend <local|siliconflow|dashscope|off> | key <backend> <key> | model <name> | rerank <name|none> | mirror <url> | rebuild | test <一句话>'].join('\n');
|
|
379
|
+
};
|
|
380
|
+
if (!verb) return fmt();
|
|
381
|
+
if (verb === 'backend') {
|
|
382
|
+
if (!BACKENDS.includes(a)) return `backend: one of ${BACKENDS.join(', ')}`;
|
|
383
|
+
writeRecallSettings({ backend: a }); forgetFailures(); audit('recall-backend', { backend: a });
|
|
384
|
+
void sync();
|
|
385
|
+
return `${fmt()}\n(switched; each backend keeps its own index, built in the background)`;
|
|
386
|
+
}
|
|
387
|
+
if (verb === 'key') {
|
|
388
|
+
if (!KEY_ENV[a] || !rest[0]) return 'usage: /dot memory recall key <siliconflow|dashscope> <key> ("-" removes it)';
|
|
389
|
+
const keys = { ...(saved().keys ?? {}) };
|
|
390
|
+
if (rest[0] === '-') delete keys[a]; else keys[a] = rest[0];
|
|
391
|
+
writeRecallSettings({ keys }); forgetFailures(); audit('recall-key', { backend: a, set: rest[0] !== '-' });
|
|
392
|
+
if (backend() === a) void sync();
|
|
393
|
+
return `${a} key ${rest[0] === '-' ? 'removed' : `saved (${mask(rest[0])}) to ${settingsFile()}`}\n${fmt()}`;
|
|
394
|
+
}
|
|
395
|
+
if (verb === 'model' || verb === 'rerank') {
|
|
396
|
+
const b = backend();
|
|
397
|
+
if (b === 'off') return 'recall is off: /dot memory recall backend <…> first';
|
|
398
|
+
if (verb === 'rerank' && b === 'local') return 'local has no reranker (it scored 85/108 without one); rerank is for siliconflow / dashscope';
|
|
399
|
+
if (!a) return fmt();
|
|
400
|
+
const field = verb;
|
|
401
|
+
const cur = { ...(saved()[field] ?? {}) };
|
|
402
|
+
if (a === 'default') delete cur[b]; else cur[b] = a;
|
|
403
|
+
writeRecallSettings({ [field]: cur }); forgetFailures(); audit('recall-model', { backend: b, [field]: a });
|
|
404
|
+
void sync();
|
|
405
|
+
return `${fmt()}\n(a new model gets its own index, built in the background; the old one is kept)`;
|
|
406
|
+
}
|
|
407
|
+
if (verb === 'mirror') {
|
|
408
|
+
if (!a) return `mirror: ${mirror()}`;
|
|
409
|
+
if (a !== 'default' && !/^https?:\/\/\S+$/.test(a)) return 'mirror: an http(s) URL, or "default"';
|
|
410
|
+
writeRecallSettings({ mirror: a === 'default' ? '' : a }); forgetFailures();
|
|
411
|
+
void sync();
|
|
412
|
+
return `mirror: ${mirror()} (used for the next model download)`;
|
|
413
|
+
}
|
|
414
|
+
if (verb === 'rebuild') {
|
|
415
|
+
if (backend() === 'off') return 'recall is off';
|
|
416
|
+
index().clear(); forgetFailures(); audit('recall-rebuild', { backend: backend() });
|
|
417
|
+
await sync();
|
|
418
|
+
return `rebuilt\n${fmt()}`;
|
|
419
|
+
}
|
|
420
|
+
if (verb === 'test') {
|
|
421
|
+
const q = [a, ...rest].join(' ');
|
|
422
|
+
if (!q) return 'usage: /dot memory recall test <一句话>';
|
|
423
|
+
// As if this came in now from a session she is not in: nothing is excluded as "in the window".
|
|
424
|
+
const r = await query(q, { session: '\u0000test', record: false });
|
|
425
|
+
return `${r.mode} · ${r.ms} ms · ${r.hits.length} hit(s)\n\n${r.hits.map((h) => `(${h.score}) ${h.text}`).join('\n\n') || '(没翻到)'}`;
|
|
426
|
+
}
|
|
427
|
+
return `unknown: recall ${verb}\n${fmt()}`;
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
return {
|
|
431
|
+
sync, query, prepare, status, command,
|
|
432
|
+
/** Wait until the backend is usable (install / download / key check) and the index caught up. */
|
|
433
|
+
async warm() { const ok = Boolean(await ready()); await sync(); return ok; },
|
|
434
|
+
/** A turn of hers ended: every transcript line so far belongs to a finished turn. */
|
|
435
|
+
turnEnded() { completeThrough = Date.now(); void sync(); },
|
|
436
|
+
/** What her prompt shows now for `session` ('' when nothing or not ready yet). */
|
|
437
|
+
textFor: (session) => results.get(session)?.text ?? '',
|
|
438
|
+
/** The in-flight query for `session`, if any (prompt assembly waits for it, bounded). */
|
|
439
|
+
pending: (session) => { const r = results.get(session); return r && !r.done ? r.promise : null; },
|
|
440
|
+
clear: (session) => results.delete(session),
|
|
441
|
+
backend,
|
|
442
|
+
};
|
|
443
|
+
}
|