@hzttt/multimodal-rag 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -16
- package/index.ts +64 -6
- package/openclaw.plugin.json +35 -0
- package/package.json +1 -1
- package/src/notifier.ts +189 -0
- package/src/setup.ts +86 -16
- package/src/types.ts +19 -0
- package/src/watcher.ts +10 -1
package/README.md
CHANGED
|
@@ -11,6 +11,7 @@ OpenClaw 多模态 RAG 插件 — 使用本地 AI 模型对图像和音频进行
|
|
|
11
11
|
- **自动监听**:实时监听文件夹变化,自动索引新增文件
|
|
12
12
|
- **向量存储**:使用 LanceDB 高效存储和检索
|
|
13
13
|
- **智能去重**:基于文件 SHA256 哈希去重
|
|
14
|
+
- **索引通知**:批次聚合索引事件,通过 agent 自动通知到所有聊天渠道
|
|
14
15
|
|
|
15
16
|
## 前置条件
|
|
16
17
|
|
|
@@ -30,7 +31,7 @@ ollama pull qwen3-embedding:latest
|
|
|
30
31
|
### 方式一:从 npm 安装(推荐)
|
|
31
32
|
|
|
32
33
|
```bash
|
|
33
|
-
openclaw plugins install @hzttt/multimodal-rag
|
|
34
|
+
openclaw plugins install @hzttt/multimodal-rag@latest
|
|
34
35
|
```
|
|
35
36
|
|
|
36
37
|
插件会自动安装到 `~/.openclaw/extensions/multimodal-rag/`,并自动安装所有运行时依赖。
|
|
@@ -64,11 +65,55 @@ openclaw multimodal-rag setup
|
|
|
64
65
|
- **视觉模型**: `qwen3-vl:2b` (图像描述)
|
|
65
66
|
- **嵌入模型**: `qwen3-embedding:latest` (向量生成)
|
|
66
67
|
- **嵌入提供者**: `ollama` (本地)
|
|
67
|
-
- **数据库路径**:
|
|
68
|
+
- **数据库路径**: `~/.openclaw/multimodal-rag.lance`
|
|
68
69
|
- **启动时索引**: `true` (自动索引已有文件)
|
|
69
70
|
|
|
70
71
|
你只需要指定要监听的文件夹路径即可。
|
|
71
72
|
|
|
73
|
+
### 非交互式配置(适合脚本/远程部署)
|
|
74
|
+
|
|
75
|
+
通过命令行参数一次性完成配置,无需交互输入,适用于 SSH 远程部署、CI/CD 脚本等场景:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
# 最简用法:只指定监听路径(其他使用默认值)
|
|
79
|
+
openclaw multimodal-rag setup --non-interactive --watch ~/photos --watch ~/audio
|
|
80
|
+
|
|
81
|
+
# 简写形式
|
|
82
|
+
openclaw multimodal-rag setup -n -w ~/photos -w ~/audio
|
|
83
|
+
|
|
84
|
+
# 逗号分隔多个路径
|
|
85
|
+
openclaw multimodal-rag setup -n --watch ~/photos,~/mic-recordings,~/usb_data
|
|
86
|
+
|
|
87
|
+
# 自定义所有参数
|
|
88
|
+
openclaw multimodal-rag setup -n \
|
|
89
|
+
--watch ~/photos --watch ~/audio \
|
|
90
|
+
--ollama-url http://192.168.0.100:11434 \
|
|
91
|
+
--vision-model qwen3-vl:2b \
|
|
92
|
+
--embed-model qwen3-embedding:latest \
|
|
93
|
+
--db-path ~/.openclaw/my-rag.lance \
|
|
94
|
+
--no-index-on-start
|
|
95
|
+
|
|
96
|
+
# 使用 OpenAI 嵌入
|
|
97
|
+
openclaw multimodal-rag setup -n \
|
|
98
|
+
--watch ~/photos \
|
|
99
|
+
--embedding-provider openai \
|
|
100
|
+
--openai-api-key sk-xxx \
|
|
101
|
+
--openai-model text-embedding-3-small
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
| 选项 | 简写 | 说明 | 默认值 |
|
|
105
|
+
| --- | --- | --- | --- |
|
|
106
|
+
| `--non-interactive` | `-n` | 启用非交互式模式 | - |
|
|
107
|
+
| `--watch <paths...>` | `-w` | 监听路径(可多次指定或逗号分隔) | 必填 |
|
|
108
|
+
| `--ollama-url <url>` | - | Ollama 服务地址 | `http://127.0.0.1:11434` |
|
|
109
|
+
| `--vision-model <model>` | - | 视觉模型 | `qwen3-vl:2b` |
|
|
110
|
+
| `--embed-model <model>` | - | 嵌入模型 | `qwen3-embedding:latest` |
|
|
111
|
+
| `--embedding-provider <p>` | - | 嵌入提供者: `ollama` / `openai` | `ollama` |
|
|
112
|
+
| `--openai-api-key <key>` | - | OpenAI API Key | - |
|
|
113
|
+
| `--openai-model <model>` | - | OpenAI 嵌入模型 | `text-embedding-3-small` |
|
|
114
|
+
| `--db-path <path>` | - | LanceDB 数据库路径 | `~/.openclaw/multimodal-rag.lance` |
|
|
115
|
+
| `--no-index-on-start` | - | 启动时不索引已有文件 | `false` |
|
|
116
|
+
|
|
72
117
|
### 手动配置
|
|
73
118
|
|
|
74
119
|
如需自定义配置,编辑 `~/.openclaw/openclaw.json`:
|
|
@@ -90,7 +135,12 @@ openclaw multimodal-rag setup
|
|
|
90
135
|
"provider": "ollama"
|
|
91
136
|
},
|
|
92
137
|
"dbPath": "/home/lucy/.openclaw/multimodal-rag.lance",
|
|
93
|
-
"indexExistingOnStart": true
|
|
138
|
+
"indexExistingOnStart": true,
|
|
139
|
+
"notifications": {
|
|
140
|
+
"enabled": true,
|
|
141
|
+
"quietWindowMs": 30000,
|
|
142
|
+
"batchTimeoutMs": 600000
|
|
143
|
+
}
|
|
94
144
|
}
|
|
95
145
|
}
|
|
96
146
|
}
|
|
@@ -100,23 +150,76 @@ openclaw multimodal-rag setup
|
|
|
100
150
|
|
|
101
151
|
### 配置项说明
|
|
102
152
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
|
106
|
-
| `
|
|
107
|
-
| `ollama.
|
|
108
|
-
| `ollama.
|
|
109
|
-
| `
|
|
110
|
-
| `embedding.
|
|
111
|
-
| `embedding.
|
|
112
|
-
| `
|
|
113
|
-
| `
|
|
114
|
-
| `
|
|
153
|
+
|
|
154
|
+
| 配置项 | 类型 | 默认值 | 说明 |
|
|
155
|
+
| ---------------------------- | -------- | ---------------------------------- | -------------------------------------- |
|
|
156
|
+
| `watchPaths` | string[] | `[]` | 监听的文件夹路径(支持 `~` 展开) |
|
|
157
|
+
| `ollama.baseUrl` | string | `http://127.0.0.1:11434` | Ollama 服务地址 |
|
|
158
|
+
| `ollama.visionModel` | string | `qwen3-vl:2b` | 用于图像描述的视觉模型 |
|
|
159
|
+
| `ollama.embedModel` | string | `qwen3-embedding:latest` | 用于生成嵌入向量的模型 |
|
|
160
|
+
| `embedding.provider` | string | `ollama` | 嵌入提供者: `ollama` 或 `openai` |
|
|
161
|
+
| `embedding.openaiApiKey` | string | - | OpenAI API Key(仅 openai 时需要) |
|
|
162
|
+
| `embedding.openaiModel` | string | `text-embedding-3-small` | OpenAI 嵌入模型 |
|
|
163
|
+
| `dbPath` | string | `~/.openclaw/multimodal-rag.lance` | LanceDB 数据库路径 |
|
|
164
|
+
| `watchDebounceMs` | number | `1000` | 文件监听去抖延迟(毫秒) |
|
|
165
|
+
| `indexExistingOnStart` | boolean | `true` | 启动时是否索引已有文件 |
|
|
166
|
+
| `notifications.enabled` | boolean | `false` | 启用索引完成通知(通过 agent 发送到所有渠道) |
|
|
167
|
+
| `notifications.quietWindowMs` | number | `30000` | 静默窗口:最后一个文件处理完后等待多久再发送总结(毫秒) |
|
|
168
|
+
| `notifications.batchTimeoutMs` | number | `600000` | 批次最大超时:超过此时间强制发送总结(毫秒),防止大批量索引时等太久 |
|
|
169
|
+
|
|
115
170
|
|
|
116
171
|
配置完成后,重启 OpenClaw Gateway 使配置生效。
|
|
117
172
|
|
|
118
173
|
## 使用方法
|
|
119
174
|
|
|
175
|
+
### 索引通知(可选功能)
|
|
176
|
+
|
|
177
|
+
插件支持在索引新文件时自动发送通知到所有已配置的聊天渠道。通知采用批次聚合机制,避免逐文件通知造成的消息轰炸。
|
|
178
|
+
|
|
179
|
+
#### 工作原理
|
|
180
|
+
|
|
181
|
+
1. **开始通知**:检测到第一个新文件时,agent 会收到"开始索引"事件并通知用户
|
|
182
|
+
2. **批次聚合**:持续聚合多个文件的索引状态,避免频繁通知
|
|
183
|
+
3. **完成通知**:所有文件处理完成后(静默窗口到期),agent 会发送索引总结通知
|
|
184
|
+
|
|
185
|
+
#### 通知示例
|
|
186
|
+
|
|
187
|
+
**开始通知**:
|
|
188
|
+
```
|
|
189
|
+
[Multimodal RAG] 检测到 3 个新的媒体文件正在被索引...
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**完成通知**:
|
|
193
|
+
```
|
|
194
|
+
[Multimodal RAG] 索引完成
|
|
195
|
+
共处理 5 个文件,成功 4 个 (2 张图片, 2 个音频),失败 1 个
|
|
196
|
+
耗时 2 分 30 秒
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
#### 启用通知
|
|
200
|
+
|
|
201
|
+
在配置中设置 `notifications.enabled: true`:
|
|
202
|
+
|
|
203
|
+
```json
|
|
204
|
+
{
|
|
205
|
+
"plugins": {
|
|
206
|
+
"entries": {
|
|
207
|
+
"multimodal-rag": {
|
|
208
|
+
"config": {
|
|
209
|
+
"notifications": {
|
|
210
|
+
"enabled": true,
|
|
211
|
+
"quietWindowMs": 30000,
|
|
212
|
+
"batchTimeoutMs": 600000
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
**注意**:通知功能需要配置 heartbeat 才能保证 agent 及时响应。参考 [Heartbeat 文档](https://docs.openclaw.ai/configuration#heartbeat)。
|
|
222
|
+
|
|
120
223
|
### Agent 工具
|
|
121
224
|
|
|
122
225
|
插件注册 4 个 Agent 工具,可以在对话中自然地调用:
|
|
@@ -155,6 +258,9 @@ Agent:[调用 media_stats] → 总计 120 个文件,图片 80,音频 40
|
|
|
155
258
|
# 交互式配置
|
|
156
259
|
openclaw multimodal-rag setup
|
|
157
260
|
|
|
261
|
+
# 非交互式配置
|
|
262
|
+
openclaw multimodal-rag setup -n --watch ~/photos --watch ~/audio
|
|
263
|
+
|
|
158
264
|
# 手动索引文件或文件夹
|
|
159
265
|
openclaw multimodal-rag index ~/Pictures/photo.jpg
|
|
160
266
|
|
|
@@ -205,4 +311,4 @@ openclaw plugins list | grep multimodal-rag
|
|
|
205
311
|
|
|
206
312
|
## 许可证
|
|
207
313
|
|
|
208
|
-
MIT
|
|
314
|
+
MIT
|
package/index.ts
CHANGED
|
@@ -9,14 +9,15 @@ import { MediaStorage } from "./src/storage.js";
|
|
|
9
9
|
import { createEmbeddingProvider } from "./src/embeddings.js";
|
|
10
10
|
import { createMediaProcessor } from "./src/processor.js";
|
|
11
11
|
import { MediaWatcher } from "./src/watcher.js";
|
|
12
|
+
import { IndexNotifier } from "./src/notifier.js";
|
|
12
13
|
import {
|
|
13
14
|
createMediaSearchTool,
|
|
14
15
|
createMediaDescribeTool,
|
|
15
16
|
createMediaListTool,
|
|
16
17
|
createMediaStatsTool,
|
|
17
18
|
} from "./src/tools.js";
|
|
18
|
-
import { runSetup } from "./src/setup.js";
|
|
19
|
-
import type { PluginConfig } from "./src/types.js";
|
|
19
|
+
import { runSetup, runNonInteractiveSetup } from "./src/setup.js";
|
|
20
|
+
import type { PluginConfig, NotificationConfig } from "./src/types.js";
|
|
20
21
|
|
|
21
22
|
const multimodalRagPlugin = {
|
|
22
23
|
id: "multimodal-rag",
|
|
@@ -48,6 +49,11 @@ const multimodalRagPlugin = {
|
|
|
48
49
|
dbPath: userConfig.dbPath || "~/.openclaw/multimodal-rag.lance",
|
|
49
50
|
watchDebounceMs: userConfig.watchDebounceMs || 1000,
|
|
50
51
|
indexExistingOnStart: userConfig.indexExistingOnStart !== false,
|
|
52
|
+
notifications: {
|
|
53
|
+
enabled: userConfig.notifications?.enabled ?? false,
|
|
54
|
+
quietWindowMs: userConfig.notifications?.quietWindowMs ?? 30000,
|
|
55
|
+
batchTimeoutMs: userConfig.notifications?.batchTimeoutMs ?? 600000,
|
|
56
|
+
},
|
|
51
57
|
};
|
|
52
58
|
|
|
53
59
|
// 解析数据库路径
|
|
@@ -76,8 +82,20 @@ const multimodalRagPlugin = {
|
|
|
76
82
|
visionModel: cfg.ollama.visionModel,
|
|
77
83
|
});
|
|
78
84
|
|
|
85
|
+
// 创建通知器(如果启用)
|
|
86
|
+
let notifier: IndexNotifier | undefined;
|
|
87
|
+
if (cfg.notifications?.enabled) {
|
|
88
|
+
notifier = new IndexNotifier(
|
|
89
|
+
cfg.notifications,
|
|
90
|
+
api.runtime,
|
|
91
|
+
api.config,
|
|
92
|
+
api.logger,
|
|
93
|
+
);
|
|
94
|
+
api.logger.info?.("multimodal-rag: Notifications enabled");
|
|
95
|
+
}
|
|
96
|
+
|
|
79
97
|
// 创建文件监听器
|
|
80
|
-
const watcher = new MediaWatcher(cfg, storage, embeddings, processor, api.logger);
|
|
98
|
+
const watcher = new MediaWatcher(cfg, storage, embeddings, processor, api.logger, notifier);
|
|
81
99
|
|
|
82
100
|
// ========================================================================
|
|
83
101
|
// 注册工具
|
|
@@ -288,11 +306,51 @@ const multimodalRagPlugin = {
|
|
|
288
306
|
});
|
|
289
307
|
|
|
290
308
|
// openclaw multimodal-rag setup
|
|
309
|
+
// 支持交互式和非交互式两种模式:
|
|
310
|
+
// 交互式: openclaw multimodal-rag setup
|
|
311
|
+
// 非交互式: openclaw multimodal-rag setup -n --watch ~/photos --watch ~/audio
|
|
291
312
|
rag
|
|
292
313
|
.command("setup")
|
|
293
|
-
.description("
|
|
294
|
-
.
|
|
295
|
-
|
|
314
|
+
.description("配置插件(支持交互式和非交互式模式)")
|
|
315
|
+
.option("-n, --non-interactive", "非交互式模式(需配合 --watch 使用)")
|
|
316
|
+
.option("-w, --watch <paths...>", "监听路径(可多次指定或逗号分隔)")
|
|
317
|
+
.option("--ollama-url <url>", "Ollama 服务地址", "http://127.0.0.1:11434")
|
|
318
|
+
.option("--vision-model <model>", "视觉模型名称", "qwen3-vl:2b")
|
|
319
|
+
.option("--embed-model <model>", "嵌入模型名称", "qwen3-embedding:latest")
|
|
320
|
+
.option("--embedding-provider <provider>", "嵌入提供者: ollama 或 openai", "ollama")
|
|
321
|
+
.option("--openai-api-key <key>", "OpenAI API Key(仅 openai 时需要)")
|
|
322
|
+
.option("--openai-model <model>", "OpenAI 嵌入模型")
|
|
323
|
+
.option("--db-path <path>", "LanceDB 数据库路径")
|
|
324
|
+
.option("--no-index-on-start", "启动时不索引已有文件")
|
|
325
|
+
.action(async (opts: {
|
|
326
|
+
nonInteractive?: boolean;
|
|
327
|
+
watch?: string[];
|
|
328
|
+
ollamaUrl?: string;
|
|
329
|
+
visionModel?: string;
|
|
330
|
+
embedModel?: string;
|
|
331
|
+
embeddingProvider?: string;
|
|
332
|
+
openaiApiKey?: string;
|
|
333
|
+
openaiModel?: string;
|
|
334
|
+
dbPath?: string;
|
|
335
|
+
noIndexOnStart?: boolean;
|
|
336
|
+
}) => {
|
|
337
|
+
if (opts.nonInteractive) {
|
|
338
|
+
// 非交互式模式:展开逗号分隔的路径
|
|
339
|
+
const watchPaths = (opts.watch || []).flatMap((p) => p.split(",").map((s) => s.trim()).filter(Boolean));
|
|
340
|
+
await runNonInteractiveSetup({
|
|
341
|
+
watch: watchPaths,
|
|
342
|
+
ollamaUrl: opts.ollamaUrl,
|
|
343
|
+
visionModel: opts.visionModel,
|
|
344
|
+
embedModel: opts.embedModel,
|
|
345
|
+
embeddingProvider: opts.embeddingProvider as "ollama" | "openai" | undefined,
|
|
346
|
+
openaiApiKey: opts.openaiApiKey,
|
|
347
|
+
openaiModel: opts.openaiModel,
|
|
348
|
+
dbPath: opts.dbPath,
|
|
349
|
+
noIndexOnStart: opts.noIndexOnStart,
|
|
350
|
+
});
|
|
351
|
+
} else {
|
|
352
|
+
await runSetup();
|
|
353
|
+
}
|
|
296
354
|
});
|
|
297
355
|
}, { commands: ["multimodal-rag"] });
|
|
298
356
|
|
package/openclaw.plugin.json
CHANGED
|
@@ -82,6 +82,27 @@
|
|
|
82
82
|
"type": "boolean",
|
|
83
83
|
"default": true,
|
|
84
84
|
"description": "启动时是否索引现有文件"
|
|
85
|
+
},
|
|
86
|
+
"notifications": {
|
|
87
|
+
"type": "object",
|
|
88
|
+
"properties": {
|
|
89
|
+
"enabled": {
|
|
90
|
+
"type": "boolean",
|
|
91
|
+
"default": false,
|
|
92
|
+
"description": "启用索引完成通知(通过 agent 发送到所有渠道)"
|
|
93
|
+
},
|
|
94
|
+
"quietWindowMs": {
|
|
95
|
+
"type": "number",
|
|
96
|
+
"default": 30000,
|
|
97
|
+
"description": "静默窗口:最后一个文件处理完后等待多久再发送总结(毫秒)"
|
|
98
|
+
},
|
|
99
|
+
"batchTimeoutMs": {
|
|
100
|
+
"type": "number",
|
|
101
|
+
"default": 600000,
|
|
102
|
+
"description": "批次最大超时:超过此时间强制发送总结(毫秒)"
|
|
103
|
+
}
|
|
104
|
+
},
|
|
105
|
+
"default": {}
|
|
85
106
|
}
|
|
86
107
|
}
|
|
87
108
|
},
|
|
@@ -109,6 +130,20 @@
|
|
|
109
130
|
"watchDebounceMs": {
|
|
110
131
|
"label": "去抖延迟(毫秒)",
|
|
111
132
|
"advanced": true
|
|
133
|
+
},
|
|
134
|
+
"notifications.enabled": {
|
|
135
|
+
"label": "启用通知",
|
|
136
|
+
"help": "索引完成后通过 agent 发送通知到所有渠道"
|
|
137
|
+
},
|
|
138
|
+
"notifications.quietWindowMs": {
|
|
139
|
+
"label": "静默窗口(毫秒)",
|
|
140
|
+
"advanced": true,
|
|
141
|
+
"help": "最后一个文件处理完后等待多久再发送总结"
|
|
142
|
+
},
|
|
143
|
+
"notifications.batchTimeoutMs": {
|
|
144
|
+
"label": "批次超时(毫秒)",
|
|
145
|
+
"advanced": true,
|
|
146
|
+
"help": "超过此时间强制发送总结,防止大批量索引时等太久"
|
|
112
147
|
}
|
|
113
148
|
}
|
|
114
149
|
}
|
package/package.json
CHANGED
package/src/notifier.ts
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 索引通知器 - 批次聚合 + Agent 触发
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import type { PluginRuntime } from "openclaw/plugin-sdk";
|
|
6
|
+
import type { MediaType, NotificationConfig, IndexEventCallbacks } from "./types.js";
|
|
7
|
+
|
|
8
|
+
type BatchFileStatus = "queued" | "indexed" | "failed";
|
|
9
|
+
type BatchFile = { status: BatchFileStatus; fileType?: MediaType; error?: string };
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* IndexNotifier 负责聚合索引事件并触发 agent 通知
|
|
13
|
+
*
|
|
14
|
+
* 工作流程:
|
|
15
|
+
* 1. 首个文件入队 -> 发送"开始索引"系统事件 -> 唤醒 agent
|
|
16
|
+
* 2. 持续聚合文件状态,重置静默计时器
|
|
17
|
+
* 3. 静默窗口到期且无待处理文件 -> 发送"索引完成总结"系统事件 -> 唤醒 agent
|
|
18
|
+
*/
|
|
19
|
+
export class IndexNotifier implements IndexEventCallbacks {
|
|
20
|
+
private state: "idle" | "batching" = "idle";
|
|
21
|
+
private batch: Map<string, BatchFile> = new Map();
|
|
22
|
+
private batchStartTime = 0;
|
|
23
|
+
private quietTimer: NodeJS.Timeout | null = null;
|
|
24
|
+
private batchTimeoutTimer: NodeJS.Timeout | null = null;
|
|
25
|
+
|
|
26
|
+
constructor(
|
|
27
|
+
private readonly config: NotificationConfig,
|
|
28
|
+
private readonly runtime: PluginRuntime,
|
|
29
|
+
private readonly openclawConfig: {
|
|
30
|
+
session?: { mainKey?: string };
|
|
31
|
+
agents?: { list?: Array<{ id?: string; default?: boolean }> };
|
|
32
|
+
},
|
|
33
|
+
private readonly logger: { info?: (msg: string) => void; warn?: (msg: string) => void },
|
|
34
|
+
) {}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* 文件入队事件
|
|
38
|
+
*/
|
|
39
|
+
onFileQueued(filePath: string): void {
|
|
40
|
+
this.batch.set(filePath, { status: "queued" });
|
|
41
|
+
|
|
42
|
+
if (this.state === "idle") {
|
|
43
|
+
// 首个文件,开始批次
|
|
44
|
+
this.state = "batching";
|
|
45
|
+
this.batchStartTime = Date.now();
|
|
46
|
+
|
|
47
|
+
// 发送"开始索引"系统事件,唤醒 agent
|
|
48
|
+
this.triggerAgent(this.buildStartMessage());
|
|
49
|
+
|
|
50
|
+
// 设置批次最大超时
|
|
51
|
+
this.batchTimeoutTimer = setTimeout(() => {
|
|
52
|
+
this.finalizeBatch();
|
|
53
|
+
}, this.config.batchTimeoutMs);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
this.resetQuietTimer();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* 文件索引成功事件
|
|
61
|
+
*/
|
|
62
|
+
onFileIndexed(filePath: string, fileType: MediaType): void {
|
|
63
|
+
this.batch.set(filePath, { status: "indexed", fileType });
|
|
64
|
+
this.resetQuietTimer();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* 文件索引失败事件
|
|
69
|
+
*/
|
|
70
|
+
onFileFailed(filePath: string, error: string): void {
|
|
71
|
+
this.batch.set(filePath, { status: "failed", error });
|
|
72
|
+
this.resetQuietTimer();
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* 重置静默窗口计时器
|
|
77
|
+
*/
|
|
78
|
+
private resetQuietTimer(): void {
|
|
79
|
+
if (this.quietTimer) {
|
|
80
|
+
clearTimeout(this.quietTimer);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
this.quietTimer = setTimeout(() => {
|
|
84
|
+
// 检查是否还有文件在处理中
|
|
85
|
+
const hasQueued = [...this.batch.values()].some((f) => f.status === "queued");
|
|
86
|
+
|
|
87
|
+
if (!hasQueued) {
|
|
88
|
+
// 所有文件都处理完了,发送总结
|
|
89
|
+
this.finalizeBatch();
|
|
90
|
+
} else {
|
|
91
|
+
// 还有文件在排队,继续等
|
|
92
|
+
this.resetQuietTimer();
|
|
93
|
+
}
|
|
94
|
+
}, this.config.quietWindowMs);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* 批次完成,发送总结
|
|
99
|
+
*/
|
|
100
|
+
private finalizeBatch(): void {
|
|
101
|
+
if (this.state !== "batching") {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
this.clearTimers();
|
|
106
|
+
this.triggerAgent(this.buildSummaryMessage());
|
|
107
|
+
|
|
108
|
+
// 重置状态
|
|
109
|
+
this.batch.clear();
|
|
110
|
+
this.state = "idle";
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* 清理所有计时器
|
|
115
|
+
*/
|
|
116
|
+
private clearTimers(): void {
|
|
117
|
+
if (this.quietTimer) {
|
|
118
|
+
clearTimeout(this.quietTimer);
|
|
119
|
+
this.quietTimer = null;
|
|
120
|
+
}
|
|
121
|
+
if (this.batchTimeoutTimer) {
|
|
122
|
+
clearTimeout(this.batchTimeoutTimer);
|
|
123
|
+
this.batchTimeoutTimer = null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* 触发 agent: enqueueSystemEvent + requestHeartbeatNow
|
|
129
|
+
*/
|
|
130
|
+
private triggerAgent(text: string): void {
|
|
131
|
+
try {
|
|
132
|
+
const sessionKey = this.runtime.system.resolveMainSessionKey(this.openclawConfig);
|
|
133
|
+
this.runtime.system.enqueueSystemEvent(text, { sessionKey });
|
|
134
|
+
this.runtime.system.requestHeartbeatNow({ reason: "multimodal-rag:notification" });
|
|
135
|
+
this.logger.info?.(`Notification triggered: ${text.slice(0, 80)}...`);
|
|
136
|
+
} catch (err) {
|
|
137
|
+
this.logger.warn?.(`Failed to trigger agent notification: ${String(err)}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* 构建"开始索引"消息
|
|
143
|
+
*/
|
|
144
|
+
private buildStartMessage(): string {
|
|
145
|
+
const count = this.batch.size;
|
|
146
|
+
return `[Multimodal RAG] 新文件索引通知: 检测到 ${count} 个新的媒体文件正在被索引,请通知用户。`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* 构建"索引完成总结"消息
|
|
151
|
+
*/
|
|
152
|
+
private buildSummaryMessage(): string {
|
|
153
|
+
const files = [...this.batch.values()];
|
|
154
|
+
const total = files.length;
|
|
155
|
+
const succeeded = files.filter((f) => f.status === "indexed");
|
|
156
|
+
const failed = files.filter((f) => f.status === "failed");
|
|
157
|
+
|
|
158
|
+
// 统计成功文件的类型
|
|
159
|
+
const images = succeeded.filter((f) => f.fileType === "image").length;
|
|
160
|
+
const audios = succeeded.filter((f) => f.fileType === "audio").length;
|
|
161
|
+
|
|
162
|
+
// 计算耗时
|
|
163
|
+
const durationMs = Date.now() - this.batchStartTime;
|
|
164
|
+
const durationSec = Math.floor(durationMs / 1000);
|
|
165
|
+
const minutes = Math.floor(durationSec / 60);
|
|
166
|
+
const seconds = durationSec % 60;
|
|
167
|
+
const durationStr =
|
|
168
|
+
minutes > 0 ? `${minutes} 分 ${seconds} 秒` : `${seconds} 秒`;
|
|
169
|
+
|
|
170
|
+
// 构建消息
|
|
171
|
+
let message = `[Multimodal RAG] 索引完成通知: 共处理 ${total} 个文件,`;
|
|
172
|
+
message += `成功 ${succeeded.length} 个`;
|
|
173
|
+
|
|
174
|
+
if (images > 0 || audios > 0) {
|
|
175
|
+
const parts: string[] = [];
|
|
176
|
+
if (images > 0) parts.push(`${images} 张图片`);
|
|
177
|
+
if (audios > 0) parts.push(`${audios} 个音频`);
|
|
178
|
+
message += ` (${parts.join(", ")})`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (failed.length > 0) {
|
|
182
|
+
message += `,失败 ${failed.length} 个`;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
message += `。耗时 ${durationStr}。请发送索引完成总结通知给用户。`;
|
|
186
|
+
|
|
187
|
+
return message;
|
|
188
|
+
}
|
|
189
|
+
}
|
package/src/setup.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* 引导配置(交互式 & 非交互式)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* 交互式: `openclaw multimodal-rag setup`
|
|
5
|
+
* 非交互式: `openclaw multimodal-rag setup --watch ~/photos,~/audio --non-interactive`
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import * as readline from "node:readline/promises";
|
|
@@ -31,6 +31,19 @@ type PluginConfigPartial = {
|
|
|
31
31
|
indexExistingOnStart?: boolean;
|
|
32
32
|
};
|
|
33
33
|
|
|
34
|
+
/** 非交互式 setup 的选项 */
|
|
35
|
+
export type NonInteractiveSetupOpts = {
|
|
36
|
+
watch: string[];
|
|
37
|
+
ollamaUrl?: string;
|
|
38
|
+
visionModel?: string;
|
|
39
|
+
embedModel?: string;
|
|
40
|
+
embeddingProvider?: "ollama" | "openai";
|
|
41
|
+
openaiApiKey?: string;
|
|
42
|
+
openaiModel?: string;
|
|
43
|
+
dbPath?: string;
|
|
44
|
+
noIndexOnStart?: boolean;
|
|
45
|
+
};
|
|
46
|
+
|
|
34
47
|
function loadOpenClawConfig(): Record<string, unknown> {
|
|
35
48
|
if (!fs.existsSync(CONFIG_FILE)) {
|
|
36
49
|
return {};
|
|
@@ -54,6 +67,75 @@ function getExistingPluginConfig(config: Record<string, unknown>): PluginConfigP
|
|
|
54
67
|
return (entry?.config as PluginConfigPartial) || {};
|
|
55
68
|
}
|
|
56
69
|
|
|
70
|
+
/**
|
|
71
|
+
* 将插件配置写入 openclaw.json
|
|
72
|
+
*/
|
|
73
|
+
function writePluginConfig(pluginConfig: PluginConfigPartial): void {
|
|
74
|
+
const config = loadOpenClawConfig();
|
|
75
|
+
const plugins = (config.plugins || {}) as Record<string, unknown>;
|
|
76
|
+
const entries = (plugins.entries || {}) as Record<string, unknown>;
|
|
77
|
+
const pluginEntry = (entries["multimodal-rag"] || {}) as Record<string, unknown>;
|
|
78
|
+
|
|
79
|
+
pluginEntry.enabled = true;
|
|
80
|
+
pluginEntry.config = pluginConfig;
|
|
81
|
+
|
|
82
|
+
entries["multimodal-rag"] = pluginEntry;
|
|
83
|
+
plugins.entries = entries;
|
|
84
|
+
config.plugins = plugins;
|
|
85
|
+
|
|
86
|
+
saveOpenClawConfig(config);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* 非交互式配置
|
|
91
|
+
*
|
|
92
|
+
* 适用于脚本自动化、SSH 远程部署等场景。
|
|
93
|
+
* 所有参数通过命令行选项传入,不读取 stdin。
|
|
94
|
+
*
|
|
95
|
+
* 用法:
|
|
96
|
+
* openclaw multimodal-rag setup --non-interactive --watch ~/photos,~/audio
|
|
97
|
+
* openclaw multimodal-rag setup -n -w ~/photos -w ~/audio --ollama-url http://host:11434
|
|
98
|
+
*/
|
|
99
|
+
export async function runNonInteractiveSetup(opts: NonInteractiveSetupOpts): Promise<void> {
|
|
100
|
+
if (opts.watch.length === 0) {
|
|
101
|
+
console.error("✗ 非交互式模式必须通过 --watch 指定至少一个监听路径");
|
|
102
|
+
console.error(" 示例: openclaw multimodal-rag setup --non-interactive --watch ~/photos");
|
|
103
|
+
process.exit(1);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const existing = getExistingPluginConfig(loadOpenClawConfig());
|
|
107
|
+
|
|
108
|
+
const pluginConfig: PluginConfigPartial = {
|
|
109
|
+
watchPaths: opts.watch,
|
|
110
|
+
ollama: {
|
|
111
|
+
baseUrl: opts.ollamaUrl || existing.ollama?.baseUrl || "http://127.0.0.1:11434",
|
|
112
|
+
visionModel: opts.visionModel || existing.ollama?.visionModel || "qwen3-vl:2b",
|
|
113
|
+
embedModel: opts.embedModel || existing.ollama?.embedModel || "qwen3-embedding:latest",
|
|
114
|
+
},
|
|
115
|
+
embedding: {
|
|
116
|
+
provider: opts.embeddingProvider || existing.embedding?.provider || "ollama",
|
|
117
|
+
...(opts.openaiApiKey && { openaiApiKey: opts.openaiApiKey }),
|
|
118
|
+
...(opts.openaiModel && { openaiModel: opts.openaiModel }),
|
|
119
|
+
},
|
|
120
|
+
dbPath: opts.dbPath || existing.dbPath || "~/.openclaw/multimodal-rag.lance",
|
|
121
|
+
indexExistingOnStart: opts.noIndexOnStart ? false : (existing.indexExistingOnStart !== false),
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
writePluginConfig(pluginConfig);
|
|
125
|
+
|
|
126
|
+
console.log("✓ 配置已保存到 ~/.openclaw/openclaw.json\n");
|
|
127
|
+
console.log("配置摘要:");
|
|
128
|
+
console.log(` 监听路径: ${pluginConfig.watchPaths!.join(", ")}`);
|
|
129
|
+
console.log(` Ollama 地址: ${pluginConfig.ollama!.baseUrl}`);
|
|
130
|
+
console.log(` 视觉模型: ${pluginConfig.ollama!.visionModel}`);
|
|
131
|
+
console.log(` 嵌入模型: ${pluginConfig.ollama!.embedModel}`);
|
|
132
|
+
console.log(` 嵌入提供者: ${pluginConfig.embedding!.provider}`);
|
|
133
|
+
console.log(` 数据库路径: ${pluginConfig.dbPath}`);
|
|
134
|
+
console.log(` 启动时索引: ${pluginConfig.indexExistingOnStart ? "是" : "否"}`);
|
|
135
|
+
console.log();
|
|
136
|
+
console.log("提示: 重启 OpenClaw Gateway 以加载新配置");
|
|
137
|
+
}
|
|
138
|
+
|
|
57
139
|
async function checkOllamaHealth(baseUrl: string): Promise<boolean> {
|
|
58
140
|
try {
|
|
59
141
|
const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(5000) });
|
|
@@ -148,19 +230,7 @@ export async function runSetup(): Promise<void> {
|
|
|
148
230
|
// ================================================================
|
|
149
231
|
console.log("\n── 保存配置 ──\n");
|
|
150
232
|
|
|
151
|
-
|
|
152
|
-
const plugins = (config.plugins || {}) as Record<string, unknown>;
|
|
153
|
-
const entries = (plugins.entries || {}) as Record<string, unknown>;
|
|
154
|
-
const pluginEntry = (entries["multimodal-rag"] || {}) as Record<string, unknown>;
|
|
155
|
-
|
|
156
|
-
pluginEntry.enabled = true;
|
|
157
|
-
pluginEntry.config = pluginConfig;
|
|
158
|
-
|
|
159
|
-
entries["multimodal-rag"] = pluginEntry;
|
|
160
|
-
plugins.entries = entries;
|
|
161
|
-
config.plugins = plugins;
|
|
162
|
-
|
|
163
|
-
saveOpenClawConfig(config);
|
|
233
|
+
writePluginConfig(pluginConfig);
|
|
164
234
|
|
|
165
235
|
console.log("✓ 配置已保存到 ~/.openclaw/openclaw.json\n");
|
|
166
236
|
|
package/src/types.ts
CHANGED
|
@@ -30,6 +30,24 @@ export type MediaSearchResult = {
|
|
|
30
30
|
score: number; // 相似度分数 (0-1)
|
|
31
31
|
};
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* 通知配置
|
|
35
|
+
*/
|
|
36
|
+
export type NotificationConfig = {
|
|
37
|
+
enabled: boolean; // 是否启用通知,默认 false
|
|
38
|
+
quietWindowMs: number; // 静默窗口(毫秒),默认 30000
|
|
39
|
+
batchTimeoutMs: number; // 批次最大超时(毫秒),默认 600000
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* 索引事件回调接口(用于 watcher -> notifier 通信)
|
|
44
|
+
*/
|
|
45
|
+
export type IndexEventCallbacks = {
|
|
46
|
+
onFileQueued: (filePath: string) => void;
|
|
47
|
+
onFileIndexed: (filePath: string, fileType: MediaType) => void;
|
|
48
|
+
onFileFailed: (filePath: string, error: string) => void;
|
|
49
|
+
};
|
|
50
|
+
|
|
33
51
|
/**
|
|
34
52
|
* 插件配置
|
|
35
53
|
*/
|
|
@@ -52,6 +70,7 @@ export type PluginConfig = {
|
|
|
52
70
|
dbPath: string;
|
|
53
71
|
watchDebounceMs: number;
|
|
54
72
|
indexExistingOnStart: boolean;
|
|
73
|
+
notifications?: NotificationConfig;
|
|
55
74
|
};
|
|
56
75
|
|
|
57
76
|
/**
|
package/src/watcher.ts
CHANGED
|
@@ -8,7 +8,7 @@ import { basename, extname, resolve, join } from "node:path";
|
|
|
8
8
|
import { createHash } from "node:crypto";
|
|
9
9
|
import { readFile } from "node:fs/promises";
|
|
10
10
|
import { homedir } from "node:os";
|
|
11
|
-
import type { MediaType, PluginConfig } from "./types.js";
|
|
11
|
+
import type { MediaType, PluginConfig, IndexEventCallbacks } from "./types.js";
|
|
12
12
|
import type { MediaStorage } from "./storage.js";
|
|
13
13
|
import type { IEmbeddingProvider } from "./types.js";
|
|
14
14
|
import type { IMediaProcessor } from "./types.js";
|
|
@@ -45,6 +45,7 @@ export class MediaWatcher {
|
|
|
45
45
|
private readonly embeddings: IEmbeddingProvider,
|
|
46
46
|
private readonly processor: IMediaProcessor,
|
|
47
47
|
private readonly logger: { info?: (msg: string) => void; warn?: (msg: string) => void },
|
|
48
|
+
private readonly callbacks?: IndexEventCallbacks,
|
|
48
49
|
) {}
|
|
49
50
|
|
|
50
51
|
/**
|
|
@@ -156,6 +157,8 @@ export class MediaWatcher {
|
|
|
156
157
|
*/
|
|
157
158
|
private enqueueFile(filePath: string): void {
|
|
158
159
|
this.processQueue.add(filePath);
|
|
160
|
+
// 通知回调:文件已入队
|
|
161
|
+
this.callbacks?.onFileQueued(filePath);
|
|
159
162
|
this.processNextFile();
|
|
160
163
|
}
|
|
161
164
|
|
|
@@ -278,6 +281,9 @@ export class MediaWatcher {
|
|
|
278
281
|
|
|
279
282
|
// 索引成功,清除失败记录
|
|
280
283
|
this.failedFiles.delete(filePath);
|
|
284
|
+
|
|
285
|
+
// 通知回调:文件索引成功
|
|
286
|
+
this.callbacks?.onFileIndexed(filePath, fileType);
|
|
281
287
|
} catch (error) {
|
|
282
288
|
const errorMsg = String(error);
|
|
283
289
|
this.logger.warn?.(`Failed to index ${filePath}: ${errorMsg}`);
|
|
@@ -301,6 +307,9 @@ export class MediaWatcher {
|
|
|
301
307
|
setTimeout(() => {
|
|
302
308
|
this.enqueueFile(filePath);
|
|
303
309
|
}, 60000);
|
|
310
|
+
} else {
|
|
311
|
+
// 达到最大重试次数或非临时错误,通知回调:文件索引失败
|
|
312
|
+
this.callbacks?.onFileFailed(filePath, errorMsg);
|
|
304
313
|
}
|
|
305
314
|
}
|
|
306
315
|
}
|