@xiaofanzhao/dsh-plugin-knowledge-base 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -1
- package/client.js +127 -2
- package/docs/DETAIL.md +14 -1
- package/index.js +17 -2
- package/lib/pipeline.mjs +49 -1
- package/lib/routes.mjs +69 -23
- package/lib/settings.mjs +67 -26
- package/lib/vision.mjs +140 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# DSH 个人知识库插件
|
|
2
2
|
|
|
3
|
+
[](https://www.npmjs.com/package/@xiaofanzhao/dsh-plugin-knowledge-base)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
3
6
|
把本地文件整理成树状知识库,在 DeepSeek Harness 里直接读、直接搜,AI 也能检索并针对某一篇文档对话。
|
|
4
7
|
|
|
5
8
|
数据全在本机(默认 `~/.dsh/kb`),不经过任何云端。
|
|
@@ -41,6 +44,7 @@ dsh plugin --profile web add @xiaofanzhao/dsh-plugin-knowledge-base@0.1.0
|
|
|
41
44
|
- **AI 检索**:模型侧有三个工具 `kb_list` / `kb_search` / `kb_read`,任何会话里问"知识库里有没有…"都会自动用上
|
|
42
45
|
- **针对文档对话**:「问 AI」把带 docId 的提问预填到会话输入框,模型先读文档再回答,并标注出处
|
|
43
46
|
- **看板**:知识库规模、文件类型分布、解析状态、7 天检索趋势、热门检索词、零命中率、最近检索与最近添加——检索记录只存本地,最多 2000 条
|
|
47
|
+
- **图片也能搜**(可选):接一个视觉模型,上传/重建时把图片描述成结构化文字(画面 / 文字 / 数据 / 场景),再走同一套关键词 + 向量检索,AI 也能读出图里写了什么
|
|
44
48
|
|
|
45
49
|
## 安装
|
|
46
50
|
|
|
@@ -52,6 +56,8 @@ dsh plugin --profile web add @xiaofanzhao/dsh-plugin-knowledge-base@0.1.0
|
|
|
52
56
|
dsh plugin --profile web add @xiaofanzhao/dsh-plugin-knowledge-base@0.1.0
|
|
53
57
|
```
|
|
54
58
|
|
|
59
|
+
已发布在 npm:[`@xiaofanzhao/dsh-plugin-knowledge-base`](https://www.npmjs.com/package/@xiaofanzhao/dsh-plugin-knowledge-base)
|
|
60
|
+
|
|
55
61
|
一条命令就够:它会把包装进 profile、把包名写进 `dsh.profile.bundles`,profile 启动时再合并包内自带的 `cordis.patch.yml`。**不需要手改任何 profile 文件**。
|
|
56
62
|
|
|
57
63
|
- 装哪个 profile 就把 `web` 换成你的(桌面端是 `desktop`)
|
|
@@ -162,6 +168,29 @@ git push origin v0.1.0
|
|
|
162
168
|
|
|
163
169
|
> 桌面端不允许插件页面开新窗口(Electron 拦掉了 `target=_blank`),所以大屏是应用内全屏;要更大或用 PowerPoint 放映,用「下载」拿原件。
|
|
164
170
|
|
|
171
|
+
### 让图片也能被搜到(可选)
|
|
172
|
+
|
|
173
|
+
图片本身没有文字层,默认只能看、搜不到。打开「图片理解」后,上传或重建索引时会调用一个**视觉模型**把图片描述成结构化文字,再走和文档一样的索引:
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
图片 → 视觉模型 → 【画面】【文字】【数据】【场景】四段描述
|
|
177
|
+
→ 当正文存起来 → 切块 → 关键词 + 向量 → 可被搜索、可被 AI 读取
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
**怎么开**:知识库页面 → 右上角**齿轮** → 「图片理解(可选)」→ 填 Key(留空则复用向量检索那把)→ 勾选启用 → **测试连接** → 保存 → 点「重建全部索引」给已有图片补描述。
|
|
181
|
+
|
|
182
|
+
| 服务 | 接口地址 | 视觉模型 |
|
|
183
|
+
|---|---|---|
|
|
184
|
+
| 阿里云百炼 | `https://dashscope.aliyuncs.com/compatible-mode/v1` | `qwen3-vl-plus` |
|
|
185
|
+
| OpenAI | `https://api.openai.com/v1` | `gpt-4o-mini` |
|
|
186
|
+
|
|
187
|
+
几点说明:
|
|
188
|
+
|
|
189
|
+
- 默认**关闭**;没开的时候图片照常能看,只是搜不到
|
|
190
|
+
- 每张图一次模型调用(图片会发给该服务商,注意这一点)
|
|
191
|
+
- 单次转换失败会在文档状态里留下原因(比如 `图片理解接口报错:HTTP 401`),修好后点「重建」重试
|
|
192
|
+
- 只对 `png / jpg / jpeg / gif / webp / bmp` 生效;扫描件 PDF 仍按「没有文字层」处理(可以把页面截成图再上传)
|
|
193
|
+
|
|
165
194
|
## AI 工具
|
|
166
195
|
|
|
167
196
|
| 工具 | 作用 | 主要参数 |
|
|
@@ -251,7 +280,7 @@ $DSH_HOME/kb/
|
|
|
251
280
|
| docx / pptx / xlsx | ✅(转 PDF 再逐页出图) | ✅ | ✅ |
|
|
252
281
|
| doc / ppt / xls(老格式) | ✅ | — | ❌ 会提示 |
|
|
253
282
|
| pdf | ✅(浏览器阅读器) | — | ✅ 有文字层的;扫描件会提示 |
|
|
254
|
-
| png / jpg / gif / webp / bmp / svg | ✅ | — |
|
|
283
|
+
| png / jpg / gif / webp / bmp / svg | ✅ | — | 开「图片理解」后可搜(描述成文字再索引) |
|
|
255
284
|
|
|
256
285
|
## 架构
|
|
257
286
|
|
package/client.js
CHANGED
|
@@ -1330,6 +1330,12 @@ window.__ModuleLoader__.load({
|
|
|
1330
1330
|
const [probing, setProbing] = React.useState(false);
|
|
1331
1331
|
const [probe, setProbe] = React.useState(null);
|
|
1332
1332
|
const [formNotice, setFormNotice] = React.useState('');
|
|
1333
|
+
// 图片理解(视觉模型)那一块自己的表单状态
|
|
1334
|
+
const [visionForm, setVisionForm] = React.useState(null);
|
|
1335
|
+
const [visionSaving, setVisionSaving] = React.useState(false);
|
|
1336
|
+
const [visionProbing, setVisionProbing] = React.useState(false);
|
|
1337
|
+
const [visionProbe, setVisionProbe] = React.useState(null);
|
|
1338
|
+
const [visionNotice, setVisionNotice] = React.useState('');
|
|
1333
1339
|
const [rebuilding, setRebuilding] = React.useState(false);
|
|
1334
1340
|
const watchRef = React.useRef(null);
|
|
1335
1341
|
|
|
@@ -1370,11 +1376,64 @@ window.__ModuleLoader__.load({
|
|
|
1370
1376
|
if (!state.loaded) loadState();
|
|
1371
1377
|
let cancelled = false;
|
|
1372
1378
|
api('/settings')
|
|
1373
|
-
.then((data) => {
|
|
1374
|
-
|
|
1379
|
+
.then((data) => {
|
|
1380
|
+
if (cancelled) return;
|
|
1381
|
+
setForm({ ...data.embedding, apiKey: '' });
|
|
1382
|
+
setVisionForm({ ...(data.vision || { enabled: false, baseUrl: '', model: '' }), apiKey: '' });
|
|
1383
|
+
})
|
|
1384
|
+
.catch(() => {
|
|
1385
|
+
if (cancelled) return;
|
|
1386
|
+
setForm({ enabled: false, baseUrl: '', model: '', apiKey: '', hasKey: false, keyTail: '' });
|
|
1387
|
+
setVisionForm({ enabled: false, baseUrl: '', model: '', apiKey: '', hasKey: false, keyTail: '' });
|
|
1388
|
+
});
|
|
1375
1389
|
return () => { cancelled = true; };
|
|
1376
1390
|
}, []);
|
|
1377
1391
|
|
|
1392
|
+
const saveVision = async () => {
|
|
1393
|
+
if (!visionForm || visionSaving) return;
|
|
1394
|
+
setVisionSaving(true);
|
|
1395
|
+
setVisionNotice('');
|
|
1396
|
+
try {
|
|
1397
|
+
const data = await api('/settings', {
|
|
1398
|
+
method: 'POST',
|
|
1399
|
+
headers: { 'content-type': 'application/json' },
|
|
1400
|
+
body: JSON.stringify({
|
|
1401
|
+
vision: {
|
|
1402
|
+
enabled: Boolean(visionForm.enabled),
|
|
1403
|
+
baseUrl: visionForm.baseUrl,
|
|
1404
|
+
model: visionForm.model,
|
|
1405
|
+
apiKey: visionForm.apiKey,
|
|
1406
|
+
},
|
|
1407
|
+
}),
|
|
1408
|
+
});
|
|
1409
|
+
setVisionForm({ ...(data.vision || {}), apiKey: '' });
|
|
1410
|
+
setVisionNotice('已保存,立即生效');
|
|
1411
|
+
await loadState(state.kbId, true);
|
|
1412
|
+
} catch (error) {
|
|
1413
|
+
setVisionNotice(`保存失败:${error.message || error}`);
|
|
1414
|
+
} finally {
|
|
1415
|
+
setVisionSaving(false);
|
|
1416
|
+
}
|
|
1417
|
+
};
|
|
1418
|
+
|
|
1419
|
+
const testVision = async () => {
|
|
1420
|
+
if (!visionForm || visionProbing) return;
|
|
1421
|
+
setVisionProbing(true);
|
|
1422
|
+
setVisionProbe(null);
|
|
1423
|
+
try {
|
|
1424
|
+
const data = await api('/vision-test', {
|
|
1425
|
+
method: 'POST',
|
|
1426
|
+
headers: { 'content-type': 'application/json' },
|
|
1427
|
+
body: JSON.stringify({ baseUrl: visionForm.baseUrl, model: visionForm.model, apiKey: visionForm.apiKey }),
|
|
1428
|
+
});
|
|
1429
|
+
setVisionProbe(data.probe || { ok: false, error: '没有返回结果' });
|
|
1430
|
+
} catch (error) {
|
|
1431
|
+
setVisionProbe({ ok: false, error: error.message || String(error) });
|
|
1432
|
+
} finally {
|
|
1433
|
+
setVisionProbing(false);
|
|
1434
|
+
}
|
|
1435
|
+
};
|
|
1436
|
+
|
|
1378
1437
|
const saveEmbedding = async () => {
|
|
1379
1438
|
if (!form || saving) return;
|
|
1380
1439
|
setSaving(true);
|
|
@@ -1539,6 +1598,72 @@ window.__ModuleLoader__.load({
|
|
|
1539
1598
|
embedding?.enabled
|
|
1540
1599
|
? h('p', { className: 'dsh-kb-hint' }, `当前状态:已启用 · 模型 ${embedding.model} · 检索时与关键词结果融合(RRF)`)
|
|
1541
1600
|
: null),
|
|
1601
|
+
// ── 图片理解(视觉模型)──────────────────────────────────────
|
|
1602
|
+
h('section', { className: 'dsh-kb-card' },
|
|
1603
|
+
h('h4', null, '图片理解(可选)'),
|
|
1604
|
+
h('p', { className: 'dsh-kb-hint' },
|
|
1605
|
+
'图片本身没有文字,所以默认搜不到。打开这里,上传或重建时会用视觉模型把图「描述成文字」,再走同一套关键词 + 向量检索 —— 命中后 AI 也能读出图里写了什么。'),
|
|
1606
|
+
visionForm
|
|
1607
|
+
? h('div', { className: 'dsh-kb-form' },
|
|
1608
|
+
h('label', { className: 'dsh-kb-check' },
|
|
1609
|
+
h('input', {
|
|
1610
|
+
type: 'checkbox',
|
|
1611
|
+
checked: Boolean(visionForm.enabled),
|
|
1612
|
+
onChange: (event) => setVisionForm({ ...visionForm, enabled: event.target.checked }),
|
|
1613
|
+
}),
|
|
1614
|
+
'启用图片理解'),
|
|
1615
|
+
h('label', { className: 'dsh-kb-field' },
|
|
1616
|
+
h('span', { className: 'dsh-kb-label' }, '接口地址'),
|
|
1617
|
+
h('input', {
|
|
1618
|
+
type: 'text',
|
|
1619
|
+
value: visionForm.baseUrl || '',
|
|
1620
|
+
placeholder: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
1621
|
+
onChange: (event) => setVisionForm({ ...visionForm, baseUrl: event.target.value }),
|
|
1622
|
+
}),
|
|
1623
|
+
h('span', { className: 'dsh-kb-hint' }, '和向量检索一样填到 /v1 为止')),
|
|
1624
|
+
h('label', { className: 'dsh-kb-field' },
|
|
1625
|
+
h('span', { className: 'dsh-kb-label' }, '视觉模型'),
|
|
1626
|
+
h('input', {
|
|
1627
|
+
type: 'text',
|
|
1628
|
+
value: visionForm.model || '',
|
|
1629
|
+
placeholder: 'qwen3-vl-plus',
|
|
1630
|
+
onChange: (event) => setVisionForm({ ...visionForm, model: event.target.value }),
|
|
1631
|
+
}),
|
|
1632
|
+
h('span', { className: 'dsh-kb-hint' }, '百炼用 qwen3-vl-plus;OpenAI 用 gpt-4o-mini 也行')),
|
|
1633
|
+
h('label', { className: 'dsh-kb-field' },
|
|
1634
|
+
h('span', { className: 'dsh-kb-label' }, 'API Key'),
|
|
1635
|
+
h('input', {
|
|
1636
|
+
type: 'password',
|
|
1637
|
+
value: visionForm.apiKey || '',
|
|
1638
|
+
placeholder: visionForm.hasKey
|
|
1639
|
+
? `已保存(尾号 ${visionForm.keyTail}),留空表示不改`
|
|
1640
|
+
: '留空则复用向量检索那把 Key',
|
|
1641
|
+
onChange: (event) => setVisionForm({ ...visionForm, apiKey: event.target.value }),
|
|
1642
|
+
})),
|
|
1643
|
+
h('div', { className: 'dsh-kb-row2' },
|
|
1644
|
+
h('button', {
|
|
1645
|
+
type: 'button', className: 'dsh-kb-btn dsh-kb-btn--primary',
|
|
1646
|
+
onClick: saveVision, disabled: visionSaving,
|
|
1647
|
+
}, visionSaving ? '保存中…' : '保存'),
|
|
1648
|
+
h('button', {
|
|
1649
|
+
type: 'button', className: 'dsh-kb-btn',
|
|
1650
|
+
onClick: testVision, disabled: visionProbing,
|
|
1651
|
+
}, visionProbing ? '测试中…' : '测试连接')),
|
|
1652
|
+
visionProbe
|
|
1653
|
+
? h('div', { className: cx('dsh-kb-banner', visionProbe.ok ? 'dsh-kb-banner--ok' : 'dsh-kb-banner--bad'), role: 'status' },
|
|
1654
|
+
visionProbe.ok
|
|
1655
|
+
? `连接成功:${visionProbe.model} · ${visionProbe.ms} ms · 返回「${visionProbe.sample || ''}…」`
|
|
1656
|
+
: `连接失败:${visionProbe.error}`)
|
|
1657
|
+
: null,
|
|
1658
|
+
visionNotice ? h('p', { className: 'dsh-kb-hint' }, visionNotice) : null,
|
|
1659
|
+
visionForm.enabled && !current.caps?.vision?.enabled
|
|
1660
|
+
? h('p', { className: 'dsh-kb-hint' }, '已勾选但还没生效:检查接口地址和模型,保存后这里会显示「已启用」。')
|
|
1661
|
+
: null)
|
|
1662
|
+
: h('p', { className: 'dsh-kb-hint' }, '读取配置中…'),
|
|
1663
|
+
current.caps?.vision?.enabled
|
|
1664
|
+
? h('p', { className: 'dsh-kb-hint' },
|
|
1665
|
+
`当前状态:已启用 · 模型 ${current.caps.vision.model} · 新建或重建索引时会把图片描述成文字`)
|
|
1666
|
+
: h('p', { className: 'dsh-kb-hint' }, '当前状态:未启用(图片只能看,搜不到)')),
|
|
1542
1667
|
current.error
|
|
1543
1668
|
? h('p', { className: 'dsh-kb-banner dsh-kb-banner--bad', role: 'alert' }, current.error)
|
|
1544
1669
|
: null,
|
package/docs/DETAIL.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
# 个人知识库
|
|
2
2
|
|
|
3
3
|
把本机文件整理成**树状知识库**:上传后能在应用里直接读,AI 可以检索它,也可以针对某一篇文档对话。文件只存在这台电脑上(默认 `~/.dsh/kb`,可配置)。
|
|
4
4
|
|
|
@@ -50,6 +50,19 @@
|
|
|
50
50
|
|
|
51
51
|
检索记录写在本机 `kb.db` 的 `search_log` 表里,只保留最近 2000 条,不外发、不影响检索速度(写失败也不会影响结果返回)。右上角「刷新」重新拉一次。
|
|
52
52
|
|
|
53
|
+
## 图片理解(可选)
|
|
54
|
+
|
|
55
|
+
图片没有文字层,默认只能看、搜不到。设置里打开「图片理解」后,上传或重建索引时会调用视觉模型(默认百炼 qwen3-vl-plus,可换成任意 OpenAI 兼容的视觉接口)把图片描述成四段文字:
|
|
56
|
+
|
|
57
|
+
【画面】是什么 / 【文字】逐字抄出可读文字 / 【数据】表格数值 / 【场景】用途
|
|
58
|
+
|
|
59
|
+
这段描述会当正文存进 doc_text 并切块索引,所以:
|
|
60
|
+
|
|
61
|
+
- 搜索框和 kb_search 都能命中图片
|
|
62
|
+
- AI 通过 kb_read 能读到图里写了什么,并能引用出处
|
|
63
|
+
|
|
64
|
+
默认关闭、只在本机存配置(Key 与向量检索同一把时可留空复用);关闭时图片照常能看。每张图一次模型调用,图片会发给所填的服务商。
|
|
65
|
+
|
|
53
66
|
## 已知限制
|
|
54
67
|
|
|
55
68
|
- 老版 Office(`.doc/.ppt/.xls`)能看原版式,但不抽正文,所以搜不到里面的字。
|
package/index.js
CHANGED
|
@@ -18,6 +18,7 @@ import { createPipeline } from './lib/pipeline.mjs';
|
|
|
18
18
|
import { createPreviewer } from './lib/preview.mjs';
|
|
19
19
|
import { createKnowledgeBaseTools } from './lib/tools.mjs';
|
|
20
20
|
import { createSettingsStore } from './lib/settings.mjs';
|
|
21
|
+
import { createVisionClient } from './lib/vision.mjs';
|
|
21
22
|
import { createRoutes } from './lib/routes.mjs';
|
|
22
23
|
|
|
23
24
|
export const name = 'dsh-knowledge-base';
|
|
@@ -57,16 +58,30 @@ export function apply(ctx, config = {}) {
|
|
|
57
58
|
|
|
58
59
|
const embedder = createEmbedder({ resolve: resolveEmbedding, logger });
|
|
59
60
|
const index = createIndex({ store, embedder });
|
|
61
|
+
|
|
62
|
+
// 图片理解(视觉模型):把图描述成文字再索引,优先级同样 settings.json > YAML
|
|
63
|
+
const resolveVision = () => {
|
|
64
|
+
if (settings.hasSaved('vision')) {
|
|
65
|
+
const saved = settings.vision();
|
|
66
|
+
// 没单独填 key 时复用嵌入那个(通常同一个服务商)
|
|
67
|
+
const embeddingKey = settings.hasSaved() ? settings.embedding().apiKey : (config.embedding?.apiKey ?? '');
|
|
68
|
+
return { ...saved, apiKey: saved.apiKey || embeddingKey };
|
|
69
|
+
}
|
|
70
|
+
const base = config.vision ?? {};
|
|
71
|
+
return { enabled: Boolean(base.enabled), baseUrl: base.baseUrl ?? '', model: base.model ?? '', apiKey: base.apiKey ?? '' };
|
|
72
|
+
};
|
|
73
|
+
const vision = createVisionClient({ resolve: resolveVision, logger });
|
|
74
|
+
|
|
60
75
|
// Office 原版式预览要用宿主的 officeKit(LibreOffice 转换),它可能比本插件晚加载,
|
|
61
76
|
// 所以用 getter 在调用时取。
|
|
62
77
|
const previewer = createPreviewer({ store, getOfficeKit: () => ctx.get('officeKit'), logger });
|
|
63
|
-
const pipeline = createPipeline({ store, index, embedder, logger, config, previewer });
|
|
78
|
+
const pipeline = createPipeline({ store, index, embedder, vision, logger, config, previewer });
|
|
64
79
|
|
|
65
80
|
// ── HTTP 路由 ──────────────────────────────────────────────────────────
|
|
66
81
|
// 二进制预览(<img>/<iframe>)带不了自定义头,给它一条带启动令牌的通道。
|
|
67
82
|
const token = crypto.randomUUID();
|
|
68
83
|
const { routes } = createRoutes({
|
|
69
|
-
store, index, pipeline, embedder, settings, config, logger, webServer, token, previewer,
|
|
84
|
+
store, index, pipeline, embedder, vision, settings, config, logger, webServer, token, previewer,
|
|
70
85
|
});
|
|
71
86
|
for (const route of routes) {
|
|
72
87
|
ctx.effect(() => webServer.register(route), `knowledge-base: ${route.path}`);
|
package/lib/pipeline.mjs
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import fs from 'node:fs';
|
|
8
8
|
import { extract } from './extract.mjs';
|
|
9
9
|
import { renderDocument, chunkDocument } from './chunker.mjs';
|
|
10
|
+
import { describeImageFile, isVisionExtension } from './vision.mjs';
|
|
10
11
|
|
|
11
12
|
function friendly(error) {
|
|
12
13
|
const message = error instanceof Error ? error.message : String(error ?? '');
|
|
@@ -16,10 +17,12 @@ function friendly(error) {
|
|
|
16
17
|
if (/暂不支持/.test(message)) return message;
|
|
17
18
|
if (/过大/.test(message)) return message;
|
|
18
19
|
if (/ENOENT/.test(message)) return '文件不见了(可能被外部删除)';
|
|
20
|
+
if (/图片理解未启用/.test(message)) return '图片理解未启用:设置里打开「图片理解」后重建索引即可';
|
|
21
|
+
if (/HTTP 4|HTTP 5/.test(message)) return `图片理解接口报错:${message.slice(0, 200)}`;
|
|
19
22
|
return `解析失败:${message.slice(0, 300)}`;
|
|
20
23
|
}
|
|
21
24
|
|
|
22
|
-
export function createPipeline({ store, index, embedder, logger, config, previewer }) {
|
|
25
|
+
export function createPipeline({ store, index, embedder, vision, logger, config, previewer }) {
|
|
23
26
|
const queue = [];
|
|
24
27
|
const queued = new Set();
|
|
25
28
|
let active = 0;
|
|
@@ -55,6 +58,10 @@ export function createPipeline({ store, index, embedder, logger, config, preview
|
|
|
55
58
|
store.setDocStatus(nodeId, 'failed', { error: '缺少原始文件路径', updatedAt: Date.now() });
|
|
56
59
|
return;
|
|
57
60
|
}
|
|
61
|
+
if (isVisionExtension(node.ext)) {
|
|
62
|
+
await ingestImage(node, filePath);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
58
65
|
store.setDocStatus(nodeId, 'parsing', { error: null, updatedAt: Date.now() });
|
|
59
66
|
try {
|
|
60
67
|
const stat = await fs.promises.stat(filePath);
|
|
@@ -87,6 +94,47 @@ export function createPipeline({ store, index, embedder, logger, config, preview
|
|
|
87
94
|
}
|
|
88
95
|
}
|
|
89
96
|
|
|
97
|
+
/** 图片:交给视觉模型描述成文字,再走和文档一样的索引流程。 */
|
|
98
|
+
async function ingestImage(node, filePath) {
|
|
99
|
+
const nodeId = node.id;
|
|
100
|
+
if (!vision?.enabled) {
|
|
101
|
+
// 没开图片理解:保持「可看」,但不假装索引过
|
|
102
|
+
store.setDocStatus(nodeId, 'ready', {
|
|
103
|
+
error: null, pageCount: 1, chars: 0, indexedAt: Date.now(), updatedAt: Date.now(),
|
|
104
|
+
});
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
store.setDocStatus(nodeId, 'parsing', { error: null, updatedAt: Date.now() });
|
|
108
|
+
try {
|
|
109
|
+
const stat = await fs.promises.stat(filePath);
|
|
110
|
+
if (stat.size > maxFileBytes) throw new Error(`文件过大(上限 ${Math.round(maxFileBytes / 1048576)} MB)`);
|
|
111
|
+
const description = await describeImageFile(vision, filePath, node.ext);
|
|
112
|
+
const blocks = [
|
|
113
|
+
{ kind: 'heading', level: 1, text: node.name },
|
|
114
|
+
...description.split('\n').map((line) => line.trim()).filter(Boolean)
|
|
115
|
+
.map((text) => ({ kind: 'paragraph', text })),
|
|
116
|
+
];
|
|
117
|
+
const { text, lines } = renderDocument(blocks);
|
|
118
|
+
const chunks = chunkDocument(lines);
|
|
119
|
+
|
|
120
|
+
store.saveText(nodeId, text, Date.now());
|
|
121
|
+
index.insertChunks({ nodeId, kbId: node.kbId, chunks });
|
|
122
|
+
const vectorCount = await index.insertVectors({ nodeId, chunks });
|
|
123
|
+
store.setDocStatus(nodeId, 'ready', {
|
|
124
|
+
error: null,
|
|
125
|
+
pageCount: 1,
|
|
126
|
+
chars: text.length,
|
|
127
|
+
indexedAt: Date.now(),
|
|
128
|
+
updatedAt: Date.now(),
|
|
129
|
+
});
|
|
130
|
+
logger?.info?.(`[knowledge-base] described image ${node.name}: ${text.length} chars`
|
|
131
|
+
+ `${vectorCount ? `, ${vectorCount} vectors` : ''}`);
|
|
132
|
+
} catch (error) {
|
|
133
|
+
logger?.warn?.(`[knowledge-base] image describe failed for ${node.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
134
|
+
store.setDocStatus(nodeId, 'failed', { error: friendly(error), updatedAt: Date.now() });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
90
138
|
/** 启动时把上次没跑完的任务捡回来。 */
|
|
91
139
|
function recover() {
|
|
92
140
|
const rows = store.raw.q(`SELECT id FROM nodes WHERE kind = 'doc' AND status IN ('pending', 'parsing')`).all();
|
package/lib/routes.mjs
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
} from './util.mjs';
|
|
18
18
|
import { MIME_BY_EXT, SUPPORTED_EXTENSIONS, isIndexable } from './extract.mjs';
|
|
19
19
|
import { IMAGE_EXTENSIONS } from './preview.mjs';
|
|
20
|
+
import { isVisionExtension, createVisionClient } from './vision.mjs';
|
|
20
21
|
import { createEmbedder } from './index.mjs';
|
|
21
22
|
|
|
22
23
|
function fail(res, status, error) {
|
|
@@ -31,12 +32,13 @@ function safeOrigin(webServer) {
|
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
34
|
|
|
34
|
-
export function createRoutes({ store, index, pipeline, embedder, settings, config, logger, webServer, token, previewer }) {
|
|
35
|
+
export function createRoutes({ store, index, pipeline, embedder, vision, settings, config, logger, webServer, token, previewer }) {
|
|
35
36
|
const origin = safeOrigin(webServer);
|
|
36
37
|
const maxFileBytes = Number(config?.maxFileBytes) || 64 * 1024 * 1024;
|
|
37
38
|
|
|
38
39
|
const capabilities = () => ({
|
|
39
40
|
embedding: embedder?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
41
|
+
vision: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
40
42
|
officePreview: previewer?.available?.() ?? false,
|
|
41
43
|
maxFileBytes,
|
|
42
44
|
formats: [...SUPPORTED_EXTENSIONS],
|
|
@@ -134,7 +136,10 @@ export function createRoutes({ store, index, pipeline, embedder, settings, confi
|
|
|
134
136
|
|
|
135
137
|
const finalPath = path.join(store.filesDir, `${docId}.${ext}`);
|
|
136
138
|
fs.renameSync(tmpPath, finalPath);
|
|
137
|
-
|
|
139
|
+
// 图片没有文字层,只有在开了「图片理解」时才需要进队列
|
|
140
|
+
const indexed = isIndexable(ext);
|
|
141
|
+
const needsVision = isVisionExtension(ext) && Boolean(vision?.enabled);
|
|
142
|
+
const indexable = indexed || needsVision;
|
|
138
143
|
const node = store.createNode({
|
|
139
144
|
id: docId,
|
|
140
145
|
kbId,
|
|
@@ -292,31 +297,65 @@ export function createRoutes({ store, index, pipeline, embedder, settings, confi
|
|
|
292
297
|
path: `${API_BASE}/settings`,
|
|
293
298
|
handler: async (req, res) => {
|
|
294
299
|
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
}
|
|
300
|
+
const snapshot = () => ({
|
|
301
|
+
ok: true,
|
|
302
|
+
embedding: settings.publicEmbedding(config?.embedding),
|
|
303
|
+
vision: settings.publicVision(config?.vision),
|
|
304
|
+
status: embedder.describe(),
|
|
305
|
+
visionStatus: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
306
|
+
stats: store.stats(),
|
|
307
|
+
});
|
|
308
|
+
if (req.method === 'GET') return sendJson(res, 200, snapshot());
|
|
303
309
|
if (req.method !== 'POST') return fail(res, 405, '只接受 GET / POST');
|
|
304
310
|
const body = await readBody(req);
|
|
305
311
|
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
306
|
-
const
|
|
307
|
-
//
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
+
const value = body.value && typeof body.value === 'object' ? body.value : {};
|
|
313
|
+
// 支持 { embedding: {...} } / { vision: {...} };也兼容老的「直接把字段放在顶层」
|
|
314
|
+
const jobs = [];
|
|
315
|
+
if (value.embedding && typeof value.embedding === 'object') jobs.push(['embedding', value.embedding]);
|
|
316
|
+
if (value.vision && typeof value.vision === 'object') jobs.push(['vision', value.vision]);
|
|
317
|
+
if (!jobs.length) jobs.push(['embedding', value]);
|
|
318
|
+
|
|
319
|
+
for (const [section, patch] of jobs) {
|
|
320
|
+
if (patch.enabled) {
|
|
321
|
+
const baseUrl = String(patch.baseUrl ?? '').trim();
|
|
322
|
+
if (!/^https?:\/\//i.test(baseUrl)) return fail(res, 400, '接口地址要以 http:// 或 https:// 开头');
|
|
323
|
+
if (!String(patch.model ?? '').trim()) return fail(res, 400, '要填模型名');
|
|
324
|
+
}
|
|
325
|
+
const saved = section === 'vision' ? settings.saveVision(patch) : settings.saveEmbedding(patch);
|
|
326
|
+
if (!saved.ok) return fail(res, 500, '写入设置失败,检查一下磁盘权限');
|
|
312
327
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
328
|
+
return sendJson(res, 200, snapshot());
|
|
329
|
+
},
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
routes.push({
|
|
333
|
+
kind: 'exact',
|
|
334
|
+
path: `${API_BASE}/vision-test`,
|
|
335
|
+
handler: async (req, res) => {
|
|
336
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
337
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
338
|
+
const body = await readBody(req);
|
|
339
|
+
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
340
|
+
const form = body.value && typeof body.value === 'object' ? body.value : {};
|
|
341
|
+
// 用表单里正在填的值测,不必先保存
|
|
342
|
+
const saved = settings.vision();
|
|
343
|
+
// 图片理解常和向量检索同一个服务商:没单独填 key 就复用嵌入那把
|
|
344
|
+
const fallbackKey = settings.embedding().apiKey || process.env.DSH_KB_EMBEDDING_KEY || '';
|
|
345
|
+
const probeClient = createVisionClient({
|
|
346
|
+
resolve: () => ({
|
|
347
|
+
enabled: true,
|
|
348
|
+
baseUrl: typeof form.baseUrl === 'string' && form.baseUrl.trim() ? form.baseUrl : saved.baseUrl,
|
|
349
|
+
model: typeof form.model === 'string' && form.model.trim() ? form.model : saved.model,
|
|
350
|
+
apiKey: (typeof form.apiKey === 'string' && form.apiKey.trim()) ? form.apiKey.trim() : (saved.apiKey || fallbackKey),
|
|
351
|
+
}),
|
|
352
|
+
logger,
|
|
319
353
|
});
|
|
354
|
+
try {
|
|
355
|
+
return sendJson(res, 200, { ok: true, probe: await probeClient.probe() });
|
|
356
|
+
} catch (error) {
|
|
357
|
+
return sendJson(res, 200, { ok: true, probe: { ok: false, error: error instanceof Error ? error.message : String(error) } });
|
|
358
|
+
}
|
|
320
359
|
},
|
|
321
360
|
});
|
|
322
361
|
|
|
@@ -522,7 +561,14 @@ export function createRoutes({ store, index, pipeline, embedder, settings, confi
|
|
|
522
561
|
}
|
|
523
562
|
}
|
|
524
563
|
|
|
525
|
-
if (action === 'reindex' && req.method === 'POST') {
|
|
564
|
+
if (action === 'reindex' && req.method === 'POST') {
|
|
565
|
+
// 图片在有「图片理解」时也能重建(会重新描述一遍)
|
|
566
|
+
const canIndex = isIndexable(node.ext) || (isVisionExtension(node.ext) && Boolean(vision?.enabled));
|
|
567
|
+
if (!canIndex) {
|
|
568
|
+
return fail(res, 400, isVisionExtension(node.ext)
|
|
569
|
+
? '图片没有正文索引;在设置里打开「图片理解」后可以重建'
|
|
570
|
+
: '这个格式没有正文索引');
|
|
571
|
+
}
|
|
526
572
|
store.setDocStatus(node.id, 'pending', { error: null, updatedAt: nowMs() });
|
|
527
573
|
pipeline.enqueue(node.id);
|
|
528
574
|
return sendJson(res, 200, { ok: true, node: store.getNode(node.id) });
|
package/lib/settings.mjs
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
// ============================================================================
|
|
2
2
|
// 插件自己的设置文件:$DSH_HOME/kb/settings.json(0600)
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// 放两类会反复试的配置:向量检索(embeddings)和图片理解(视觉模型)。
|
|
5
|
+
// 理由:改 YAML 要重载插件才生效,而这两样用户会来回换服务、换模型、填错 key。
|
|
6
|
+
// 放这里可以即改即用,且 KEY 不落到 patch 里。
|
|
7
|
+
// 优先级:settings.json > 插件 config。
|
|
7
8
|
// ============================================================================
|
|
8
9
|
import fs from 'node:fs';
|
|
9
10
|
import path from 'node:path';
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
|
-
*
|
|
13
|
+
* 出厂的默认端点(只有地址和模型名,**没有任何密钥**)。
|
|
13
14
|
* 用户装上后在设置里只需要填 API Key + 勾选启用。
|
|
14
15
|
*/
|
|
15
16
|
export const DEFAULT_EMBEDDING = {
|
|
@@ -17,11 +18,16 @@ export const DEFAULT_EMBEDDING = {
|
|
|
17
18
|
model: 'text-embedding-v4',
|
|
18
19
|
};
|
|
19
20
|
|
|
21
|
+
export const DEFAULT_VISION = {
|
|
22
|
+
baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
23
|
+
model: 'qwen3-vl-plus',
|
|
24
|
+
};
|
|
25
|
+
|
|
20
26
|
function clean(value) {
|
|
21
27
|
return typeof value === 'string' ? value.trim() : '';
|
|
22
28
|
}
|
|
23
29
|
|
|
24
|
-
function
|
|
30
|
+
function normalizeEndpoint(input) {
|
|
25
31
|
const raw = input && typeof input === 'object' ? input : {};
|
|
26
32
|
return {
|
|
27
33
|
enabled: Boolean(raw.enabled),
|
|
@@ -56,21 +62,22 @@ export function createSettingsStore({ root, logger }) {
|
|
|
56
62
|
|
|
57
63
|
/** 读当前向量配置(未填过就是空值)。 */
|
|
58
64
|
function embedding() {
|
|
59
|
-
return
|
|
65
|
+
return normalizeEndpoint(readAll().embedding);
|
|
60
66
|
}
|
|
61
67
|
|
|
62
|
-
/**
|
|
63
|
-
function
|
|
64
|
-
|
|
68
|
+
/** 读当前图片理解配置。 */
|
|
69
|
+
function vision() {
|
|
70
|
+
return normalizeEndpoint(readAll().vision);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** 某个配置块有没有被界面保存过(没保存就回落到插件 YAML)。 */
|
|
74
|
+
function hasSaved(section = 'embedding') {
|
|
75
|
+
const raw = readAll()[section];
|
|
65
76
|
return Boolean(raw && typeof raw === 'object');
|
|
66
77
|
}
|
|
67
78
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
* @param {object} patch - { enabled?, baseUrl?, model?, apiKey?, clearKey? }
|
|
71
|
-
*/
|
|
72
|
-
function saveEmbedding(patch) {
|
|
73
|
-
const current = embedding();
|
|
79
|
+
function saveSection(section, patch) {
|
|
80
|
+
const current = normalizeEndpoint(readAll()[section]);
|
|
74
81
|
const next = { ...current };
|
|
75
82
|
if (patch && typeof patch === 'object') {
|
|
76
83
|
if ('enabled' in patch) next.enabled = Boolean(patch.enabled);
|
|
@@ -80,28 +87,62 @@ export function createSettingsStore({ root, logger }) {
|
|
|
80
87
|
else if (typeof patch.apiKey === 'string' && patch.apiKey.trim()) next.apiKey = patch.apiKey.trim();
|
|
81
88
|
}
|
|
82
89
|
const all = readAll();
|
|
83
|
-
all
|
|
90
|
+
all[section] = next;
|
|
84
91
|
const ok = writeAll(all);
|
|
85
|
-
return { ok,
|
|
92
|
+
return { ok, [section]: next };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* 保存向量配置。
|
|
97
|
+
* @param {object} patch - { enabled?, baseUrl?, model?, apiKey?, clearKey? }
|
|
98
|
+
*/
|
|
99
|
+
function saveEmbedding(patch) {
|
|
100
|
+
const result = saveSection('embedding', patch);
|
|
101
|
+
return { ok: result.ok, embedding: result.embedding };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** 保存图片理解配置。 */
|
|
105
|
+
function saveVision(patch) {
|
|
106
|
+
const result = saveSection('vision', patch);
|
|
107
|
+
return { ok: result.ok, vision: result.vision };
|
|
86
108
|
}
|
|
87
109
|
|
|
88
110
|
/** 给界面看的版本:key 只留尾号。 */
|
|
89
|
-
function
|
|
90
|
-
const saved = readAll()
|
|
91
|
-
const effective = saved && typeof saved === 'object' ?
|
|
92
|
-
const key = effective.apiKey || process.env.
|
|
111
|
+
function publicSection(section, fallback, defaults, keyFallback = '') {
|
|
112
|
+
const saved = readAll()[section];
|
|
113
|
+
const effective = saved && typeof saved === 'object' ? normalizeEndpoint(saved) : normalizeEndpoint(fallback);
|
|
114
|
+
const key = effective.apiKey || keyFallback || process.env[`DSH_KB_${section.toUpperCase()}_KEY`] || '';
|
|
93
115
|
return {
|
|
94
116
|
enabled: effective.enabled,
|
|
95
117
|
// 没填过地址/模型就回落到出厂默认,用户只需要补 Key
|
|
96
|
-
baseUrl: effective.baseUrl ||
|
|
97
|
-
model: effective.model ||
|
|
98
|
-
defaultBaseUrl:
|
|
99
|
-
defaultModel:
|
|
118
|
+
baseUrl: effective.baseUrl || defaults.baseUrl,
|
|
119
|
+
model: effective.model || defaults.model,
|
|
120
|
+
defaultBaseUrl: defaults.baseUrl,
|
|
121
|
+
defaultModel: defaults.model,
|
|
100
122
|
hasKey: Boolean(key),
|
|
101
123
|
keyTail: key ? key.slice(-4) : '',
|
|
102
124
|
fromFile: Boolean(saved && typeof saved === 'object'),
|
|
103
125
|
};
|
|
104
126
|
}
|
|
105
127
|
|
|
106
|
-
|
|
128
|
+
function publicEmbedding(fallback = {}) {
|
|
129
|
+
return publicSection('embedding', fallback, DEFAULT_EMBEDDING);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** 图片理解常和向量检索用同一个服务商,没单独填 key 时可以复用嵌入的 key。 */
|
|
133
|
+
function publicVision(fallback = {}) {
|
|
134
|
+
const embeddingKey = embedding().apiKey || process.env.DSH_KB_EMBEDDING_KEY || '';
|
|
135
|
+
return publicSection('vision', fallback, DEFAULT_VISION, embeddingKey);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
file,
|
|
140
|
+
embedding,
|
|
141
|
+
vision,
|
|
142
|
+
hasSaved,
|
|
143
|
+
saveEmbedding,
|
|
144
|
+
saveVision,
|
|
145
|
+
publicEmbedding,
|
|
146
|
+
publicVision,
|
|
147
|
+
};
|
|
107
148
|
}
|
package/lib/vision.mjs
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// 图片理解:把图片交给视觉模型,产出「结构化描述」当正文。
|
|
3
|
+
//
|
|
4
|
+
// 为什么这么做(而不是图片向量):描述文字能直接进现有的 FTS + 向量链路,
|
|
5
|
+
// 检索命中后 AI 还能通过 kb_read 读到图里写了什么,出处可解释。
|
|
6
|
+
// 失败时就当作没有正文,不影响图片本身的查看。
|
|
7
|
+
// ============================================================================
|
|
8
|
+
import fs from 'node:fs';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
|
|
11
|
+
export const VISION_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']);
|
|
12
|
+
|
|
13
|
+
const MIME_BY_EXT = {
|
|
14
|
+
png: 'image/png',
|
|
15
|
+
jpg: 'image/jpeg',
|
|
16
|
+
jpeg: 'image/jpeg',
|
|
17
|
+
gif: 'image/gif',
|
|
18
|
+
webp: 'image/webp',
|
|
19
|
+
bmp: 'image/bmp',
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
const PROMPT = [
|
|
23
|
+
'请把这张图片转成便于全文检索的文字,用中文,按下面的小标题输出,没有的部分写「无」:',
|
|
24
|
+
'【画面】一句话说清这是什么(照片 / 截图 / 表格 / 示意图 / 扫描件 …),主体是什么。',
|
|
25
|
+
'【文字】逐字抄出图中所有可读的文字、标题、按钮、标注;保持原有顺序。',
|
|
26
|
+
'【数据】如果图里有表格、清单、数值、日期、编号,按行整理出来。',
|
|
27
|
+
'【场景】这张图大概用在什么场景(培训材料 / 会议 / 资质文件 / 现场照片 …)。',
|
|
28
|
+
'不要编造看不到的内容;看不清的字写「(字迹不清)」。',
|
|
29
|
+
].join('\n');
|
|
30
|
+
|
|
31
|
+
// 64×64 的浅灰格纹 PNG,用于「测试连接」。
|
|
32
|
+
// 不能用 1×1:百炼的视觉模型要求宽高都大于 10,太小会直接 400。
|
|
33
|
+
const PROBE_PNG = 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAgElEQVR4nO3RoQ3AAAzAsP7/anH5jhiwpoUEW8rc3e5+t8MFLztc0AEt6IAWdEALOqAFHdCCDmhBB7SgA1rQAS3ogBZ0QAs6oAUd0IIOaEEHtKADWtABLeiAFnRACzqgBR3Qgg5oQQe0oANa0AEt6IAWdEALOqAFHdCCDmjB3w880FECpN+biKgAAAAASUVORK5CYII=';
|
|
34
|
+
|
|
35
|
+
const TIMEOUT_MS = 90_000;
|
|
36
|
+
|
|
37
|
+
function textOf(content) {
|
|
38
|
+
if (typeof content === 'string') return content.trim();
|
|
39
|
+
if (Array.isArray(content)) {
|
|
40
|
+
return content
|
|
41
|
+
.map((part) => (typeof part === 'string' ? part : part?.text ?? ''))
|
|
42
|
+
.join('\n')
|
|
43
|
+
.trim();
|
|
44
|
+
}
|
|
45
|
+
return '';
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* OpenAI 兼容的视觉客户端(chat/completions + image_url)。
|
|
50
|
+
* 配置每次调用都重新读,界面上改完即时生效。
|
|
51
|
+
*/
|
|
52
|
+
export function createVisionClient({ resolve, logger }) {
|
|
53
|
+
function effective() {
|
|
54
|
+
const raw = (typeof resolve === 'function' ? resolve() : {}) || {};
|
|
55
|
+
const enabled = Boolean(raw.enabled);
|
|
56
|
+
const baseUrl = String(raw.baseUrl ?? '').replace(/\/+$/, '');
|
|
57
|
+
const model = String(raw.model ?? '');
|
|
58
|
+
const key = String(raw.apiKey || process.env.DSH_KB_VISION_KEY || '');
|
|
59
|
+
return { enabled, baseUrl, model, key, usable: enabled && Boolean(baseUrl) && Boolean(model) };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function call(dataUrl) {
|
|
63
|
+
const now = effective();
|
|
64
|
+
if (!now.usable) throw new Error('图片理解未启用或缺少接口地址/模型');
|
|
65
|
+
const response = await fetch(`${now.baseUrl}/chat/completions`, {
|
|
66
|
+
method: 'POST',
|
|
67
|
+
headers: {
|
|
68
|
+
'content-type': 'application/json',
|
|
69
|
+
...(now.key ? { authorization: `Bearer ${now.key}` } : {}),
|
|
70
|
+
},
|
|
71
|
+
body: JSON.stringify({
|
|
72
|
+
model: now.model,
|
|
73
|
+
messages: [{
|
|
74
|
+
role: 'user',
|
|
75
|
+
content: [
|
|
76
|
+
{ type: 'text', text: PROMPT },
|
|
77
|
+
{ type: 'image_url', image_url: { url: dataUrl } },
|
|
78
|
+
],
|
|
79
|
+
}],
|
|
80
|
+
}),
|
|
81
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
82
|
+
});
|
|
83
|
+
if (!response.ok) {
|
|
84
|
+
const detail = await response.text().catch(() => '');
|
|
85
|
+
throw new Error(`HTTP ${response.status}${detail ? `:${detail.slice(0, 300)}` : ''}`);
|
|
86
|
+
}
|
|
87
|
+
const data = await response.json();
|
|
88
|
+
const text = textOf(data?.choices?.[0]?.message?.content);
|
|
89
|
+
if (!text) throw new Error('模型没有返回文字内容');
|
|
90
|
+
return text;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
return {
|
|
94
|
+
get enabled() {
|
|
95
|
+
return effective().usable;
|
|
96
|
+
},
|
|
97
|
+
describe() {
|
|
98
|
+
const now = effective();
|
|
99
|
+
return {
|
|
100
|
+
configured: now.enabled,
|
|
101
|
+
enabled: now.usable,
|
|
102
|
+
model: now.model,
|
|
103
|
+
baseUrl: now.baseUrl,
|
|
104
|
+
hasKey: Boolean(now.key),
|
|
105
|
+
missing: now.usable || !now.enabled ? [] : ['baseUrl', 'model'].filter((field) => !now[field]),
|
|
106
|
+
};
|
|
107
|
+
},
|
|
108
|
+
/** 给「测试连接」用:拿一张 1×1 图请求一次,能返回文字就算通。 */
|
|
109
|
+
async probe() {
|
|
110
|
+
const now = effective();
|
|
111
|
+
if (!now.usable) return { ok: false, error: '还没填完整(需要接口地址 + 模型,且已启用)' };
|
|
112
|
+
const startedAt = Date.now();
|
|
113
|
+
try {
|
|
114
|
+
const text = await call(`data:image/png;base64,${PROBE_PNG}`);
|
|
115
|
+
return { ok: true, model: now.model, ms: Date.now() - startedAt, sample: text.slice(0, 80) };
|
|
116
|
+
} catch (error) {
|
|
117
|
+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
/** 把一张图描述成文字;失败抛错,由流水线记成「图片理解失败」。 */
|
|
121
|
+
async describeImage(buffer, ext) {
|
|
122
|
+
const mime = MIME_BY_EXT[ext] ?? 'image/png';
|
|
123
|
+
return call(`data:${mime};base64,${buffer.toString('base64')}`);
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** 从文件读图片并描述(流水线用)。 */
|
|
129
|
+
export async function describeImageFile(client, filePath, ext) {
|
|
130
|
+
const buffer = await fs.promises.readFile(filePath);
|
|
131
|
+
return client.describeImage(buffer, ext);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function isVisionExtension(ext) {
|
|
135
|
+
return VISION_EXTENSIONS.has(String(ext || '').toLowerCase());
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function visionFileName(filePath) {
|
|
139
|
+
return path.basename(filePath);
|
|
140
|
+
}
|