@xiaofanzhao/dsh-plugin-knowledge-base 0.2.0 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/client.js +2000 -1934
- package/lib/preview.mjs +81 -24
- package/lib/routes.mjs +14 -9
- package/lib/vision.mjs +53 -2
- package/package.json +1 -1
package/lib/preview.mjs
CHANGED
|
@@ -11,6 +11,15 @@ import path from 'node:path';
|
|
|
11
11
|
export const OFFICE_EXTENSIONS = new Set(['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx']);
|
|
12
12
|
export const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'svg']);
|
|
13
13
|
|
|
14
|
+
/**
|
|
15
|
+
* officeKit 转换的输入上限(超过会报 "The Office source exceeds maxInputBytes")。
|
|
16
|
+
* 50 MB 是宿主那边的实际限制;这里提前判掉,省得界面白等。
|
|
17
|
+
*/
|
|
18
|
+
export const OFFICE_MAX_INPUT_BYTES = 50 * 1024 * 1024;
|
|
19
|
+
|
|
20
|
+
/** 逐页预览首屏先渲这么多页,剩下的后台继续 —— 大文档不用等全部渲染完。 */
|
|
21
|
+
const FIRST_PAGES = 6;
|
|
22
|
+
|
|
14
23
|
function unavailable(message) {
|
|
15
24
|
const error = new Error(message);
|
|
16
25
|
error.code = 'unavailable';
|
|
@@ -61,6 +70,14 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
|
|
|
61
70
|
const hit = cached(node, sourcePath);
|
|
62
71
|
if (hit) return hit;
|
|
63
72
|
|
|
73
|
+
// officeKit 的转换有输入上限(超过就报 "exceeds maxInputBytes"),
|
|
74
|
+
// 提前判掉,省得界面在那儿转圈转到超时。
|
|
75
|
+
const sourceStat = fs.statSync(sourcePath);
|
|
76
|
+
if (sourceStat.size > OFFICE_MAX_INPUT_BYTES) {
|
|
77
|
+
throw unavailable(`文件 ${(sourceStat.size / 1048576).toFixed(1)} MB,超过 Office 转换上限 `
|
|
78
|
+
+ `${Math.round(OFFICE_MAX_INPUT_BYTES / 1048576)} MB,生成不了原版式预览;可以切「文字版」或「下载」原件`);
|
|
79
|
+
}
|
|
80
|
+
|
|
64
81
|
const running = jobs.get(node.id);
|
|
65
82
|
if (running) return running;
|
|
66
83
|
|
|
@@ -119,6 +136,7 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
|
|
|
119
136
|
}
|
|
120
137
|
|
|
121
138
|
const setJobs = new Map();
|
|
139
|
+
const partialJobs = new Set();
|
|
122
140
|
|
|
123
141
|
async function renderPageSet(node, spec) {
|
|
124
142
|
const pdfPath = await ensure(node); // 先保证有 PDF
|
|
@@ -155,43 +173,82 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
|
|
|
155
173
|
...extra,
|
|
156
174
|
});
|
|
157
175
|
|
|
158
|
-
|
|
159
|
-
|
|
176
|
+
const dir = setDirOf(node, spec.suffix);
|
|
177
|
+
const writeImages = (images) => {
|
|
178
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
179
|
+
for (const [index, image] of images.entries()) {
|
|
180
|
+
const page = Number(image.page) || index + 1;
|
|
181
|
+
fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
const writeManifest = (manifest) => {
|
|
185
|
+
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
// 第一批只渲首屏几页:大文档不用等全部渲染完就能看
|
|
189
|
+
let first;
|
|
160
190
|
try {
|
|
161
|
-
|
|
191
|
+
first = await request({ maxImages: Math.min(FIRST_PAGES, spec.maxPages) });
|
|
162
192
|
} catch (error) {
|
|
163
193
|
const message = error instanceof Error ? error.message : String(error);
|
|
164
|
-
// 页数超过上限:退化成只渲染前 maxPages 页
|
|
165
194
|
if (/maxPages/i.test(message)) {
|
|
166
|
-
|
|
167
|
-
|
|
195
|
+
first = await request({
|
|
196
|
+
maxImages: Math.min(FIRST_PAGES, spec.maxPages),
|
|
197
|
+
pages: Array.from({ length: Math.min(FIRST_PAGES, spec.maxPages) }, (_, index) => index + 1),
|
|
198
|
+
});
|
|
168
199
|
} else {
|
|
169
200
|
throw error;
|
|
170
201
|
}
|
|
171
202
|
}
|
|
172
|
-
const
|
|
173
|
-
if (!
|
|
174
|
-
|
|
203
|
+
const firstImages = first?.images ?? [];
|
|
204
|
+
if (!firstImages.length) throw new Error('渲染没有返回任何页面');
|
|
205
|
+
|
|
206
|
+
const pageCount = Number(first?.pageCount) || firstImages.length;
|
|
207
|
+
const renderLimit = Math.min(pageCount, spec.maxPages);
|
|
175
208
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
|
|
180
|
-
});
|
|
181
|
-
const manifest = {
|
|
209
|
+
writeImages(firstImages);
|
|
210
|
+
|
|
211
|
+
const baseManifest = {
|
|
182
212
|
dpi: spec.dpi,
|
|
183
|
-
pages:
|
|
184
|
-
pageCount
|
|
185
|
-
truncated,
|
|
213
|
+
pages: firstImages.length,
|
|
214
|
+
pageCount,
|
|
215
|
+
truncated: pageCount > renderLimit,
|
|
216
|
+
partial: firstImages.length < renderLimit,
|
|
186
217
|
pdfMtime,
|
|
187
|
-
width:
|
|
188
|
-
height:
|
|
218
|
+
width: firstImages[0]?.width ?? 0,
|
|
219
|
+
height: firstImages[0]?.height ?? 0,
|
|
189
220
|
generatedAt: Date.now(),
|
|
190
221
|
};
|
|
191
|
-
|
|
192
|
-
logger?.info?.(`[knowledge-base] rendered ${
|
|
193
|
-
|
|
194
|
-
|
|
222
|
+
writeManifest(baseManifest);
|
|
223
|
+
logger?.info?.(`[knowledge-base] rendered first ${firstImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
|
|
224
|
+
|
|
225
|
+
// 后台补齐剩下的页,补完把 manifest 的 partial 改成 false
|
|
226
|
+
if (baseManifest.partial && !partialJobs.has(jobKey)) {
|
|
227
|
+
const total = renderLimit;
|
|
228
|
+
const rest = Array.from({ length: total - firstImages.length }, (_, index) => firstImages.length + index + 1);
|
|
229
|
+
partialJobs.add(jobKey);
|
|
230
|
+
(async () => {
|
|
231
|
+
try {
|
|
232
|
+
const more = await request({ maxImages: rest.length, pages: rest });
|
|
233
|
+
const moreImages = more?.images ?? [];
|
|
234
|
+
writeImages(moreImages);
|
|
235
|
+
writeManifest({
|
|
236
|
+
...baseManifest,
|
|
237
|
+
pages: firstImages.length + moreImages.length,
|
|
238
|
+
partial: false,
|
|
239
|
+
generatedAt: Date.now(),
|
|
240
|
+
});
|
|
241
|
+
logger?.info?.(`[knowledge-base] finished ${firstImages.length + moreImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
|
|
242
|
+
} catch (error) {
|
|
243
|
+
logger?.warn?.(`[knowledge-base] rest pages failed for ${node.name}: `
|
|
244
|
+
+ `${error instanceof Error ? error.message : String(error)}`);
|
|
245
|
+
} finally {
|
|
246
|
+
partialJobs.delete(jobKey);
|
|
247
|
+
}
|
|
248
|
+
})();
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return baseManifest;
|
|
195
252
|
})().finally(() => setJobs.delete(jobKey));
|
|
196
253
|
|
|
197
254
|
setJobs.set(jobKey, job);
|
package/lib/routes.mjs
CHANGED
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
} from './util.mjs';
|
|
18
18
|
import { MIME_BY_EXT, SUPPORTED_EXTENSIONS, isIndexable } from './extract.mjs';
|
|
19
19
|
import { IMAGE_EXTENSIONS } from './preview.mjs';
|
|
20
|
-
import { isVisionExtension, createVisionClient } from './vision.mjs';
|
|
20
|
+
import { isVisionExtension, createVisionClient, VISION_PRESETS, presetOf } from './vision.mjs';
|
|
21
21
|
import { createEmbedder } from './index.mjs';
|
|
22
22
|
|
|
23
23
|
function fail(res, status, error) {
|
|
@@ -297,14 +297,18 @@ export function createRoutes({ store, index, pipeline, embedder, vision, setting
|
|
|
297
297
|
path: `${API_BASE}/settings`,
|
|
298
298
|
handler: async (req, res) => {
|
|
299
299
|
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
300
|
-
const snapshot = () =>
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
300
|
+
const snapshot = () => {
|
|
301
|
+
const visionView = settings.publicVision(config?.vision);
|
|
302
|
+
return {
|
|
303
|
+
ok: true,
|
|
304
|
+
embedding: settings.publicEmbedding(config?.embedding),
|
|
305
|
+
vision: { ...visionView, preset: presetOf(visionView.baseUrl) },
|
|
306
|
+
visionPresets: VISION_PRESETS,
|
|
307
|
+
status: embedder.describe(),
|
|
308
|
+
visionStatus: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
309
|
+
stats: store.stats(),
|
|
310
|
+
};
|
|
311
|
+
};
|
|
308
312
|
if (req.method === 'GET') return sendJson(res, 200, snapshot());
|
|
309
313
|
if (req.method !== 'POST') return fail(res, 405, '只接受 GET / POST');
|
|
310
314
|
const body = await readBody(req);
|
|
@@ -511,6 +515,7 @@ export function createRoutes({ store, index, pipeline, embedder, vision, setting
|
|
|
511
515
|
pages: manifest.pages,
|
|
512
516
|
pageCount: manifest.pageCount,
|
|
513
517
|
truncated: Boolean(manifest.truncated),
|
|
518
|
+
partial: Boolean(manifest.partial),
|
|
514
519
|
dpi: manifest.dpi,
|
|
515
520
|
width: manifest.width,
|
|
516
521
|
height: manifest.height,
|
package/lib/vision.mjs
CHANGED
|
@@ -10,6 +10,56 @@ import path from 'node:path';
|
|
|
10
10
|
|
|
11
11
|
export const VISION_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']);
|
|
12
12
|
|
|
13
|
+
/**
|
|
14
|
+
* 常见的「OpenAI 兼容」视觉服务预设:选了会自动填接口地址和模型,
|
|
15
|
+
* 也可以选「自定义」手填任意兼容 /chat/completions 的端点。
|
|
16
|
+
*/
|
|
17
|
+
export const VISION_PRESETS = [
|
|
18
|
+
{
|
|
19
|
+
id: 'dashscope',
|
|
20
|
+
label: '阿里云百炼',
|
|
21
|
+
baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
|
22
|
+
model: 'qwen3-vl-plus',
|
|
23
|
+
hint: '国内直连;账号里若没有该模型,可在百炼控制台查看可用列表',
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: 'siliconflow',
|
|
27
|
+
label: '硅基流动',
|
|
28
|
+
baseUrl: 'https://api.siliconflow.cn/v1',
|
|
29
|
+
model: 'Qwen/Qwen2.5-VL-72B-Instruct',
|
|
30
|
+
hint: '有免费额度,模型名要带前缀',
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: 'openai',
|
|
34
|
+
label: 'OpenAI',
|
|
35
|
+
baseUrl: 'https://api.openai.com/v1',
|
|
36
|
+
model: 'gpt-4o-mini',
|
|
37
|
+
hint: '需要能访问外网',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'ollama',
|
|
41
|
+
label: '本机 Ollama',
|
|
42
|
+
baseUrl: 'http://127.0.0.1:11434/v1',
|
|
43
|
+
model: 'qwen2.5vl:7b',
|
|
44
|
+
hint: '完全离线;key 随便填一个非空值,或留空',
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
id: 'custom',
|
|
48
|
+
label: '自定义(任意 OpenAI 兼容端点)',
|
|
49
|
+
baseUrl: '',
|
|
50
|
+
model: '',
|
|
51
|
+
hint: '只要支持 POST /chat/completions 且能接收 image_url 就能用',
|
|
52
|
+
},
|
|
53
|
+
];
|
|
54
|
+
|
|
55
|
+
/** 根据 baseUrl 反推当前是哪个预设。 */
|
|
56
|
+
export function presetOf(baseUrl) {
|
|
57
|
+
const normalized = String(baseUrl || '').replace(/\/+$/, '');
|
|
58
|
+
if (!normalized) return 'dashscope';
|
|
59
|
+
const hit = VISION_PRESETS.find((preset) => preset.baseUrl && preset.baseUrl === normalized);
|
|
60
|
+
return hit ? hit.id : 'custom';
|
|
61
|
+
}
|
|
62
|
+
|
|
13
63
|
const MIME_BY_EXT = {
|
|
14
64
|
png: 'image/png',
|
|
15
65
|
jpg: 'image/jpeg',
|
|
@@ -49,9 +99,10 @@ function textOf(content) {
|
|
|
49
99
|
* OpenAI 兼容的视觉客户端(chat/completions + image_url)。
|
|
50
100
|
* 配置每次调用都重新读,界面上改完即时生效。
|
|
51
101
|
*/
|
|
52
|
-
export function createVisionClient({ resolve, logger }) {
|
|
102
|
+
export function createVisionClient({ resolve, config, logger }) {
|
|
53
103
|
function effective() {
|
|
54
|
-
|
|
104
|
+
// resolve() 每次重读(界面上改完即时生效);没有 resolve 就用静态 config
|
|
105
|
+
const raw = (typeof resolve === 'function' ? resolve() : config) || {};
|
|
55
106
|
const enabled = Boolean(raw.enabled);
|
|
56
107
|
const baseUrl = String(raw.baseUrl ?? '').replace(/\/+$/, '');
|
|
57
108
|
const model = String(raw.model ?? '');
|