@xiaofanzhao/dsh-plugin-knowledge-base 0.2.0 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/preview.mjs CHANGED
@@ -11,6 +11,15 @@ import path from 'node:path';
11
11
  export const OFFICE_EXTENSIONS = new Set(['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx']);
12
12
  export const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'svg']);
13
13
 
14
+ /**
15
+ * officeKit 转换的输入上限(超过会报 "The Office source exceeds maxInputBytes")。
16
+ * 50 MB 是宿主那边的实际限制;这里提前判掉,省得界面白等。
17
+ */
18
+ export const OFFICE_MAX_INPUT_BYTES = 50 * 1024 * 1024;
19
+
20
+ /** 逐页预览首屏先渲这么多页,剩下的后台继续 —— 大文档不用等全部渲染完。 */
21
+ const FIRST_PAGES = 6;
22
+
14
23
  function unavailable(message) {
15
24
  const error = new Error(message);
16
25
  error.code = 'unavailable';
@@ -61,6 +70,14 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
61
70
  const hit = cached(node, sourcePath);
62
71
  if (hit) return hit;
63
72
 
73
+ // officeKit 的转换有输入上限(超过就报 "exceeds maxInputBytes"),
74
+ // 提前判掉,省得界面在那儿转圈转到超时。
75
+ const sourceStat = fs.statSync(sourcePath);
76
+ if (sourceStat.size > OFFICE_MAX_INPUT_BYTES) {
77
+ throw unavailable(`文件 ${(sourceStat.size / 1048576).toFixed(1)} MB,超过 Office 转换上限 `
78
+ + `${Math.round(OFFICE_MAX_INPUT_BYTES / 1048576)} MB,生成不了原版式预览;可以切「文字版」或「下载」原件`);
79
+ }
80
+
64
81
  const running = jobs.get(node.id);
65
82
  if (running) return running;
66
83
 
@@ -119,6 +136,7 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
119
136
  }
120
137
 
121
138
  const setJobs = new Map();
139
+ const partialJobs = new Set();
122
140
 
123
141
  async function renderPageSet(node, spec) {
124
142
  const pdfPath = await ensure(node); // 先保证有 PDF
@@ -155,43 +173,82 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
155
173
  ...extra,
156
174
  });
157
175
 
158
- let result;
159
- let truncated = false;
176
+ const dir = setDirOf(node, spec.suffix);
177
+ const writeImages = (images) => {
178
+ fs.mkdirSync(dir, { recursive: true });
179
+ for (const [index, image] of images.entries()) {
180
+ const page = Number(image.page) || index + 1;
181
+ fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
182
+ }
183
+ };
184
+ const writeManifest = (manifest) => {
185
+ fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
186
+ };
187
+
188
+ // 第一批只渲首屏几页:大文档不用等全部渲染完就能看
189
+ let first;
160
190
  try {
161
- result = await request();
191
+ first = await request({ maxImages: Math.min(FIRST_PAGES, spec.maxPages) });
162
192
  } catch (error) {
163
193
  const message = error instanceof Error ? error.message : String(error);
164
- // 页数超过上限:退化成只渲染前 maxPages 页
165
194
  if (/maxPages/i.test(message)) {
166
- result = await request({ pages: Array.from({ length: spec.maxPages }, (_, index) => index + 1) });
167
- truncated = true;
195
+ first = await request({
196
+ maxImages: Math.min(FIRST_PAGES, spec.maxPages),
197
+ pages: Array.from({ length: Math.min(FIRST_PAGES, spec.maxPages) }, (_, index) => index + 1),
198
+ });
168
199
  } else {
169
200
  throw error;
170
201
  }
171
202
  }
172
- const images = result?.images ?? [];
173
- if (!images.length) throw new Error('渲染没有返回任何页面');
174
- const dir = setDirOf(node, spec.suffix);
203
+ const firstImages = first?.images ?? [];
204
+ if (!firstImages.length) throw new Error('渲染没有返回任何页面');
205
+
206
+ const pageCount = Number(first?.pageCount) || firstImages.length;
207
+ const renderLimit = Math.min(pageCount, spec.maxPages);
175
208
  fs.rmSync(dir, { recursive: true, force: true });
176
- fs.mkdirSync(dir, { recursive: true });
177
- images.forEach((image, index) => {
178
- const page = Number(image.page) || index + 1;
179
- fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
180
- });
181
- const manifest = {
209
+ writeImages(firstImages);
210
+
211
+ const baseManifest = {
182
212
  dpi: spec.dpi,
183
- pages: images.length,
184
- pageCount: Number(result.pageCount) || images.length,
185
- truncated,
213
+ pages: firstImages.length,
214
+ pageCount,
215
+ truncated: pageCount > renderLimit,
216
+ partial: firstImages.length < renderLimit,
186
217
  pdfMtime,
187
- width: images[0]?.width ?? 0,
188
- height: images[0]?.height ?? 0,
218
+ width: firstImages[0]?.width ?? 0,
219
+ height: firstImages[0]?.height ?? 0,
189
220
  generatedAt: Date.now(),
190
221
  };
191
- fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
192
- logger?.info?.(`[knowledge-base] rendered ${images.length} ${spec.suffix} pages for ${node.name}`
193
- + `${truncated ? ` (truncated from ${manifest.pageCount})` : ''}`);
194
- return manifest;
222
+ writeManifest(baseManifest);
223
+ logger?.info?.(`[knowledge-base] rendered first ${firstImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
224
+
225
+ // 后台补齐剩下的页,补完把 manifest 的 partial 改成 false
226
+ if (baseManifest.partial && !partialJobs.has(jobKey)) {
227
+ const total = renderLimit;
228
+ const rest = Array.from({ length: total - firstImages.length }, (_, index) => firstImages.length + index + 1);
229
+ partialJobs.add(jobKey);
230
+ (async () => {
231
+ try {
232
+ const more = await request({ maxImages: rest.length, pages: rest });
233
+ const moreImages = more?.images ?? [];
234
+ writeImages(moreImages);
235
+ writeManifest({
236
+ ...baseManifest,
237
+ pages: firstImages.length + moreImages.length,
238
+ partial: false,
239
+ generatedAt: Date.now(),
240
+ });
241
+ logger?.info?.(`[knowledge-base] finished ${firstImages.length + moreImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
242
+ } catch (error) {
243
+ logger?.warn?.(`[knowledge-base] rest pages failed for ${node.name}: `
244
+ + `${error instanceof Error ? error.message : String(error)}`);
245
+ } finally {
246
+ partialJobs.delete(jobKey);
247
+ }
248
+ })();
249
+ }
250
+
251
+ return baseManifest;
195
252
  })().finally(() => setJobs.delete(jobKey));
196
253
 
197
254
  setJobs.set(jobKey, job);
package/lib/routes.mjs CHANGED
@@ -17,7 +17,7 @@ import {
17
17
  } from './util.mjs';
18
18
  import { MIME_BY_EXT, SUPPORTED_EXTENSIONS, isIndexable } from './extract.mjs';
19
19
  import { IMAGE_EXTENSIONS } from './preview.mjs';
20
- import { isVisionExtension, createVisionClient } from './vision.mjs';
20
+ import { isVisionExtension, createVisionClient, VISION_PRESETS, presetOf } from './vision.mjs';
21
21
  import { createEmbedder } from './index.mjs';
22
22
 
23
23
  function fail(res, status, error) {
@@ -297,14 +297,18 @@ export function createRoutes({ store, index, pipeline, embedder, vision, setting
297
297
  path: `${API_BASE}/settings`,
298
298
  handler: async (req, res) => {
299
299
  if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
300
- const snapshot = () => ({
301
- ok: true,
302
- embedding: settings.publicEmbedding(config?.embedding),
303
- vision: settings.publicVision(config?.vision),
304
- status: embedder.describe(),
305
- visionStatus: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
306
- stats: store.stats(),
307
- });
300
+ const snapshot = () => {
301
+ const visionView = settings.publicVision(config?.vision);
302
+ return {
303
+ ok: true,
304
+ embedding: settings.publicEmbedding(config?.embedding),
305
+ vision: { ...visionView, preset: presetOf(visionView.baseUrl) },
306
+ visionPresets: VISION_PRESETS,
307
+ status: embedder.describe(),
308
+ visionStatus: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
309
+ stats: store.stats(),
310
+ };
311
+ };
308
312
  if (req.method === 'GET') return sendJson(res, 200, snapshot());
309
313
  if (req.method !== 'POST') return fail(res, 405, '只接受 GET / POST');
310
314
  const body = await readBody(req);
@@ -511,6 +515,7 @@ export function createRoutes({ store, index, pipeline, embedder, vision, setting
511
515
  pages: manifest.pages,
512
516
  pageCount: manifest.pageCount,
513
517
  truncated: Boolean(manifest.truncated),
518
+ partial: Boolean(manifest.partial),
514
519
  dpi: manifest.dpi,
515
520
  width: manifest.width,
516
521
  height: manifest.height,
package/lib/vision.mjs CHANGED
@@ -10,6 +10,56 @@ import path from 'node:path';
10
10
 
11
11
  export const VISION_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp']);
12
12
 
13
+ /**
14
+ * 常见的「OpenAI 兼容」视觉服务预设:选了会自动填接口地址和模型,
15
+ * 也可以选「自定义」手填任意兼容 /chat/completions 的端点。
16
+ */
17
+ export const VISION_PRESETS = [
18
+ {
19
+ id: 'dashscope',
20
+ label: '阿里云百炼',
21
+ baseUrl: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
22
+ model: 'qwen3-vl-plus',
23
+ hint: '国内直连;账号里若没有该模型,可在百炼控制台查看可用列表',
24
+ },
25
+ {
26
+ id: 'siliconflow',
27
+ label: '硅基流动',
28
+ baseUrl: 'https://api.siliconflow.cn/v1',
29
+ model: 'Qwen/Qwen2.5-VL-72B-Instruct',
30
+ hint: '有免费额度,模型名要带前缀',
31
+ },
32
+ {
33
+ id: 'openai',
34
+ label: 'OpenAI',
35
+ baseUrl: 'https://api.openai.com/v1',
36
+ model: 'gpt-4o-mini',
37
+ hint: '需要能访问外网',
38
+ },
39
+ {
40
+ id: 'ollama',
41
+ label: '本机 Ollama',
42
+ baseUrl: 'http://127.0.0.1:11434/v1',
43
+ model: 'qwen2.5vl:7b',
44
+ hint: '完全离线;key 随便填一个非空值,或留空',
45
+ },
46
+ {
47
+ id: 'custom',
48
+ label: '自定义(任意 OpenAI 兼容端点)',
49
+ baseUrl: '',
50
+ model: '',
51
+ hint: '只要支持 POST /chat/completions 且能接收 image_url 就能用',
52
+ },
53
+ ];
54
+
55
+ /** 根据 baseUrl 反推当前是哪个预设。 */
56
+ export function presetOf(baseUrl) {
57
+ const normalized = String(baseUrl || '').replace(/\/+$/, '');
58
+ if (!normalized) return 'dashscope';
59
+ const hit = VISION_PRESETS.find((preset) => preset.baseUrl && preset.baseUrl === normalized);
60
+ return hit ? hit.id : 'custom';
61
+ }
62
+
13
63
  const MIME_BY_EXT = {
14
64
  png: 'image/png',
15
65
  jpg: 'image/jpeg',
@@ -49,9 +99,10 @@ function textOf(content) {
49
99
  * OpenAI 兼容的视觉客户端(chat/completions + image_url)。
50
100
  * 配置每次调用都重新读,界面上改完即时生效。
51
101
  */
52
- export function createVisionClient({ resolve, logger }) {
102
+ export function createVisionClient({ resolve, config, logger }) {
53
103
  function effective() {
54
- const raw = (typeof resolve === 'function' ? resolve() : {}) || {};
104
+ // resolve() 每次重读(界面上改完即时生效);没有 resolve 就用静态 config
105
+ const raw = (typeof resolve === 'function' ? resolve() : config) || {};
55
106
  const enabled = Boolean(raw.enabled);
56
107
  const baseUrl = String(raw.baseUrl ?? '').replace(/\/+$/, '');
57
108
  const model = String(raw.model ?? '');
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@xiaofanzhao/dsh-plugin-knowledge-base",
3
- "version": "0.2.0",
3
+ "version": "0.2.3",
4
4
  "description": "个人知识库:树状目录、文件上传与在线查看、全文检索(含 PDF 与中文),AI 可通过工具检索并针对文档对话。",
5
5
  "type": "module",
6
6
  "main": "./index.js",