@xiaofanzhao/dsh-plugin-knowledge-base 0.2.1 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/preview.mjs CHANGED
@@ -11,6 +11,17 @@ import path from 'node:path';
11
11
  export const OFFICE_EXTENSIONS = new Set(['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx']);
12
12
  export const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'svg']);
13
13
 
14
+ /**
15
+ * officeKit(DSH 的 @deepseek-ai/dsh-office-kit)默认的输入上限是 50 MB,
16
+ * 可以通过 profile 里 office-kit 行的 config.maxInputBytes 调大。
17
+ * 这里只用来把英文报错翻译成人话,不再自己做硬性拦截 ——
18
+ * 否则宿主把上限调大后,插件这边反而先挡住了。
19
+ */
20
+ export const OFFICE_DEFAULT_MAX_INPUT_BYTES = 50 * 1024 * 1024;
21
+
22
+ /** 逐页预览首屏先渲这么多页,剩下的后台继续 —— 大文档不用等全部渲染完。 */
23
+ const FIRST_PAGES = 6;
24
+
14
25
  function unavailable(message) {
15
26
  const error = new Error(message);
16
27
  error.code = 'unavailable';
@@ -67,16 +78,31 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
67
78
  const job = (async () => {
68
79
  const stat = fs.statSync(sourcePath);
69
80
  const version = `${stat.size}-${Math.round(stat.mtimeMs)}`;
70
- const result = await kit.convert({
71
- extension: node.ext,
72
- priority,
73
- source: {
74
- key: `${node.id}:${version}`,
75
- version,
76
- bytes: stat.size,
77
- read: async () => ({ bytes: new Uint8Array(await fs.promises.readFile(sourcePath)), version }),
78
- },
79
- });
81
+ let result;
82
+ try {
83
+ result = await kit.convert({
84
+ extension: node.ext,
85
+ priority,
86
+ source: {
87
+ key: `${node.id}:${version}`,
88
+ version,
89
+ bytes: stat.size,
90
+ read: async () => ({ bytes: new Uint8Array(await fs.promises.readFile(sourcePath)), version }),
91
+ },
92
+ });
93
+ } catch (error) {
94
+ const message = error instanceof Error ? error.message : String(error);
95
+ if (/maxInputBytes/i.test(message)) {
96
+ throw unavailable(`文件 ${(stat.size / 1048576).toFixed(1)} MB,超过这台机器 Office 转换的输入上限`
97
+ + `(office-kit 默认 ${Math.round(OFFICE_DEFAULT_MAX_INPUT_BYTES / 1048576)} MB,可在 profile 配置里调大 `
98
+ + 'maxInputBytes);也可以把文件拆小,或先另存为 PDF 再上传。');
99
+ }
100
+ if (/timeout|deadline/i.test(message)) {
101
+ throw unavailable(`这台机器转换超时(${Math.round(stat.size / 1048576)} MB 的文件太大或太复杂);`
102
+ + '可以调大 office-kit 的 timeoutMs,或先另存为 PDF 再上传。');
103
+ }
104
+ throw error;
105
+ }
80
106
  const pdf = result?.pdf ?? result;
81
107
  if (!pdf || typeof pdf.length !== 'number' || !pdf.length) {
82
108
  throw new Error('转换没有返回 PDF 内容');
@@ -119,6 +145,7 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
119
145
  }
120
146
 
121
147
  const setJobs = new Map();
148
+ const partialJobs = new Set();
122
149
 
123
150
  async function renderPageSet(node, spec) {
124
151
  const pdfPath = await ensure(node); // 先保证有 PDF
@@ -155,43 +182,82 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
155
182
  ...extra,
156
183
  });
157
184
 
158
- let result;
159
- let truncated = false;
185
+ const dir = setDirOf(node, spec.suffix);
186
+ const writeImages = (images) => {
187
+ fs.mkdirSync(dir, { recursive: true });
188
+ for (const [index, image] of images.entries()) {
189
+ const page = Number(image.page) || index + 1;
190
+ fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
191
+ }
192
+ };
193
+ const writeManifest = (manifest) => {
194
+ fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
195
+ };
196
+
197
+ // 第一批只渲首屏几页:大文档不用等全部渲染完就能看
198
+ let first;
160
199
  try {
161
- result = await request();
200
+ first = await request({ maxImages: Math.min(FIRST_PAGES, spec.maxPages) });
162
201
  } catch (error) {
163
202
  const message = error instanceof Error ? error.message : String(error);
164
- // 页数超过上限:退化成只渲染前 maxPages 页
165
203
  if (/maxPages/i.test(message)) {
166
- result = await request({ pages: Array.from({ length: spec.maxPages }, (_, index) => index + 1) });
167
- truncated = true;
204
+ first = await request({
205
+ maxImages: Math.min(FIRST_PAGES, spec.maxPages),
206
+ pages: Array.from({ length: Math.min(FIRST_PAGES, spec.maxPages) }, (_, index) => index + 1),
207
+ });
168
208
  } else {
169
209
  throw error;
170
210
  }
171
211
  }
172
- const images = result?.images ?? [];
173
- if (!images.length) throw new Error('渲染没有返回任何页面');
174
- const dir = setDirOf(node, spec.suffix);
212
+ const firstImages = first?.images ?? [];
213
+ if (!firstImages.length) throw new Error('渲染没有返回任何页面');
214
+
215
+ const pageCount = Number(first?.pageCount) || firstImages.length;
216
+ const renderLimit = Math.min(pageCount, spec.maxPages);
175
217
  fs.rmSync(dir, { recursive: true, force: true });
176
- fs.mkdirSync(dir, { recursive: true });
177
- images.forEach((image, index) => {
178
- const page = Number(image.page) || index + 1;
179
- fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
180
- });
181
- const manifest = {
218
+ writeImages(firstImages);
219
+
220
+ const baseManifest = {
182
221
  dpi: spec.dpi,
183
- pages: images.length,
184
- pageCount: Number(result.pageCount) || images.length,
185
- truncated,
222
+ pages: firstImages.length,
223
+ pageCount,
224
+ truncated: pageCount > renderLimit,
225
+ partial: firstImages.length < renderLimit,
186
226
  pdfMtime,
187
- width: images[0]?.width ?? 0,
188
- height: images[0]?.height ?? 0,
227
+ width: firstImages[0]?.width ?? 0,
228
+ height: firstImages[0]?.height ?? 0,
189
229
  generatedAt: Date.now(),
190
230
  };
191
- fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
192
- logger?.info?.(`[knowledge-base] rendered ${images.length} ${spec.suffix} pages for ${node.name}`
193
- + `${truncated ? ` (truncated from ${manifest.pageCount})` : ''}`);
194
- return manifest;
231
+ writeManifest(baseManifest);
232
+ logger?.info?.(`[knowledge-base] rendered first ${firstImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
233
+
234
+ // 后台补齐剩下的页,补完把 manifest 的 partial 改成 false
235
+ if (baseManifest.partial && !partialJobs.has(jobKey)) {
236
+ const total = renderLimit;
237
+ const rest = Array.from({ length: total - firstImages.length }, (_, index) => firstImages.length + index + 1);
238
+ partialJobs.add(jobKey);
239
+ (async () => {
240
+ try {
241
+ const more = await request({ maxImages: rest.length, pages: rest });
242
+ const moreImages = more?.images ?? [];
243
+ writeImages(moreImages);
244
+ writeManifest({
245
+ ...baseManifest,
246
+ pages: firstImages.length + moreImages.length,
247
+ partial: false,
248
+ generatedAt: Date.now(),
249
+ });
250
+ logger?.info?.(`[knowledge-base] finished ${firstImages.length + moreImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
251
+ } catch (error) {
252
+ logger?.warn?.(`[knowledge-base] rest pages failed for ${node.name}: `
253
+ + `${error instanceof Error ? error.message : String(error)}`);
254
+ } finally {
255
+ partialJobs.delete(jobKey);
256
+ }
257
+ })();
258
+ }
259
+
260
+ return baseManifest;
195
261
  })().finally(() => setJobs.delete(jobKey));
196
262
 
197
263
  setJobs.set(jobKey, job);
package/lib/routes.mjs CHANGED
@@ -515,6 +515,7 @@ export function createRoutes({ store, index, pipeline, embedder, vision, setting
515
515
  pages: manifest.pages,
516
516
  pageCount: manifest.pageCount,
517
517
  truncated: Boolean(manifest.truncated),
518
+ partial: Boolean(manifest.partial),
518
519
  dpi: manifest.dpi,
519
520
  width: manifest.width,
520
521
  height: manifest.height,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@xiaofanzhao/dsh-plugin-knowledge-base",
3
- "version": "0.2.1",
3
+ "version": "0.2.4",
4
4
  "description": "个人知识库:树状目录、文件上传与在线查看、全文检索(含 PDF 与中文),AI 可通过工具检索并针对文档对话。",
5
5
  "type": "module",
6
6
  "main": "./index.js",