@xiaofanzhao/dsh-plugin-knowledge-base 0.2.1 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -1
- package/client.js +2000 -1962
- package/lib/preview.mjs +100 -34
- package/lib/routes.mjs +1 -0
- package/package.json +1 -1
package/lib/preview.mjs
CHANGED
|
@@ -11,6 +11,17 @@ import path from 'node:path';
|
|
|
11
11
|
export const OFFICE_EXTENSIONS = new Set(['doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx']);
|
|
12
12
|
export const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'svg']);
|
|
13
13
|
|
|
14
|
+
/**
|
|
15
|
+
* officeKit(DSH 的 @deepseek-ai/dsh-office-kit)默认的输入上限是 50 MB,
|
|
16
|
+
* 可以通过 profile 里 office-kit 行的 config.maxInputBytes 调大。
|
|
17
|
+
* 这里只用来把英文报错翻译成人话,不再自己做硬性拦截 ——
|
|
18
|
+
* 否则宿主把上限调大后,插件这边反而先挡住了。
|
|
19
|
+
*/
|
|
20
|
+
export const OFFICE_DEFAULT_MAX_INPUT_BYTES = 50 * 1024 * 1024;
|
|
21
|
+
|
|
22
|
+
/** 逐页预览首屏先渲这么多页,剩下的后台继续 —— 大文档不用等全部渲染完。 */
|
|
23
|
+
const FIRST_PAGES = 6;
|
|
24
|
+
|
|
14
25
|
function unavailable(message) {
|
|
15
26
|
const error = new Error(message);
|
|
16
27
|
error.code = 'unavailable';
|
|
@@ -67,16 +78,31 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
|
|
|
67
78
|
const job = (async () => {
|
|
68
79
|
const stat = fs.statSync(sourcePath);
|
|
69
80
|
const version = `${stat.size}-${Math.round(stat.mtimeMs)}`;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
let result;
|
|
82
|
+
try {
|
|
83
|
+
result = await kit.convert({
|
|
84
|
+
extension: node.ext,
|
|
85
|
+
priority,
|
|
86
|
+
source: {
|
|
87
|
+
key: `${node.id}:${version}`,
|
|
88
|
+
version,
|
|
89
|
+
bytes: stat.size,
|
|
90
|
+
read: async () => ({ bytes: new Uint8Array(await fs.promises.readFile(sourcePath)), version }),
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
} catch (error) {
|
|
94
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
95
|
+
if (/maxInputBytes/i.test(message)) {
|
|
96
|
+
throw unavailable(`文件 ${(stat.size / 1048576).toFixed(1)} MB,超过这台机器 Office 转换的输入上限`
|
|
97
|
+
+ `(office-kit 默认 ${Math.round(OFFICE_DEFAULT_MAX_INPUT_BYTES / 1048576)} MB,可在 profile 配置里调大 `
|
|
98
|
+
+ 'maxInputBytes);也可以把文件拆小,或先另存为 PDF 再上传。');
|
|
99
|
+
}
|
|
100
|
+
if (/timeout|deadline/i.test(message)) {
|
|
101
|
+
throw unavailable(`这台机器转换超时(${Math.round(stat.size / 1048576)} MB 的文件太大或太复杂);`
|
|
102
|
+
+ '可以调大 office-kit 的 timeoutMs,或先另存为 PDF 再上传。');
|
|
103
|
+
}
|
|
104
|
+
throw error;
|
|
105
|
+
}
|
|
80
106
|
const pdf = result?.pdf ?? result;
|
|
81
107
|
if (!pdf || typeof pdf.length !== 'number' || !pdf.length) {
|
|
82
108
|
throw new Error('转换没有返回 PDF 内容');
|
|
@@ -119,6 +145,7 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
|
|
|
119
145
|
}
|
|
120
146
|
|
|
121
147
|
const setJobs = new Map();
|
|
148
|
+
const partialJobs = new Set();
|
|
122
149
|
|
|
123
150
|
async function renderPageSet(node, spec) {
|
|
124
151
|
const pdfPath = await ensure(node); // 先保证有 PDF
|
|
@@ -155,43 +182,82 @@ export function createPreviewer({ store, getOfficeKit, logger }) {
|
|
|
155
182
|
...extra,
|
|
156
183
|
});
|
|
157
184
|
|
|
158
|
-
|
|
159
|
-
|
|
185
|
+
const dir = setDirOf(node, spec.suffix);
|
|
186
|
+
const writeImages = (images) => {
|
|
187
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
188
|
+
for (const [index, image] of images.entries()) {
|
|
189
|
+
const page = Number(image.page) || index + 1;
|
|
190
|
+
fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
const writeManifest = (manifest) => {
|
|
194
|
+
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest));
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
// 第一批只渲首屏几页:大文档不用等全部渲染完就能看
|
|
198
|
+
let first;
|
|
160
199
|
try {
|
|
161
|
-
|
|
200
|
+
first = await request({ maxImages: Math.min(FIRST_PAGES, spec.maxPages) });
|
|
162
201
|
} catch (error) {
|
|
163
202
|
const message = error instanceof Error ? error.message : String(error);
|
|
164
|
-
// 页数超过上限:退化成只渲染前 maxPages 页
|
|
165
203
|
if (/maxPages/i.test(message)) {
|
|
166
|
-
|
|
167
|
-
|
|
204
|
+
first = await request({
|
|
205
|
+
maxImages: Math.min(FIRST_PAGES, spec.maxPages),
|
|
206
|
+
pages: Array.from({ length: Math.min(FIRST_PAGES, spec.maxPages) }, (_, index) => index + 1),
|
|
207
|
+
});
|
|
168
208
|
} else {
|
|
169
209
|
throw error;
|
|
170
210
|
}
|
|
171
211
|
}
|
|
172
|
-
const
|
|
173
|
-
if (!
|
|
174
|
-
|
|
212
|
+
const firstImages = first?.images ?? [];
|
|
213
|
+
if (!firstImages.length) throw new Error('渲染没有返回任何页面');
|
|
214
|
+
|
|
215
|
+
const pageCount = Number(first?.pageCount) || firstImages.length;
|
|
216
|
+
const renderLimit = Math.min(pageCount, spec.maxPages);
|
|
175
217
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
fs.writeFileSync(path.join(dir, `p${page}.png`), Buffer.from(image.png));
|
|
180
|
-
});
|
|
181
|
-
const manifest = {
|
|
218
|
+
writeImages(firstImages);
|
|
219
|
+
|
|
220
|
+
const baseManifest = {
|
|
182
221
|
dpi: spec.dpi,
|
|
183
|
-
pages:
|
|
184
|
-
pageCount
|
|
185
|
-
truncated,
|
|
222
|
+
pages: firstImages.length,
|
|
223
|
+
pageCount,
|
|
224
|
+
truncated: pageCount > renderLimit,
|
|
225
|
+
partial: firstImages.length < renderLimit,
|
|
186
226
|
pdfMtime,
|
|
187
|
-
width:
|
|
188
|
-
height:
|
|
227
|
+
width: firstImages[0]?.width ?? 0,
|
|
228
|
+
height: firstImages[0]?.height ?? 0,
|
|
189
229
|
generatedAt: Date.now(),
|
|
190
230
|
};
|
|
191
|
-
|
|
192
|
-
logger?.info?.(`[knowledge-base] rendered ${
|
|
193
|
-
|
|
194
|
-
|
|
231
|
+
writeManifest(baseManifest);
|
|
232
|
+
logger?.info?.(`[knowledge-base] rendered first ${firstImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
|
|
233
|
+
|
|
234
|
+
// 后台补齐剩下的页,补完把 manifest 的 partial 改成 false
|
|
235
|
+
if (baseManifest.partial && !partialJobs.has(jobKey)) {
|
|
236
|
+
const total = renderLimit;
|
|
237
|
+
const rest = Array.from({ length: total - firstImages.length }, (_, index) => firstImages.length + index + 1);
|
|
238
|
+
partialJobs.add(jobKey);
|
|
239
|
+
(async () => {
|
|
240
|
+
try {
|
|
241
|
+
const more = await request({ maxImages: rest.length, pages: rest });
|
|
242
|
+
const moreImages = more?.images ?? [];
|
|
243
|
+
writeImages(moreImages);
|
|
244
|
+
writeManifest({
|
|
245
|
+
...baseManifest,
|
|
246
|
+
pages: firstImages.length + moreImages.length,
|
|
247
|
+
partial: false,
|
|
248
|
+
generatedAt: Date.now(),
|
|
249
|
+
});
|
|
250
|
+
logger?.info?.(`[knowledge-base] finished ${firstImages.length + moreImages.length}/${pageCount} ${spec.suffix} pages for ${node.name}`);
|
|
251
|
+
} catch (error) {
|
|
252
|
+
logger?.warn?.(`[knowledge-base] rest pages failed for ${node.name}: `
|
|
253
|
+
+ `${error instanceof Error ? error.message : String(error)}`);
|
|
254
|
+
} finally {
|
|
255
|
+
partialJobs.delete(jobKey);
|
|
256
|
+
}
|
|
257
|
+
})();
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
return baseManifest;
|
|
195
261
|
})().finally(() => setJobs.delete(jobKey));
|
|
196
262
|
|
|
197
263
|
setJobs.set(jobKey, job);
|
package/lib/routes.mjs
CHANGED
|
@@ -515,6 +515,7 @@ export function createRoutes({ store, index, pipeline, embedder, vision, setting
|
|
|
515
515
|
pages: manifest.pages,
|
|
516
516
|
pageCount: manifest.pageCount,
|
|
517
517
|
truncated: Boolean(manifest.truncated),
|
|
518
|
+
partial: Boolean(manifest.partial),
|
|
518
519
|
dpi: manifest.dpi,
|
|
519
520
|
width: manifest.width,
|
|
520
521
|
height: manifest.height,
|