@octopi-agent/engine 0.55.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +38 -0
- package/dist/config-schema/context.js +39 -0
- package/dist/config-schema/engine.d.ts +42 -0
- package/dist/config-schema/engine.js +3 -1
- package/dist/config-schema/substrate.d.ts +5 -0
- package/dist/config-schema/substrate.js +11 -0
- package/dist/config.d.ts +44 -0
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/context/capabilities/document/backends/docx.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/docx.js +67 -0
- package/dist/harness/context/capabilities/document/backends/office-parser.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/office-parser.js +76 -0
- package/dist/harness/context/capabilities/document/backends/pdf.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/pdf.js +86 -0
- package/dist/harness/context/capabilities/document/backends/plain.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/plain.js +42 -0
- package/dist/harness/context/capabilities/document/backends/sheet.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/sheet.js +90 -0
- package/dist/harness/context/capabilities/document/backends/xmind.d.ts +20 -0
- package/dist/harness/context/capabilities/document/backends/xmind.js +198 -0
- package/dist/harness/context/capabilities/document/errors.d.ts +27 -0
- package/dist/harness/context/capabilities/document/errors.js +33 -0
- package/dist/harness/context/capabilities/document/format.d.ts +38 -0
- package/dist/harness/context/capabilities/document/format.js +113 -0
- package/dist/harness/context/capabilities/document/html-to-md.d.ts +14 -0
- package/dist/harness/context/capabilities/document/html-to-md.js +97 -0
- package/dist/harness/context/capabilities/document/index.d.ts +21 -0
- package/dist/harness/context/capabilities/document/index.js +20 -0
- package/dist/harness/context/capabilities/document/legacy/factory.d.ts +13 -0
- package/dist/harness/context/capabilities/document/legacy/factory.js +23 -0
- package/dist/harness/context/capabilities/document/legacy/soffice.d.ts +32 -0
- package/dist/harness/context/capabilities/document/legacy/soffice.js +186 -0
- package/dist/harness/context/capabilities/document/port.d.ts +19 -0
- package/dist/harness/context/capabilities/document/port.js +400 -0
- package/dist/harness/context/capabilities/document/types.d.ts +149 -0
- package/dist/harness/context/capabilities/document/types.js +8 -0
- package/dist/harness/context/capabilities/index.d.ts +1 -0
- package/dist/harness/context/capabilities/index.js +1 -0
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.d.ts +64 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +155 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +4 -1
- package/dist/harness/index.d.ts +3 -3
- package/dist/harness/index.js +2 -2
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/db.d.ts +4 -0
- package/dist/harness/knowledge/db.js +58 -3
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +25 -0
- package/dist/harness/knowledge/extract-document.d.ts +17 -0
- package/dist/harness/knowledge/extract-document.js +62 -0
- package/dist/harness/knowledge/index-store.d.ts +49 -5
- package/dist/harness/knowledge/index-store.js +240 -32
- package/dist/harness/knowledge/index.d.ts +2 -0
- package/dist/harness/knowledge/index.js +1 -0
- package/dist/harness/knowledge/ingest.d.ts +123 -2
- package/dist/harness/knowledge/ingest.js +661 -59
- package/dist/harness/knowledge/retriever.d.ts +8 -0
- package/dist/harness/knowledge/retriever.js +33 -1
- package/dist/harness/knowledge/source-store.d.ts +4 -0
- package/dist/harness/knowledge/source-store.js +62 -6
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
- package/dist/harness/memory/sqlite/keyword-search.js +40 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/extract.d.ts +3 -2
- package/dist/harness/session/attachments/extract.js +19 -6
- package/dist/harness/session/attachments/service.d.ts +5 -1
- package/dist/harness/session/attachments/service.js +44 -2
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/package.json +8 -3
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { LegacyConverter } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* 按配置创建 legacy 转换器;converter=none 返回 null
|
|
4
|
+
*
|
|
5
|
+
* @param options - converter / sofficePath / timeoutMs
|
|
6
|
+
* @returns LegacyConverter | null
|
|
7
|
+
*/
|
|
8
|
+
export declare function createLegacyConverterFromConfig(options: {
|
|
9
|
+
converter?: 'none' | 'soffice' | 'remote' | null;
|
|
10
|
+
sofficePath?: string | null;
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
cacheDir?: string | null;
|
|
13
|
+
}): LegacyConverter | null;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LegacyConverter 探测失败 / 配置工厂(无 soffice 时不抛)
|
|
3
|
+
*/
|
|
4
|
+
import { createSofficeLegacyConverter } from './soffice.js';
|
|
5
|
+
/**
|
|
6
|
+
* 按配置创建 legacy 转换器;converter=none 返回 null
|
|
7
|
+
*
|
|
8
|
+
* @param options - converter / sofficePath / timeoutMs
|
|
9
|
+
* @returns LegacyConverter | null
|
|
10
|
+
*/
|
|
11
|
+
export function createLegacyConverterFromConfig(options) {
|
|
12
|
+
if (!options.converter || options.converter === 'none')
|
|
13
|
+
return null;
|
|
14
|
+
if (options.converter === 'soffice') {
|
|
15
|
+
return createSofficeLegacyConverter({
|
|
16
|
+
sofficePath: options.sofficePath,
|
|
17
|
+
timeoutMs: options.timeoutMs,
|
|
18
|
+
cacheDir: options.cacheDir,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
// remote(MinerU/WPS/云转换)后续接入
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LibreOffice 老格式转换闸门(LegacyConverter)
|
|
3
|
+
*
|
|
4
|
+
* 将 .doc/.xls/.ppt 转为 OOXML 后走 T0 抽取;不常驻、用完即退。
|
|
5
|
+
*
|
|
6
|
+
* @module harness/context/capabilities/document/legacy/soffice
|
|
7
|
+
*/
|
|
8
|
+
import type { LegacyConverter } from '../types.js';
|
|
9
|
+
export interface SofficeLegacyOptions {
|
|
10
|
+
/** soffice 可执行文件;缺省 PATH 上的 soffice */
|
|
11
|
+
sofficePath?: string | null;
|
|
12
|
+
/** 转换超时 ms(默认 60000) */
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
/** 最大输入字节(默认 50MB) */
|
|
15
|
+
maxInputBytes?: number;
|
|
16
|
+
/** 转换件缓存目录;缺省不缓存 */
|
|
17
|
+
cacheDir?: string | null;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* 创建 soffice LegacyConverter
|
|
21
|
+
*
|
|
22
|
+
* @param options - sofficePath / timeoutMs / maxInputBytes
|
|
23
|
+
* @returns LegacyConverter
|
|
24
|
+
*/
|
|
25
|
+
export declare function createSofficeLegacyConverter(options?: SofficeLegacyOptions): LegacyConverter;
|
|
26
|
+
/**
|
|
27
|
+
* 探测 soffice 是否可用(`--version`)
|
|
28
|
+
*
|
|
29
|
+
* @param bin - 可执行名或路径
|
|
30
|
+
* @param timeoutMs - 探测超时
|
|
31
|
+
*/
|
|
32
|
+
export declare function probeSoffice(bin: string, timeoutMs?: number): Promise<boolean>;
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LibreOffice 老格式转换闸门(LegacyConverter)
|
|
3
|
+
*
|
|
4
|
+
* 将 .doc/.xls/.ppt 转为 OOXML 后走 T0 抽取;不常驻、用完即退。
|
|
5
|
+
*
|
|
6
|
+
* @module harness/context/capabilities/document/legacy/soffice
|
|
7
|
+
*/
|
|
8
|
+
import { spawn } from 'node:child_process';
|
|
9
|
+
import { createHash } from 'node:crypto';
|
|
10
|
+
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
11
|
+
import { tmpdir } from 'node:os';
|
|
12
|
+
import { basename, join } from 'node:path';
|
|
13
|
+
import { DocumentExtractError } from '../errors.js';
|
|
14
|
+
const EXT_TO_TARGET = {
|
|
15
|
+
doc: 'docx',
|
|
16
|
+
xls: 'xlsx',
|
|
17
|
+
ppt: 'pptx',
|
|
18
|
+
rtf: 'docx',
|
|
19
|
+
};
|
|
20
|
+
function resolveSofficeBin(explicit) {
|
|
21
|
+
return explicit && explicit.trim() ? explicit : 'soffice';
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* 创建 soffice LegacyConverter
|
|
25
|
+
*
|
|
26
|
+
* @param options - sofficePath / timeoutMs / maxInputBytes
|
|
27
|
+
* @returns LegacyConverter
|
|
28
|
+
*/
|
|
29
|
+
export function createSofficeLegacyConverter(options = {}) {
|
|
30
|
+
const bin = resolveSofficeBin(options.sofficePath);
|
|
31
|
+
const timeoutMs = options.timeoutMs ?? 60_000;
|
|
32
|
+
const maxInputBytes = options.maxInputBytes ?? 50 * 1024 * 1024;
|
|
33
|
+
const cacheDir = options.cacheDir?.trim() || null;
|
|
34
|
+
return {
|
|
35
|
+
id: 'soffice',
|
|
36
|
+
async isAvailable() {
|
|
37
|
+
return await probeSoffice(bin, timeoutMs);
|
|
38
|
+
},
|
|
39
|
+
async convert(source, target) {
|
|
40
|
+
const data = source.data ?? (await readSourcePath(source, maxInputBytes));
|
|
41
|
+
const name = source.name ?? source.path ?? `input.${source.formatHint ?? 'bin'}`;
|
|
42
|
+
const ext = name.includes('.') ? name.slice(name.lastIndexOf('.') + 1).toLowerCase() : '';
|
|
43
|
+
const wanted = EXT_TO_TARGET[ext] ?? target;
|
|
44
|
+
const from = (source.formatHint ?? 'unknown');
|
|
45
|
+
const hash = createHash('sha256').update(data).digest('hex');
|
|
46
|
+
const cacheKey = `${hash}.${wanted}`;
|
|
47
|
+
if (cacheDir) {
|
|
48
|
+
const hit = join(cacheDir, cacheKey);
|
|
49
|
+
try {
|
|
50
|
+
const cachedBytes = await readFile(hit);
|
|
51
|
+
return {
|
|
52
|
+
data: new Uint8Array(cachedBytes),
|
|
53
|
+
meta: {
|
|
54
|
+
format: wanted,
|
|
55
|
+
legacyConversion: { from, converter: 'soffice', cached: true },
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// miss → convert
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const result = await runSofficeConvert({
|
|
64
|
+
bin,
|
|
65
|
+
data,
|
|
66
|
+
name,
|
|
67
|
+
target: wanted,
|
|
68
|
+
timeoutMs,
|
|
69
|
+
from,
|
|
70
|
+
});
|
|
71
|
+
if (cacheDir) {
|
|
72
|
+
try {
|
|
73
|
+
await mkdir(cacheDir, { recursive: true });
|
|
74
|
+
await writeFile(join(cacheDir, cacheKey), result.data);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
// cache write is best-effort
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return result;
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
async function readSourcePath(source, maxInputBytes) {
|
|
85
|
+
if (!source.path) {
|
|
86
|
+
throw new DocumentExtractError('INVALID_SOURCE', 'legacy convert requires path or data');
|
|
87
|
+
}
|
|
88
|
+
const st = await stat(source.path);
|
|
89
|
+
if (st.size > maxInputBytes) {
|
|
90
|
+
throw new DocumentExtractError('FILE_TOO_LARGE', `legacy input exceeds ${maxInputBytes} bytes`);
|
|
91
|
+
}
|
|
92
|
+
return new Uint8Array(await readFile(source.path));
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* 探测 soffice 是否可用(`--version`)
|
|
96
|
+
*
|
|
97
|
+
* @param bin - 可执行名或路径
|
|
98
|
+
* @param timeoutMs - 探测超时
|
|
99
|
+
*/
|
|
100
|
+
export async function probeSoffice(bin, timeoutMs = 5000) {
|
|
101
|
+
try {
|
|
102
|
+
const { code } = await runProcess(bin, ['--version'], timeoutMs);
|
|
103
|
+
return code === 0;
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function runSofficeConvert(args) {
|
|
110
|
+
const work = await mkdtemp(join(tmpdir(), 'octopi-soffice-'));
|
|
111
|
+
try {
|
|
112
|
+
const input = join(work, basename(args.name) || `input.${args.target}`);
|
|
113
|
+
await writeFile(input, args.data);
|
|
114
|
+
const env = {
|
|
115
|
+
...process.env,
|
|
116
|
+
SAL_USE_VCLPLUGIN: 'svp',
|
|
117
|
+
};
|
|
118
|
+
const { code, stderr } = await runProcess(args.bin, ['--headless', '--norestore', '--convert-to', args.target, '--outdir', work, input], args.timeoutMs, env);
|
|
119
|
+
if (code !== 0) {
|
|
120
|
+
throw new DocumentExtractError('INVALID_SOURCE', `soffice convert failed (${code}): ${stderr.slice(0, 200)}`);
|
|
121
|
+
}
|
|
122
|
+
const stem = basename(input).replace(/\.[^.]+$/, '');
|
|
123
|
+
const outFile = join(work, `${stem}.${args.target}`);
|
|
124
|
+
const out = await readFile(outFile);
|
|
125
|
+
return {
|
|
126
|
+
data: new Uint8Array(out),
|
|
127
|
+
meta: {
|
|
128
|
+
format: args.target,
|
|
129
|
+
legacyConversion: { from: args.from, converter: 'soffice', cached: false },
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
catch (err) {
|
|
134
|
+
if (err instanceof DocumentExtractError)
|
|
135
|
+
throw err;
|
|
136
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
137
|
+
if (/ENOENT|not found/i.test(msg)) {
|
|
138
|
+
throw new DocumentExtractError('BACKEND_UNAVAILABLE', 'soffice not found', ['legacy-converter']);
|
|
139
|
+
}
|
|
140
|
+
throw new DocumentExtractError('INVALID_SOURCE', `legacy convert failed: ${msg}`);
|
|
141
|
+
}
|
|
142
|
+
finally {
|
|
143
|
+
await rm(work, { recursive: true, force: true }).catch(() => {
|
|
144
|
+
// temp cleanup is best-effort
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
function runProcess(bin, argv, timeoutMs, env) {
|
|
149
|
+
return new Promise((resolvePromise, reject) => {
|
|
150
|
+
const child = spawn(bin, argv, {
|
|
151
|
+
env: env ?? process.env,
|
|
152
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
153
|
+
windowsHide: true,
|
|
154
|
+
});
|
|
155
|
+
let stdout = '';
|
|
156
|
+
let stderr = '';
|
|
157
|
+
let settled = false;
|
|
158
|
+
const timer = setTimeout(() => {
|
|
159
|
+
if (settled)
|
|
160
|
+
return;
|
|
161
|
+
settled = true;
|
|
162
|
+
child.kill('SIGKILL');
|
|
163
|
+
reject(new DocumentExtractError('TIMEOUT', `process timed out: ${bin}`));
|
|
164
|
+
}, timeoutMs);
|
|
165
|
+
child.stdout?.on('data', (d) => {
|
|
166
|
+
stdout += d.toString();
|
|
167
|
+
});
|
|
168
|
+
child.stderr?.on('data', (d) => {
|
|
169
|
+
stderr += d.toString();
|
|
170
|
+
});
|
|
171
|
+
child.on('error', (err) => {
|
|
172
|
+
if (settled)
|
|
173
|
+
return;
|
|
174
|
+
settled = true;
|
|
175
|
+
clearTimeout(timer);
|
|
176
|
+
reject(err);
|
|
177
|
+
});
|
|
178
|
+
child.on('close', (code) => {
|
|
179
|
+
if (settled)
|
|
180
|
+
return;
|
|
181
|
+
settled = true;
|
|
182
|
+
clearTimeout(timer);
|
|
183
|
+
resolvePromise({ code: code ?? -1, stdout, stderr });
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* createDefaultDocumentPort — DocumentPort 工厂 / 默认路由
|
|
3
|
+
*
|
|
4
|
+
* @module harness/context/capabilities/document/port
|
|
5
|
+
*/
|
|
6
|
+
import type { CreateDocumentPortOptions, DocumentExtractBackend, DocumentPort } from './types.js';
|
|
7
|
+
/**
|
|
8
|
+
* 默认 T0 后端集合(依赖缺失时 isAvailable=false)
|
|
9
|
+
*
|
|
10
|
+
* @returns 后端列表(plain / unpdf / mammoth / sheet / office-parser)
|
|
11
|
+
*/
|
|
12
|
+
export declare function createDefaultBackends(): DocumentExtractBackend[];
|
|
13
|
+
/**
|
|
14
|
+
* 创建 DocumentPort
|
|
15
|
+
*
|
|
16
|
+
* @param options - config / backends / legacyConverter
|
|
17
|
+
* @returns DocumentPort
|
|
18
|
+
*/
|
|
19
|
+
export declare function createDefaultDocumentPort(options?: CreateDocumentPortOptions): DocumentPort;
|
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* createDefaultDocumentPort — DocumentPort 工厂 / 默认路由
|
|
3
|
+
*
|
|
4
|
+
* @module harness/context/capabilities/document/port
|
|
5
|
+
*/
|
|
6
|
+
import { open, readFile, stat } from 'node:fs/promises';
|
|
7
|
+
import { resolve, sep } from 'node:path';
|
|
8
|
+
import { DocumentExtractError, isDocumentExtractError } from './errors.js';
|
|
9
|
+
import { LEGACY_FORMATS, formatFromMagic, isDocumentPath, resolveFormat } from './format.js';
|
|
10
|
+
import { docxMammothBackend } from './backends/docx.js';
|
|
11
|
+
import { officeParserBackend } from './backends/office-parser.js';
|
|
12
|
+
import { plainTextBackend } from './backends/plain.js';
|
|
13
|
+
import { pdfUnpdfBackend } from './backends/pdf.js';
|
|
14
|
+
import { sheetXlsxBackend } from './backends/sheet.js';
|
|
15
|
+
import { xmindBackend } from './backends/xmind.js';
|
|
16
|
+
const DEFAULT_CONFIG = {
|
|
17
|
+
enabled: true,
|
|
18
|
+
timeoutMs: 30_000,
|
|
19
|
+
maxFileBytes: 50 * 1024 * 1024,
|
|
20
|
+
};
|
|
21
|
+
const ALL_FORMATS = [
|
|
22
|
+
'pdf',
|
|
23
|
+
'docx',
|
|
24
|
+
'xlsx',
|
|
25
|
+
'pptx',
|
|
26
|
+
'doc',
|
|
27
|
+
'xls',
|
|
28
|
+
'ppt',
|
|
29
|
+
'rtf',
|
|
30
|
+
'odt',
|
|
31
|
+
'ods',
|
|
32
|
+
'odp',
|
|
33
|
+
'odg',
|
|
34
|
+
'html',
|
|
35
|
+
'md',
|
|
36
|
+
'csv',
|
|
37
|
+
'txt',
|
|
38
|
+
'epub',
|
|
39
|
+
'tex',
|
|
40
|
+
'xmind',
|
|
41
|
+
'unknown',
|
|
42
|
+
];
|
|
43
|
+
/**
|
|
44
|
+
* 默认 T0 后端集合(依赖缺失时 isAvailable=false)
|
|
45
|
+
*
|
|
46
|
+
* @returns 后端列表(plain / unpdf / mammoth / sheet / office-parser)
|
|
47
|
+
*/
|
|
48
|
+
export function createDefaultBackends() {
|
|
49
|
+
return [
|
|
50
|
+
plainTextBackend,
|
|
51
|
+
pdfUnpdfBackend,
|
|
52
|
+
docxMammothBackend,
|
|
53
|
+
sheetXlsxBackend,
|
|
54
|
+
officeParserBackend,
|
|
55
|
+
xmindBackend,
|
|
56
|
+
];
|
|
57
|
+
}
|
|
58
|
+
function tierOrder(tier) {
|
|
59
|
+
if (tier === 't0')
|
|
60
|
+
return ['t0'];
|
|
61
|
+
if (tier === 't1')
|
|
62
|
+
return ['t1', 't0'];
|
|
63
|
+
if (tier === 't2')
|
|
64
|
+
return ['t2', 't1', 't0'];
|
|
65
|
+
return ['t0', 't1', 't2'];
|
|
66
|
+
}
|
|
67
|
+
async function resolveBytes(source, config) {
|
|
68
|
+
const maxBytes = config.maxFileBytes ?? DEFAULT_CONFIG.maxFileBytes;
|
|
69
|
+
if (source.data) {
|
|
70
|
+
if (source.data.byteLength > maxBytes) {
|
|
71
|
+
throw new DocumentExtractError('FILE_TOO_LARGE', `document exceeds maxFileBytes (${maxBytes})`);
|
|
72
|
+
}
|
|
73
|
+
return { data: source.data, name: source.name };
|
|
74
|
+
}
|
|
75
|
+
if (!source.path) {
|
|
76
|
+
throw new DocumentExtractError('INVALID_SOURCE', 'extract source requires path or data');
|
|
77
|
+
}
|
|
78
|
+
const abs = resolve(source.path);
|
|
79
|
+
if (config.allowedRoots?.length) {
|
|
80
|
+
// Windows 路径比较须大小写不敏感(对齐 platform.isUnderAttachmentRoot)
|
|
81
|
+
const win = process.platform === 'win32';
|
|
82
|
+
const norm = (s) => (win ? resolve(s).toLowerCase() : resolve(s));
|
|
83
|
+
const p = norm(abs);
|
|
84
|
+
const ok = config.allowedRoots.some((root) => {
|
|
85
|
+
const r = norm(root);
|
|
86
|
+
const prefix = r.endsWith(sep) ? r : r + sep;
|
|
87
|
+
return p === r || p.startsWith(prefix);
|
|
88
|
+
});
|
|
89
|
+
if (!ok) {
|
|
90
|
+
throw new DocumentExtractError('PATH_FORBIDDEN', `path not under allowedRoots: ${abs}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const st = await stat(abs);
|
|
94
|
+
if (!st.isFile()) {
|
|
95
|
+
throw new DocumentExtractError('INVALID_SOURCE', `not a file: ${abs}`);
|
|
96
|
+
}
|
|
97
|
+
if (st.size > maxBytes) {
|
|
98
|
+
throw new DocumentExtractError('FILE_TOO_LARGE', `document exceeds maxFileBytes (${maxBytes})`);
|
|
99
|
+
}
|
|
100
|
+
const data = new Uint8Array(await readFile(abs));
|
|
101
|
+
return { data, name: source.name ?? abs };
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* 创建 DocumentPort
|
|
105
|
+
*
|
|
106
|
+
* @param options - config / backends / legacyConverter
|
|
107
|
+
* @returns DocumentPort
|
|
108
|
+
*/
|
|
109
|
+
export function createDefaultDocumentPort(options = {}) {
|
|
110
|
+
const config = { ...DEFAULT_CONFIG, ...options.config };
|
|
111
|
+
const backends = options.backends ?? createDefaultBackends();
|
|
112
|
+
const legacy = options.legacyConverter ?? null;
|
|
113
|
+
const capsTtl = options.capabilitiesTtlMs ?? 5 * 60_000;
|
|
114
|
+
let capsCache = null;
|
|
115
|
+
async function availableBackendIds() {
|
|
116
|
+
const ids = new Set();
|
|
117
|
+
for (const b of backends) {
|
|
118
|
+
try {
|
|
119
|
+
if (await b.isAvailable())
|
|
120
|
+
ids.add(b.id);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// 依赖探测失败 = 该后端不可用,不阻断其它后端
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return ids;
|
|
127
|
+
}
|
|
128
|
+
async function loadCapabilities(force = false) {
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
if (!force && capsCache && now - capsCache.at < capsTtl) {
|
|
131
|
+
return { caps: capsCache.value, available: capsCache.available };
|
|
132
|
+
}
|
|
133
|
+
const available = await availableBackendIds();
|
|
134
|
+
let legacyAvailable = false;
|
|
135
|
+
if (legacy) {
|
|
136
|
+
try {
|
|
137
|
+
legacyAvailable = await legacy.isAvailable();
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
legacyAvailable = false;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
const formats = {};
|
|
144
|
+
for (const f of ALL_FORMATS) {
|
|
145
|
+
if (LEGACY_FORMATS.has(f) && f !== 'xls') {
|
|
146
|
+
formats[f] = legacyAvailable
|
|
147
|
+
? { level: 'via-converter', backend: legacy?.id, notes: 'legacy gate' }
|
|
148
|
+
: { level: 'none', notes: 'requires legacy-converter' };
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
const match = backends.find((b) => b.formats.includes(f) && available.has(b.id));
|
|
152
|
+
if (match) {
|
|
153
|
+
formats[f] = {
|
|
154
|
+
level: match.tier === 't0' ? 'native' : match.tier === 't1' ? 'via-enhanced' : 'via-cloud',
|
|
155
|
+
backend: match.id,
|
|
156
|
+
};
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
// xls 可由 sheet backend 直读;若无则看转换闸门
|
|
160
|
+
if (f === 'xls' && legacyAvailable) {
|
|
161
|
+
formats[f] = { level: 'via-converter', backend: legacy?.id };
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
formats[f] = { level: 'none' };
|
|
165
|
+
}
|
|
166
|
+
const caps = {
|
|
167
|
+
formats,
|
|
168
|
+
ocr: false,
|
|
169
|
+
legacyConverter: legacy ? (legacyAvailable ? legacy.id : 'none') : 'none',
|
|
170
|
+
tiers: {
|
|
171
|
+
t0: true,
|
|
172
|
+
t1: backends.some((b) => b.tier === 't1' && available.has(b.id)),
|
|
173
|
+
t2: backends.some((b) => b.tier === 't2' && available.has(b.id)),
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
capsCache = { at: now, value: caps, available };
|
|
177
|
+
return { caps, available };
|
|
178
|
+
}
|
|
179
|
+
function pickBackends(format, available, tiers) {
|
|
180
|
+
const out = [];
|
|
181
|
+
for (const tier of tiers) {
|
|
182
|
+
for (const b of backends) {
|
|
183
|
+
if (b.tier === tier && b.formats.includes(format) && available.has(b.id)) {
|
|
184
|
+
out.push(b);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
function levelOf(backend) {
|
|
191
|
+
return backend.tier === 't0' ? 'native' : backend.tier === 't1' ? 'via-enhanced' : 'via-cloud';
|
|
192
|
+
}
|
|
193
|
+
/** 仅读文件头做嗅探(probe 不读全文件) */
|
|
194
|
+
async function readHead(path, n = 8) {
|
|
195
|
+
const handle = await open(path, 'r');
|
|
196
|
+
try {
|
|
197
|
+
const buf = Buffer.alloc(n);
|
|
198
|
+
const { bytesRead } = await handle.read(buf, 0, n, 0);
|
|
199
|
+
return new Uint8Array(buf.subarray(0, bytesRead));
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
await handle.close();
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
async capabilities() {
|
|
207
|
+
const { caps } = await loadCapabilities();
|
|
208
|
+
return caps;
|
|
209
|
+
},
|
|
210
|
+
async probe(source) {
|
|
211
|
+
let head;
|
|
212
|
+
let name = source.name ?? source.path;
|
|
213
|
+
if (source.data) {
|
|
214
|
+
head = source.data.subarray(0, 8);
|
|
215
|
+
}
|
|
216
|
+
else if (source.path) {
|
|
217
|
+
head = await readHead(resolve(source.path), 8);
|
|
218
|
+
name = source.name ?? source.path;
|
|
219
|
+
}
|
|
220
|
+
else {
|
|
221
|
+
throw new DocumentExtractError('INVALID_SOURCE', 'probe requires path or data');
|
|
222
|
+
}
|
|
223
|
+
let format = resolveFormat(source, head);
|
|
224
|
+
if (format === 'unknown')
|
|
225
|
+
format = formatFromMagic(head);
|
|
226
|
+
if (format === 'unknown' && name && isDocumentPath(name)) {
|
|
227
|
+
format = resolveFormat({ name }, head);
|
|
228
|
+
}
|
|
229
|
+
const { caps, available } = await loadCapabilities();
|
|
230
|
+
if (LEGACY_FORMATS.has(format) && format !== 'xls') {
|
|
231
|
+
return { format, level: caps.formats[format]?.level ?? 'none' };
|
|
232
|
+
}
|
|
233
|
+
const candidates = pickBackends(format, available, ['t0', 't1', 't2']);
|
|
234
|
+
const first = candidates[0];
|
|
235
|
+
return {
|
|
236
|
+
format,
|
|
237
|
+
level: first
|
|
238
|
+
? levelOf(first)
|
|
239
|
+
: (caps.formats[format]?.level ?? 'none'),
|
|
240
|
+
};
|
|
241
|
+
},
|
|
242
|
+
async extract(source, extractOptions = {}) {
|
|
243
|
+
if (config.enabled === false) {
|
|
244
|
+
throw new DocumentExtractError('BACKEND_UNAVAILABLE', 'document extract is disabled');
|
|
245
|
+
}
|
|
246
|
+
const { data, name } = await resolveBytes(source, config);
|
|
247
|
+
const head = data.subarray(0, 8);
|
|
248
|
+
let format = resolveFormat(source, head);
|
|
249
|
+
if (format === 'unknown') {
|
|
250
|
+
format = formatFromMagic(head);
|
|
251
|
+
}
|
|
252
|
+
const timeoutMs = extractOptions.timeoutMs ?? config.timeoutMs ?? DEFAULT_CONFIG.timeoutMs;
|
|
253
|
+
const warnings = [];
|
|
254
|
+
const needsLegacy = (LEGACY_FORMATS.has(format) && format !== 'xls') ||
|
|
255
|
+
(format === 'xls' && !(await hasBackendFor('xls')));
|
|
256
|
+
if (needsLegacy) {
|
|
257
|
+
if (!legacy) {
|
|
258
|
+
throw new DocumentExtractError('UNSUPPORTED_LEGACY', `legacy format .${format} requires a converter`, ['legacy-converter']);
|
|
259
|
+
}
|
|
260
|
+
let available = false;
|
|
261
|
+
try {
|
|
262
|
+
available = await legacy.isAvailable();
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
// probe failure = converter unavailable
|
|
266
|
+
available = false;
|
|
267
|
+
}
|
|
268
|
+
if (!available) {
|
|
269
|
+
throw new DocumentExtractError('UNSUPPORTED_LEGACY', `legacy converter unavailable for .${format}`, ['legacy-converter']);
|
|
270
|
+
}
|
|
271
|
+
const target = format === 'doc' ? 'docx' : format === 'xls' ? 'xlsx' : 'pptx';
|
|
272
|
+
const converted = await legacy.convert({ data, name, formatHint: format }, target);
|
|
273
|
+
return runExtract({
|
|
274
|
+
data: converted.data,
|
|
275
|
+
name: name ? name.replace(/\.[^.]+$/, `.${target}`) : undefined,
|
|
276
|
+
formatHint: target,
|
|
277
|
+
password: source.password,
|
|
278
|
+
extractOptions,
|
|
279
|
+
timeoutMs,
|
|
280
|
+
warnings,
|
|
281
|
+
legacyConversion: converted.meta.legacyConversion ?? {
|
|
282
|
+
from: format,
|
|
283
|
+
converter: legacy.id,
|
|
284
|
+
cached: false,
|
|
285
|
+
},
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
return runExtract({
|
|
289
|
+
data,
|
|
290
|
+
name,
|
|
291
|
+
formatHint: format,
|
|
292
|
+
password: source.password,
|
|
293
|
+
extractOptions,
|
|
294
|
+
timeoutMs,
|
|
295
|
+
warnings,
|
|
296
|
+
});
|
|
297
|
+
async function hasBackendFor(fmt) {
|
|
298
|
+
const { available } = await loadCapabilities();
|
|
299
|
+
return pickBackends(fmt, available, ['t0', 't1', 't2']).length > 0;
|
|
300
|
+
}
|
|
301
|
+
async function runExtract(args) {
|
|
302
|
+
const { available } = await loadCapabilities();
|
|
303
|
+
const tiers = tierOrder(args.extractOptions.tier);
|
|
304
|
+
const candidates = pickBackends(args.formatHint, available, tiers);
|
|
305
|
+
if (candidates.length === 0 && args.formatHint === 'unknown') {
|
|
306
|
+
throw new DocumentExtractError('UNSUPPORTED_FORMAT', 'unable to detect document format');
|
|
307
|
+
}
|
|
308
|
+
if (candidates.length === 0) {
|
|
309
|
+
throw new DocumentExtractError('UNSUPPORTED_FORMAT', `no available backend for format ${args.formatHint}`, args.formatHint === 'pdf' || args.formatHint === 'docx'
|
|
310
|
+
? ['optional:unpdf', 'optional:mammoth']
|
|
311
|
+
: undefined);
|
|
312
|
+
}
|
|
313
|
+
const resolved = {
|
|
314
|
+
name: args.name,
|
|
315
|
+
data: args.data,
|
|
316
|
+
formatHint: args.formatHint,
|
|
317
|
+
password: args.password,
|
|
318
|
+
};
|
|
319
|
+
let lastErr;
|
|
320
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
321
|
+
const backend = candidates[i];
|
|
322
|
+
const isFallback = i > 0;
|
|
323
|
+
try {
|
|
324
|
+
const result = await withTimeout(backend.extract(resolved, args.extractOptions), args.timeoutMs, args.extractOptions.signal);
|
|
325
|
+
return {
|
|
326
|
+
...result,
|
|
327
|
+
degraded: isFallback ? true : result.degraded,
|
|
328
|
+
warnings: [
|
|
329
|
+
...args.warnings,
|
|
330
|
+
...(isFallback
|
|
331
|
+
? [
|
|
332
|
+
{
|
|
333
|
+
code: 'DEGRADED_BACKEND',
|
|
334
|
+
message: `fell back to ${backend.id}`,
|
|
335
|
+
},
|
|
336
|
+
]
|
|
337
|
+
: []),
|
|
338
|
+
...result.warnings,
|
|
339
|
+
],
|
|
340
|
+
meta: args.legacyConversion
|
|
341
|
+
? { ...result.meta, legacyConversion: args.legacyConversion }
|
|
342
|
+
: result.meta,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
catch (err) {
|
|
346
|
+
lastErr = err;
|
|
347
|
+
// 密码/超大等结构化错误不换后端;其余失败尝试下一档
|
|
348
|
+
if (isDocumentExtractError(err) && err.code !== 'BACKEND_UNAVAILABLE') {
|
|
349
|
+
if (i === candidates.length - 1)
|
|
350
|
+
throw err;
|
|
351
|
+
}
|
|
352
|
+
if (i === candidates.length - 1) {
|
|
353
|
+
if (isDocumentExtractError(err))
|
|
354
|
+
throw err;
|
|
355
|
+
throw lastErr;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
throw isDocumentExtractError(lastErr)
|
|
360
|
+
? lastErr
|
|
361
|
+
: new DocumentExtractError('BACKEND_UNAVAILABLE', lastErr instanceof Error ? lastErr.message : String(lastErr));
|
|
362
|
+
}
|
|
363
|
+
},
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* 带超时/取消的竞速;超时后对源 promise 吞掉后续 rejection(工作无法真正取消)
|
|
368
|
+
*
|
|
369
|
+
* @param promise - 后端抽取
|
|
370
|
+
* @param timeoutMs - 超时毫秒
|
|
371
|
+
* @param signal - 可选 AbortSignal
|
|
372
|
+
* @returns 结果或 TIMEOUT 错误
|
|
373
|
+
*/
|
|
374
|
+
async function withTimeout(promise, timeoutMs, signal) {
|
|
375
|
+
if (signal?.aborted) {
|
|
376
|
+
throw new DocumentExtractError('TIMEOUT', 'extract aborted');
|
|
377
|
+
}
|
|
378
|
+
// 超时后 backend 仍可能 reject:挂 no-op catch 防 unhandledRejection
|
|
379
|
+
promise.catch(() => {
|
|
380
|
+
// abandoned after timeout — already surfaced as TIMEOUT
|
|
381
|
+
});
|
|
382
|
+
let timer;
|
|
383
|
+
try {
|
|
384
|
+
return await Promise.race([
|
|
385
|
+
promise,
|
|
386
|
+
new Promise((_r, reject) => {
|
|
387
|
+
timer = setTimeout(() => {
|
|
388
|
+
reject(new DocumentExtractError('TIMEOUT', `extract timed out after ${timeoutMs}ms`));
|
|
389
|
+
}, timeoutMs);
|
|
390
|
+
signal?.addEventListener('abort', () => {
|
|
391
|
+
reject(new DocumentExtractError('TIMEOUT', 'extract aborted'));
|
|
392
|
+
}, { once: true });
|
|
393
|
+
}),
|
|
394
|
+
]);
|
|
395
|
+
}
|
|
396
|
+
finally {
|
|
397
|
+
if (timer)
|
|
398
|
+
clearTimeout(timer);
|
|
399
|
+
}
|
|
400
|
+
}
|