@octopi-agent/engine 0.58.0 → 0.70.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +122 -1
- package/dist/config-schema/engine.js +4 -1
- package/dist/config-schema/substrate.d.ts +123 -0
- package/dist/config-schema/substrate.js +161 -0
- package/dist/config.d.ts +101 -1
- package/dist/harness/agent/builder.d.ts +15 -1
- package/dist/harness/agent/builder.js +41 -4
- package/dist/harness/agent/config-bridge.js +17 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/capabilities/summary/gate.js +2 -2
- package/dist/harness/context/capabilities/summary/policies.js +5 -5
- package/dist/harness/context/capabilities/summary/registry.js +29 -24
- package/dist/harness/context/layer-health.js +3 -2
- package/dist/harness/context/layers.d.ts +27 -13
- package/dist/harness/context/layers.js +76 -66
- package/dist/harness/context/system-prompt-assembler.d.ts +3 -0
- package/dist/harness/context/system-prompt-assembler.js +11 -1
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.d.ts +4 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.js +3 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.d.ts +56 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.js +281 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/registry.d.ts +98 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/registry.js +250 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/stream.d.ts +124 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/stream.js +372 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.d.ts +109 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.js +8 -0
- package/dist/harness/extension/plugin-ecosystem/index.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +1 -1
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.d.ts +11 -1
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +14 -0
- package/dist/harness/index.d.ts +8 -5
- package/dist/harness/index.js +4 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +97 -19
- package/dist/harness/knowledge/source-store.js +166 -94
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/cognition-decay.d.ts +36 -0
- package/dist/harness/memory/cognition-decay.js +55 -0
- package/dist/harness/memory/cognition-gates.d.ts +60 -0
- package/dist/harness/memory/cognition-gates.js +191 -0
- package/dist/harness/memory/cognition-trigger.d.ts +30 -0
- package/dist/harness/memory/cognition-trigger.js +41 -0
- package/dist/harness/memory/cognition-types.d.ts +1 -1
- package/dist/harness/memory/cognition.d.ts +40 -8
- package/dist/harness/memory/cognition.js +432 -79
- package/dist/harness/memory/conceptualizer.d.ts +66 -0
- package/dist/harness/memory/conceptualizer.js +246 -0
- package/dist/harness/memory/index.d.ts +16 -2
- package/dist/harness/memory/index.js +11 -1
- package/dist/harness/memory/sqlite/agent-db.d.ts +9 -0
- package/dist/harness/memory/sqlite/agent-db.js +118 -15
- package/dist/harness/memory/sqlite/cognition-store.d.ts +55 -16
- package/dist/harness/memory/sqlite/cognition-store.js +591 -144
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/memory/sqlite/wisdom-store.d.ts +33 -14
- package/dist/harness/memory/sqlite/wisdom-store.js +257 -44
- package/dist/harness/memory/types.d.ts +331 -16
- package/dist/harness/memory/types.js +49 -0
- package/dist/harness/memory/wisdom-formation.d.ts +101 -0
- package/dist/harness/memory/wisdom-formation.js +304 -0
- package/dist/harness/memory/wisdom-gates.d.ts +50 -0
- package/dist/harness/memory/wisdom-gates.js +204 -0
- package/dist/harness/memory/wisdom-policy.d.ts +107 -0
- package/dist/harness/memory/wisdom-policy.js +220 -0
- package/dist/harness/memory/wisdom-types.d.ts +3 -2
- package/dist/harness/memory/wisdom-types.js +2 -2
- package/dist/harness/memory/wisdom.d.ts +27 -0
- package/dist/harness/memory/wisdom.js +181 -0
- package/dist/harness/run/agent/agent.d.ts +9 -0
- package/dist/harness/run/agent/agent.js +19 -0
- package/dist/harness/run/reliability/run-agent.js +6 -2
- package/dist/harness/run/runner.js +2 -1
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/index.d.ts +1 -0
- package/dist/harness/session/index.js +1 -0
- package/dist/harness/session/title.d.ts +86 -0
- package/dist/harness/session/title.js +268 -0
- package/dist/harness/shared/events/agent-event-map.d.ts +8 -0
- package/dist/harness/shared/events/agent-event-map.js +1 -0
- package/dist/index.d.ts +7 -3
- package/dist/index.js +3 -1
- package/dist/subsystems/cognition-steward/conceptualize/SUBSYSTEM.md +43 -0
- package/dist/subsystems/cognition-steward/conceptualize/config.yaml +50 -0
- package/dist/subsystems/cognition-steward/conceptualize/handler.d.ts +9 -0
- package/dist/subsystems/cognition-steward/conceptualize/handler.js +125 -0
- package/dist/subsystems/cognition-steward/consolidate/SUBSYSTEM.md +6 -0
- package/dist/subsystems/cognition-steward/consolidate/config.yaml +54 -0
- package/dist/subsystems/cognition-steward/consolidate/handler.d.ts +12 -0
- package/dist/subsystems/cognition-steward/consolidate/handler.js +93 -0
- package/dist/subsystems/wisdom-steward/evaluate/config.yaml +50 -0
- package/dist/subsystems/wisdom-steward/evaluate/handler.d.ts +18 -0
- package/dist/subsystems/wisdom-steward/evaluate/handler.js +91 -0
- package/dist/subsystems/wisdom-steward/formation/SUBSYSTEM.md +42 -0
- package/dist/subsystems/wisdom-steward/formation/config.yaml +54 -0
- package/dist/subsystems/wisdom-steward/formation/handler.d.ts +18 -0
- package/dist/subsystems/wisdom-steward/formation/handler.js +212 -0
- package/dist/subsystems/wisdom-steward/govern/config.yaml +55 -0
- package/dist/subsystems/wisdom-steward/govern/handler.d.ts +17 -0
- package/dist/subsystems/wisdom-steward/govern/handler.js +68 -0
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -1,25 +1,68 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Document extract worker — 在独立线程跑同步 Office/SheetJS 抽取
|
|
2
|
+
* Document extract worker — 在独立线程跑同步 Office/SheetJS 抽取 + 切块 + FTS token
|
|
3
3
|
*
|
|
4
|
-
* 避免 xlsx.read
|
|
4
|
+
* 避免 xlsx.read / 切块 / CJK 滑窗堵死 Service 事件循环。
|
|
5
|
+
* 装配走 createDocumentPortFromConfig:与 Gateway 共用 documents.* 单源。
|
|
5
6
|
*/
|
|
6
7
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
7
|
-
import {
|
|
8
|
+
import { createHash } from 'node:crypto';
|
|
9
|
+
import { createReadStream } from 'node:fs';
|
|
10
|
+
import { createDocumentPortFromConfig, } from '../capabilities/document/factory.js';
|
|
11
|
+
import { DocumentExtractError } from '../capabilities/document/errors.js';
|
|
12
|
+
import { markdownAdapter } from './adapters.js';
|
|
13
|
+
import { buildFtsTokens } from './fts.js';
|
|
14
|
+
/** 流式 SHA-256:不在 Engine/本 worker 一次性吞下整文件 Buffer */
|
|
15
|
+
async function hashFileSha256(path) {
|
|
16
|
+
const hash = createHash('sha256');
|
|
17
|
+
await new Promise((resolve, reject) => {
|
|
18
|
+
const stream = createReadStream(path);
|
|
19
|
+
stream.on('data', (chunk) => hash.update(chunk));
|
|
20
|
+
stream.once('end', () => resolve());
|
|
21
|
+
stream.once('error', reject);
|
|
22
|
+
});
|
|
23
|
+
return hash.digest('hex');
|
|
24
|
+
}
|
|
8
25
|
async function main() {
|
|
9
26
|
const job = workerData;
|
|
10
|
-
const port =
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
27
|
+
const port = createDocumentPortFromConfig(job.documentConfig);
|
|
28
|
+
const result = await port.extract({ path: job.path, name: job.name ?? job.path }, {
|
|
29
|
+
timeoutMs: job.timeoutMs ?? 60_000,
|
|
30
|
+
maxFileBytes: job.maxFileBytes,
|
|
31
|
+
maxSheets: job.maxSheets,
|
|
32
|
+
maxRowsPerSheet: job.maxRowsPerSheet,
|
|
33
|
+
maxPages: job.maxPages,
|
|
34
|
+
maxTextChars: job.maxTextChars,
|
|
16
35
|
});
|
|
17
|
-
const
|
|
18
|
-
|
|
36
|
+
const markdown = result.markdown?.trim() ?? '';
|
|
37
|
+
let chunks = [];
|
|
38
|
+
if (markdown) {
|
|
39
|
+
chunks = markdownAdapter.chunk(markdown, job.path);
|
|
40
|
+
for (const c of chunks) {
|
|
41
|
+
c.ftsToks = buildFtsTokens(c.text, job.path);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
const contentHash = await hashFileSha256(job.path);
|
|
45
|
+
parentPort?.postMessage({ ok: true, result, chunks, contentHash });
|
|
19
46
|
}
|
|
20
47
|
main().catch((err) => {
|
|
48
|
+
// 结构化错误必须跨 postMessage 存活:父进程 instanceof 会丢
|
|
49
|
+
if (err instanceof DocumentExtractError) {
|
|
50
|
+
parentPort?.postMessage({
|
|
51
|
+
ok: false,
|
|
52
|
+
error: {
|
|
53
|
+
name: 'DocumentExtractError',
|
|
54
|
+
code: err.code,
|
|
55
|
+
message: err.message,
|
|
56
|
+
requires: err.requires,
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
21
61
|
parentPort?.postMessage({
|
|
22
62
|
ok: false,
|
|
23
|
-
error:
|
|
63
|
+
error: {
|
|
64
|
+
name: err instanceof Error ? err.name : 'Error',
|
|
65
|
+
message: err instanceof Error ? err.message : String(err),
|
|
66
|
+
},
|
|
24
67
|
});
|
|
25
68
|
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EmbedRunner — Phase B 向量写入(含外发敏感策略)
|
|
3
|
+
*
|
|
4
|
+
* 从 KnowledgeIngest 拆出;不依赖 parse/watch,只吃 index + provider + 控制面回调。
|
|
5
|
+
*/
|
|
6
|
+
import type { EmbeddingProvider } from '../memory/sqlite/embedding.js';
|
|
7
|
+
import type { KnowledgeIndexStore } from './index-store.js';
|
|
8
|
+
/** embedding 外发敏感形态策略 */
|
|
9
|
+
export type EmbedSecretPolicy = 'allow' | 'redact' | 'skip';
|
|
10
|
+
export interface EmbedRunnerDeps {
|
|
11
|
+
index: KnowledgeIndexStore;
|
|
12
|
+
embedding: EmbeddingProvider | null;
|
|
13
|
+
embedBatch: number;
|
|
14
|
+
embedConcurrency: number;
|
|
15
|
+
embedMinIntervalMs: number;
|
|
16
|
+
embedSecretPolicy: EmbedSecretPolicy;
|
|
17
|
+
isAborted: (sourceId: string) => boolean;
|
|
18
|
+
heartbeatJob: (jobId: string) => void;
|
|
19
|
+
refreshCoverage: (sourceId: string) => void;
|
|
20
|
+
emitProgress: (evt: {
|
|
21
|
+
type: 'embed';
|
|
22
|
+
sourceId: string;
|
|
23
|
+
status: string;
|
|
24
|
+
detail?: string;
|
|
25
|
+
}) => void;
|
|
26
|
+
/** 密钥审计落库(knowledge_embed_secret_log) */
|
|
27
|
+
recordSecretHit: (input: {
|
|
28
|
+
sourceId: string;
|
|
29
|
+
chunkId: string;
|
|
30
|
+
action: string;
|
|
31
|
+
hits: string[];
|
|
32
|
+
}) => void;
|
|
33
|
+
}
|
|
34
|
+
export declare class EmbedRunner {
|
|
35
|
+
private readonly deps;
|
|
36
|
+
private lastEmbedAt;
|
|
37
|
+
constructor(deps: EmbedRunnerDeps);
|
|
38
|
+
/**
|
|
39
|
+
* 带硬超时的调用(批/单条)。超时后拒绝并释放槽位。
|
|
40
|
+
* 底层 HTTP 未必可取消——由 provider 负责;本层保证 **不得** 无限占住 embed 槽。
|
|
41
|
+
* (parse 路径用可 abort 的 `withTimeout`;embed 侧以槽位释放为硬约束。)
|
|
42
|
+
*/
|
|
43
|
+
private withDeadline;
|
|
44
|
+
/**
|
|
45
|
+
* Phase B:嵌入待处理 chunk
|
|
46
|
+
*
|
|
47
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
48
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
49
|
+
*/
|
|
50
|
+
embedPending(sourceId: string, maxRounds?: number, jobId?: string): Promise<void>;
|
|
51
|
+
private applyEmbedSecretPolicy;
|
|
52
|
+
private embedOneSlice;
|
|
53
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EmbedRunner — Phase B 向量写入(含外发敏感策略)
|
|
3
|
+
*
|
|
4
|
+
* 从 KnowledgeIngest 拆出;不依赖 parse/watch,只吃 index + provider + 控制面回调。
|
|
5
|
+
*/
|
|
6
|
+
import { redactSecretShapes, scanSecretShapes } from './secret-scan.js';
|
|
7
|
+
export class EmbedRunner {
|
|
8
|
+
deps;
|
|
9
|
+
lastEmbedAt = 0;
|
|
10
|
+
constructor(deps) {
|
|
11
|
+
this.deps = deps;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* 带硬超时的调用(批/单条)。超时后拒绝并释放槽位。
|
|
15
|
+
* 底层 HTTP 未必可取消——由 provider 负责;本层保证 **不得** 无限占住 embed 槽。
|
|
16
|
+
* (parse 路径用可 abort 的 `withTimeout`;embed 侧以槽位释放为硬约束。)
|
|
17
|
+
*/
|
|
18
|
+
async withDeadline(work, ms, label) {
|
|
19
|
+
let timer;
|
|
20
|
+
try {
|
|
21
|
+
return await Promise.race([
|
|
22
|
+
work(),
|
|
23
|
+
new Promise((_, reject) => {
|
|
24
|
+
timer = setTimeout(() => reject(new Error(label)), ms);
|
|
25
|
+
}),
|
|
26
|
+
]);
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
if (timer)
|
|
30
|
+
clearTimeout(timer);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Phase B:嵌入待处理 chunk
|
|
35
|
+
*
|
|
36
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
37
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
38
|
+
*/
|
|
39
|
+
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
40
|
+
const { index, embedding, isAborted, heartbeatJob, refreshCoverage, emitProgress } = this.deps;
|
|
41
|
+
if (!embedding) {
|
|
42
|
+
if (index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
|
|
43
|
+
throw new Error('embedding_provider_missing');
|
|
44
|
+
}
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const lanes = Math.max(1, this.deps.embedConcurrency);
|
|
48
|
+
const batchSize = Math.max(1, this.deps.embedBatch);
|
|
49
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
50
|
+
if (isAborted(sourceId))
|
|
51
|
+
return;
|
|
52
|
+
const pending = index
|
|
53
|
+
.listChunksMissingEmbedding(sourceId, batchSize * lanes)
|
|
54
|
+
.map((c) => ({ id: c.chunkId, text: c.text }));
|
|
55
|
+
if (pending.length === 0) {
|
|
56
|
+
refreshCoverage(sourceId);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
if (jobId)
|
|
60
|
+
heartbeatJob(jobId);
|
|
61
|
+
if (this.deps.embedMinIntervalMs > 0) {
|
|
62
|
+
const wait = this.lastEmbedAt + this.deps.embedMinIntervalMs - Date.now();
|
|
63
|
+
if (wait > 0)
|
|
64
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
65
|
+
}
|
|
66
|
+
const slices = [];
|
|
67
|
+
for (let i = 0; i < pending.length; i += batchSize) {
|
|
68
|
+
slices.push(pending.slice(i, i + batchSize));
|
|
69
|
+
}
|
|
70
|
+
const settled = await Promise.allSettled(slices.map((slice) => this.embedOneSlice(sourceId, slice)));
|
|
71
|
+
// 向量落库是同步写:批间让出,避免 Engine 线程连轴堵住 /v1/sources
|
|
72
|
+
await new Promise((r) => setImmediate(r));
|
|
73
|
+
let ok = 0;
|
|
74
|
+
let failed = false;
|
|
75
|
+
for (const s of settled) {
|
|
76
|
+
if (s.status === 'fulfilled')
|
|
77
|
+
ok += s.value;
|
|
78
|
+
else
|
|
79
|
+
failed = true;
|
|
80
|
+
}
|
|
81
|
+
this.lastEmbedAt = Date.now();
|
|
82
|
+
emitProgress({
|
|
83
|
+
type: 'embed',
|
|
84
|
+
sourceId,
|
|
85
|
+
status: 'batch_done',
|
|
86
|
+
detail: `embedded=${ok}/${pending.length} lanes=${slices.length}`,
|
|
87
|
+
});
|
|
88
|
+
if (failed)
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
refreshCoverage(sourceId);
|
|
92
|
+
}
|
|
93
|
+
applyEmbedSecretPolicy(sourceId, chunkId, text) {
|
|
94
|
+
const policy = this.deps.embedSecretPolicy;
|
|
95
|
+
if (policy === 'allow')
|
|
96
|
+
return { text, skip: false };
|
|
97
|
+
const hits = scanSecretShapes(text);
|
|
98
|
+
if (hits.length === 0)
|
|
99
|
+
return { text, skip: false };
|
|
100
|
+
const action = policy === 'skip' ? 'skipped' : 'redacted';
|
|
101
|
+
this.deps.recordSecretHit({ sourceId, chunkId, action, hits });
|
|
102
|
+
this.deps.emitProgress({
|
|
103
|
+
type: 'embed',
|
|
104
|
+
sourceId,
|
|
105
|
+
status: 'secret_policy',
|
|
106
|
+
detail: `${action} hits=${hits.join(',')}`,
|
|
107
|
+
});
|
|
108
|
+
if (policy === 'skip')
|
|
109
|
+
return { text, skip: true };
|
|
110
|
+
return { text: redactSecretShapes(text).text, skip: false };
|
|
111
|
+
}
|
|
112
|
+
async embedOneSlice(sourceId, slice) {
|
|
113
|
+
const { index, embedding, isAborted } = this.deps;
|
|
114
|
+
if (slice.length === 0 || !embedding)
|
|
115
|
+
return 0;
|
|
116
|
+
const prepared = [];
|
|
117
|
+
for (const p of slice) {
|
|
118
|
+
const r = this.applyEmbedSecretPolicy(sourceId, p.id, p.text);
|
|
119
|
+
prepared.push({ id: p.id, text: r.text, skip: r.skip });
|
|
120
|
+
}
|
|
121
|
+
const outbound = prepared.filter((p) => !p.skip);
|
|
122
|
+
for (const p of prepared) {
|
|
123
|
+
if (p.skip) {
|
|
124
|
+
index.setChunkEmbeddings([[p.id, []]]);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (outbound.length === 0)
|
|
128
|
+
return 0;
|
|
129
|
+
const timeoutMs = 60_000 + 20_000 * outbound.length;
|
|
130
|
+
let vectors = [];
|
|
131
|
+
if (!isAborted(sourceId)) {
|
|
132
|
+
try {
|
|
133
|
+
vectors = await this.withDeadline(() => embedding.embedBatch(outbound.map((p) => p.text)), timeoutMs, 'embed_batch_timeout');
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
vectors = [];
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
let ok = 0;
|
|
140
|
+
const batch = [];
|
|
141
|
+
const flush = () => {
|
|
142
|
+
if (batch.length > 0) {
|
|
143
|
+
index.setChunkEmbeddings(batch.splice(0, batch.length));
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
for (let i = 0; i < outbound.length; i++) {
|
|
147
|
+
if (isAborted(sourceId)) {
|
|
148
|
+
flush();
|
|
149
|
+
return ok;
|
|
150
|
+
}
|
|
151
|
+
let vec = vectors[i];
|
|
152
|
+
if (!vec?.length) {
|
|
153
|
+
// 单条也必须有硬超时:否则一条挂死永久占住 embed 槽
|
|
154
|
+
try {
|
|
155
|
+
vec = await this.withDeadline(() => embedding.embed(outbound[i].text), 30_000, 'embed_item_timeout');
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
vec = [];
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (vec?.length) {
|
|
162
|
+
batch.push([outbound[i].id, vec]);
|
|
163
|
+
ok += 1;
|
|
164
|
+
if (batch.length >= 8)
|
|
165
|
+
flush();
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
flush();
|
|
169
|
+
return ok;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Engine Thread — Knowledge **API** 编排(写路由 + SSE)
|
|
3
|
+
*
|
|
4
|
+
* 唯一写者是 **Writer Worker**(独立线程持 knowledge.db + ingest)。
|
|
5
|
+
* 本线程禁止打开可写 SQLite;残留读路由用只读连接 Local query。
|
|
6
|
+
* 纯读在主线程 Meta/Search Worker(serve.ts),不经过本线程。
|
|
7
|
+
*/
|
|
8
|
+
import { parentPort, workerData } from 'node:worker_threads';
|
|
9
|
+
import { KnowledgeDatabase } from './db.js';
|
|
10
|
+
import { createKnowledgeHttpApp } from './http-app.js';
|
|
11
|
+
import { LocalKnowledgeWriteService } from './writer-service.js';
|
|
12
|
+
import { LocalKnowledgeQueryService } from './query-service.js';
|
|
13
|
+
import { KnowledgeSourceStore } from './source-store.js';
|
|
14
|
+
import { KnowledgeIndexStore } from './index-store.js';
|
|
15
|
+
import { KnowledgeRetriever } from './retriever.js';
|
|
16
|
+
function createMockReq(dto, onClose) {
|
|
17
|
+
const closeHandlers = [];
|
|
18
|
+
onClose(() => {
|
|
19
|
+
for (const fn of closeHandlers)
|
|
20
|
+
fn();
|
|
21
|
+
});
|
|
22
|
+
const req = {
|
|
23
|
+
method: dto.method,
|
|
24
|
+
url: dto.url,
|
|
25
|
+
headers: dto.headers,
|
|
26
|
+
on(event, fn) {
|
|
27
|
+
if (event === 'close')
|
|
28
|
+
closeHandlers.push(fn);
|
|
29
|
+
return req;
|
|
30
|
+
},
|
|
31
|
+
off(event, fn) {
|
|
32
|
+
if (event === 'close') {
|
|
33
|
+
const i = closeHandlers.indexOf(fn);
|
|
34
|
+
if (i >= 0)
|
|
35
|
+
closeHandlers.splice(i, 1);
|
|
36
|
+
}
|
|
37
|
+
return req;
|
|
38
|
+
},
|
|
39
|
+
async *[Symbol.asyncIterator]() {
|
|
40
|
+
if (dto.body)
|
|
41
|
+
yield Buffer.from(dto.body, 'utf8');
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
return req;
|
|
45
|
+
}
|
|
46
|
+
function createMockRes(out, id) {
|
|
47
|
+
let status = 200;
|
|
48
|
+
let headers = {};
|
|
49
|
+
const chunks = [];
|
|
50
|
+
let sse = false;
|
|
51
|
+
let ended = false;
|
|
52
|
+
let headEmitted = false;
|
|
53
|
+
const emit = (msg) => out({ id, ...msg });
|
|
54
|
+
const ensureHead = () => {
|
|
55
|
+
if (headEmitted)
|
|
56
|
+
return;
|
|
57
|
+
headEmitted = true;
|
|
58
|
+
emit({ type: 'sse-start', status, headers });
|
|
59
|
+
};
|
|
60
|
+
const res = {
|
|
61
|
+
__sse: false,
|
|
62
|
+
statusCode: status,
|
|
63
|
+
writeHead(code, hdrs) {
|
|
64
|
+
status = code;
|
|
65
|
+
if (hdrs)
|
|
66
|
+
headers = { ...headers, ...hdrs };
|
|
67
|
+
const ct = String(headers['content-type'] ?? '');
|
|
68
|
+
sse = ct.includes('text/event-stream');
|
|
69
|
+
res.__sse = sse;
|
|
70
|
+
if (sse)
|
|
71
|
+
ensureHead();
|
|
72
|
+
return res;
|
|
73
|
+
},
|
|
74
|
+
setHeader(name, value) {
|
|
75
|
+
headers[name] = value;
|
|
76
|
+
return res;
|
|
77
|
+
},
|
|
78
|
+
getHeader(name) {
|
|
79
|
+
return headers[name];
|
|
80
|
+
},
|
|
81
|
+
write(chunk) {
|
|
82
|
+
const s = typeof chunk === 'string' ? chunk : String(chunk);
|
|
83
|
+
if (sse) {
|
|
84
|
+
ensureHead();
|
|
85
|
+
emit({ type: 'sse-write', chunk: s });
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
chunks.push(s);
|
|
89
|
+
}
|
|
90
|
+
return true;
|
|
91
|
+
},
|
|
92
|
+
end(chunk) {
|
|
93
|
+
if (ended)
|
|
94
|
+
return res;
|
|
95
|
+
if (chunk != null)
|
|
96
|
+
res.write(chunk);
|
|
97
|
+
ended = true;
|
|
98
|
+
if (sse)
|
|
99
|
+
emit({ type: 'sse-end' });
|
|
100
|
+
else
|
|
101
|
+
emit({
|
|
102
|
+
type: 'end',
|
|
103
|
+
status,
|
|
104
|
+
headers: {
|
|
105
|
+
...headers,
|
|
106
|
+
'content-length': Buffer.byteLength(chunks.join(''), 'utf8'),
|
|
107
|
+
},
|
|
108
|
+
body: chunks.join(''),
|
|
109
|
+
});
|
|
110
|
+
return res;
|
|
111
|
+
},
|
|
112
|
+
on() {
|
|
113
|
+
return res;
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
return res;
|
|
117
|
+
}
|
|
118
|
+
function stubEmbeddingProvider() {
|
|
119
|
+
let n = 0;
|
|
120
|
+
return {
|
|
121
|
+
name: 'test-stub',
|
|
122
|
+
dimensions: 4,
|
|
123
|
+
async embed() {
|
|
124
|
+
n += 1;
|
|
125
|
+
return [n, 0.1, 0.2, 0.3];
|
|
126
|
+
},
|
|
127
|
+
async embedBatch(texts) {
|
|
128
|
+
return texts.map((_, i) => [i + n, 0.1, 0.2, 0.3]);
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
async function resolveEmbedding(boot) {
|
|
133
|
+
if (boot.testEmbeddingStub)
|
|
134
|
+
return stubEmbeddingProvider();
|
|
135
|
+
if (boot.embeddingModels && boot.embed?.enabled !== false) {
|
|
136
|
+
const { resolveEmbeddingRuntime } = await import('../memory/sqlite/embedding-from-models.js');
|
|
137
|
+
const runtime = resolveEmbeddingRuntime(boot.embeddingModels);
|
|
138
|
+
return runtime?.provider ?? null;
|
|
139
|
+
}
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
async function main() {
|
|
143
|
+
const boot = workerData;
|
|
144
|
+
const port = parentPort;
|
|
145
|
+
if (!port)
|
|
146
|
+
throw new Error('knowledge engine thread requires parentPort');
|
|
147
|
+
const embeddingProvider = await resolveEmbedding(boot);
|
|
148
|
+
const memoryMode = boot.dbPath === ':memory:';
|
|
149
|
+
let write;
|
|
150
|
+
let query;
|
|
151
|
+
let writeDb = null;
|
|
152
|
+
let readDb = null;
|
|
153
|
+
if (memoryMode) {
|
|
154
|
+
// :memory: 无法跨连接共享 → 写/读同连接
|
|
155
|
+
writeDb = await KnowledgeDatabase.create({
|
|
156
|
+
dbPath: boot.dbPath,
|
|
157
|
+
sqliteVec: boot.sqliteVecExtensionPath
|
|
158
|
+
? { extensionPath: boot.sqliteVecExtensionPath }
|
|
159
|
+
: true,
|
|
160
|
+
});
|
|
161
|
+
write = new LocalKnowledgeWriteService({
|
|
162
|
+
db: writeDb,
|
|
163
|
+
documentConfig: boot.documentConfig ?? null,
|
|
164
|
+
embeddingProvider: embeddingProvider,
|
|
165
|
+
embed: boot.embed ?? null,
|
|
166
|
+
});
|
|
167
|
+
const sources = new KnowledgeSourceStore(writeDb);
|
|
168
|
+
const index = new KnowledgeIndexStore(writeDb);
|
|
169
|
+
const retriever = new KnowledgeRetriever({
|
|
170
|
+
sourceStore: sources,
|
|
171
|
+
indexStore: index,
|
|
172
|
+
embeddingProvider: embeddingProvider,
|
|
173
|
+
});
|
|
174
|
+
query = new LocalKnowledgeQueryService({
|
|
175
|
+
db: writeDb,
|
|
176
|
+
sources,
|
|
177
|
+
index,
|
|
178
|
+
retriever,
|
|
179
|
+
embeddingEnabled: Boolean(embeddingProvider),
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
// Writer Worker:唯一写者 + ingest
|
|
184
|
+
const { WorkerWriteService } = await import('./writer-worker-client.js');
|
|
185
|
+
write = await WorkerWriteService.start({
|
|
186
|
+
dbPath: boot.dbPath,
|
|
187
|
+
documentConfig: boot.documentConfig ?? null,
|
|
188
|
+
sqliteVecExtensionPath: boot.sqliteVecExtensionPath,
|
|
189
|
+
embeddingModels: boot.testEmbeddingStub ? null : (boot.embeddingModels ?? null),
|
|
190
|
+
embed: boot.embed ?? null,
|
|
191
|
+
testEmbeddingStub: boot.testEmbeddingStub ?? false,
|
|
192
|
+
});
|
|
193
|
+
// 残留写路由上的短读:独立只读连接(WAL)
|
|
194
|
+
readDb = await KnowledgeDatabase.create({
|
|
195
|
+
dbPath: boot.dbPath,
|
|
196
|
+
readOnly: true,
|
|
197
|
+
skipMigrate: true,
|
|
198
|
+
sqliteVec: boot.sqliteVecExtensionPath
|
|
199
|
+
? { extensionPath: boot.sqliteVecExtensionPath }
|
|
200
|
+
: true,
|
|
201
|
+
});
|
|
202
|
+
const sources = new KnowledgeSourceStore(readDb);
|
|
203
|
+
const index = new KnowledgeIndexStore(readDb);
|
|
204
|
+
const retriever = new KnowledgeRetriever({
|
|
205
|
+
sourceStore: sources,
|
|
206
|
+
indexStore: index,
|
|
207
|
+
embeddingProvider: embeddingProvider,
|
|
208
|
+
});
|
|
209
|
+
query = new LocalKnowledgeQueryService({
|
|
210
|
+
db: readDb,
|
|
211
|
+
sources,
|
|
212
|
+
index,
|
|
213
|
+
retriever,
|
|
214
|
+
embeddingEnabled: Boolean(embeddingProvider),
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const app = createKnowledgeHttpApp({
|
|
218
|
+
write,
|
|
219
|
+
query,
|
|
220
|
+
tokens: boot.tokens,
|
|
221
|
+
autoRegisterPrincipals: boot.autoRegisterPrincipals ?? true,
|
|
222
|
+
});
|
|
223
|
+
await write.startIngestRuntime();
|
|
224
|
+
const out = (msg) => port.postMessage(msg);
|
|
225
|
+
const clientCloseByReq = new Map();
|
|
226
|
+
const earlyClose = new Set();
|
|
227
|
+
port.on('message', (msg) => {
|
|
228
|
+
if (msg?.type === 'http' && msg.id != null && msg.req) {
|
|
229
|
+
const id = msg.id;
|
|
230
|
+
let closeFn = () => undefined;
|
|
231
|
+
let closeRegistered = false;
|
|
232
|
+
const runClose = () => {
|
|
233
|
+
if (!closeRegistered) {
|
|
234
|
+
earlyClose.add(id);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
const fn = clientCloseByReq.get(id);
|
|
238
|
+
clientCloseByReq.delete(id);
|
|
239
|
+
fn?.();
|
|
240
|
+
};
|
|
241
|
+
const req = createMockReq(msg.req, (fn) => {
|
|
242
|
+
closeFn = fn;
|
|
243
|
+
});
|
|
244
|
+
const res = createMockRes(out, id);
|
|
245
|
+
void app
|
|
246
|
+
.handle(req, res)
|
|
247
|
+
.then(() => {
|
|
248
|
+
if (!res.__sse)
|
|
249
|
+
return;
|
|
250
|
+
clientCloseByReq.set(id, () => closeFn());
|
|
251
|
+
closeRegistered = true;
|
|
252
|
+
if (earlyClose.delete(id))
|
|
253
|
+
runClose();
|
|
254
|
+
})
|
|
255
|
+
.catch(() => {
|
|
256
|
+
clientCloseByReq.delete(id);
|
|
257
|
+
earlyClose.delete(id);
|
|
258
|
+
});
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
if (msg?.type === 'client-close' && msg.id != null) {
|
|
262
|
+
const fn = clientCloseByReq.get(msg.id);
|
|
263
|
+
if (fn) {
|
|
264
|
+
clientCloseByReq.delete(msg.id);
|
|
265
|
+
fn();
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
earlyClose.add(msg.id);
|
|
269
|
+
}
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
if (msg?.type === 'shutdown') {
|
|
273
|
+
void (async () => {
|
|
274
|
+
try {
|
|
275
|
+
await app.dispose();
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
/* dispose 幂等 */
|
|
279
|
+
}
|
|
280
|
+
try {
|
|
281
|
+
readDb?.close();
|
|
282
|
+
}
|
|
283
|
+
catch {
|
|
284
|
+
/* already closed */
|
|
285
|
+
}
|
|
286
|
+
try {
|
|
287
|
+
writeDb?.close();
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
/* already closed */
|
|
291
|
+
}
|
|
292
|
+
port.postMessage({ type: 'shutdown-ack' });
|
|
293
|
+
})();
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
port.postMessage({ type: 'ready' });
|
|
298
|
+
}
|
|
299
|
+
main().catch((err) => {
|
|
300
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
301
|
+
parentPort?.postMessage({ type: 'error', message });
|
|
302
|
+
});
|
|
@@ -1,17 +1,33 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { DocumentCapabilityConfig } from '../capabilities/document/factory.js';
|
|
2
|
+
import type { ExtractResult } from '../capabilities/document/types.js';
|
|
3
|
+
import type { KnowledgeChunkDraft } from './adapters.js';
|
|
2
4
|
export interface DocumentExtractWorkerOptions {
|
|
3
5
|
/** 抽取超时(默认 60s;到点 terminate worker) */
|
|
4
6
|
timeoutMs?: number;
|
|
5
7
|
maxFileBytes?: number;
|
|
6
8
|
/** 中止信号:abort 时立刻 terminate worker */
|
|
7
9
|
signal?: AbortSignal;
|
|
10
|
+
/** 部分抽取限额(xlsx/pdf/text) */
|
|
11
|
+
maxSheets?: number;
|
|
12
|
+
maxRowsPerSheet?: number;
|
|
13
|
+
maxPages?: number;
|
|
14
|
+
maxTextChars?: number;
|
|
15
|
+
/** documents.* 配置(与 Gateway 同源;含 legacy/soffice) */
|
|
16
|
+
documentConfig?: DocumentCapabilityConfig | null;
|
|
17
|
+
}
|
|
18
|
+
export interface DocumentExtractWorkerResult {
|
|
19
|
+
result: ExtractResult;
|
|
20
|
+
/** 预计算 ftsToks 的切块(markdown 空时为空数组) */
|
|
21
|
+
chunks: KnowledgeChunkDraft[];
|
|
22
|
+
/** 原件字节 SHA-256(worker 内流式计算,禁止在 Engine 读全文件) */
|
|
23
|
+
contentHash: string;
|
|
8
24
|
}
|
|
9
25
|
/**
|
|
10
|
-
* 在 worker 线程抽取文档为 Markdown。
|
|
26
|
+
* 在 worker 线程抽取文档为 Markdown,并完成切块 + FTS token。
|
|
11
27
|
*
|
|
12
28
|
* @param filePath - 本地文件路径
|
|
13
|
-
* @param options - 超时 / 大小 /
|
|
14
|
-
* @returns ExtractResult
|
|
29
|
+
* @param options - 超时 / 大小 / 中止 / 部分抽取限额
|
|
30
|
+
* @returns ExtractResult + chunks
|
|
15
31
|
* @throws Error 超时 / abort / worker 失败 / 抽取错误
|
|
16
32
|
*/
|
|
17
|
-
export declare function extractDocumentInWorker(filePath: string, options?: DocumentExtractWorkerOptions): Promise<
|
|
33
|
+
export declare function extractDocumentInWorker(filePath: string, options?: DocumentExtractWorkerOptions): Promise<DocumentExtractWorkerResult>;
|