@octopi-agent/engine 0.58.0 → 0.70.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +122 -1
- package/dist/config-schema/engine.js +4 -1
- package/dist/config-schema/substrate.d.ts +123 -0
- package/dist/config-schema/substrate.js +161 -0
- package/dist/config.d.ts +101 -1
- package/dist/harness/agent/builder.d.ts +15 -1
- package/dist/harness/agent/builder.js +41 -4
- package/dist/harness/agent/config-bridge.js +17 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/capabilities/summary/gate.js +2 -2
- package/dist/harness/context/capabilities/summary/policies.js +5 -5
- package/dist/harness/context/capabilities/summary/registry.js +29 -24
- package/dist/harness/context/layer-health.js +3 -2
- package/dist/harness/context/layers.d.ts +27 -13
- package/dist/harness/context/layers.js +76 -66
- package/dist/harness/context/system-prompt-assembler.d.ts +3 -0
- package/dist/harness/context/system-prompt-assembler.js +11 -1
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.d.ts +4 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/index.js +3 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.d.ts +56 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.js +281 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/registry.d.ts +98 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/registry.js +250 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/stream.d.ts +124 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/stream.js +372 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.d.ts +109 -0
- package/dist/harness/extension/plugin-ecosystem/client-tools/types.js +8 -0
- package/dist/harness/extension/plugin-ecosystem/index.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +1 -1
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.d.ts +11 -1
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +14 -0
- package/dist/harness/index.d.ts +8 -5
- package/dist/harness/index.js +4 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +97 -19
- package/dist/harness/knowledge/source-store.js +166 -94
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/cognition-decay.d.ts +36 -0
- package/dist/harness/memory/cognition-decay.js +55 -0
- package/dist/harness/memory/cognition-gates.d.ts +60 -0
- package/dist/harness/memory/cognition-gates.js +191 -0
- package/dist/harness/memory/cognition-trigger.d.ts +30 -0
- package/dist/harness/memory/cognition-trigger.js +41 -0
- package/dist/harness/memory/cognition-types.d.ts +1 -1
- package/dist/harness/memory/cognition.d.ts +40 -8
- package/dist/harness/memory/cognition.js +432 -79
- package/dist/harness/memory/conceptualizer.d.ts +66 -0
- package/dist/harness/memory/conceptualizer.js +246 -0
- package/dist/harness/memory/index.d.ts +16 -2
- package/dist/harness/memory/index.js +11 -1
- package/dist/harness/memory/sqlite/agent-db.d.ts +9 -0
- package/dist/harness/memory/sqlite/agent-db.js +118 -15
- package/dist/harness/memory/sqlite/cognition-store.d.ts +55 -16
- package/dist/harness/memory/sqlite/cognition-store.js +591 -144
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/memory/sqlite/wisdom-store.d.ts +33 -14
- package/dist/harness/memory/sqlite/wisdom-store.js +257 -44
- package/dist/harness/memory/types.d.ts +331 -16
- package/dist/harness/memory/types.js +49 -0
- package/dist/harness/memory/wisdom-formation.d.ts +101 -0
- package/dist/harness/memory/wisdom-formation.js +304 -0
- package/dist/harness/memory/wisdom-gates.d.ts +50 -0
- package/dist/harness/memory/wisdom-gates.js +204 -0
- package/dist/harness/memory/wisdom-policy.d.ts +107 -0
- package/dist/harness/memory/wisdom-policy.js +220 -0
- package/dist/harness/memory/wisdom-types.d.ts +3 -2
- package/dist/harness/memory/wisdom-types.js +2 -2
- package/dist/harness/memory/wisdom.d.ts +27 -0
- package/dist/harness/memory/wisdom.js +181 -0
- package/dist/harness/run/agent/agent.d.ts +9 -0
- package/dist/harness/run/agent/agent.js +19 -0
- package/dist/harness/run/reliability/run-agent.js +6 -2
- package/dist/harness/run/runner.js +2 -1
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/index.d.ts +1 -0
- package/dist/harness/session/index.js +1 -0
- package/dist/harness/session/title.d.ts +86 -0
- package/dist/harness/session/title.js +268 -0
- package/dist/harness/shared/events/agent-event-map.d.ts +8 -0
- package/dist/harness/shared/events/agent-event-map.js +1 -0
- package/dist/index.d.ts +7 -3
- package/dist/index.js +3 -1
- package/dist/subsystems/cognition-steward/conceptualize/SUBSYSTEM.md +43 -0
- package/dist/subsystems/cognition-steward/conceptualize/config.yaml +50 -0
- package/dist/subsystems/cognition-steward/conceptualize/handler.d.ts +9 -0
- package/dist/subsystems/cognition-steward/conceptualize/handler.js +125 -0
- package/dist/subsystems/cognition-steward/consolidate/SUBSYSTEM.md +6 -0
- package/dist/subsystems/cognition-steward/consolidate/config.yaml +54 -0
- package/dist/subsystems/cognition-steward/consolidate/handler.d.ts +12 -0
- package/dist/subsystems/cognition-steward/consolidate/handler.js +93 -0
- package/dist/subsystems/wisdom-steward/evaluate/config.yaml +50 -0
- package/dist/subsystems/wisdom-steward/evaluate/handler.d.ts +18 -0
- package/dist/subsystems/wisdom-steward/evaluate/handler.js +91 -0
- package/dist/subsystems/wisdom-steward/formation/SUBSYSTEM.md +42 -0
- package/dist/subsystems/wisdom-steward/formation/config.yaml +54 -0
- package/dist/subsystems/wisdom-steward/formation/handler.d.ts +18 -0
- package/dist/subsystems/wisdom-steward/formation/handler.js +212 -0
- package/dist/subsystems/wisdom-steward/govern/config.yaml +55 -0
- package/dist/subsystems/wisdom-steward/govern/handler.d.ts +17 -0
- package/dist/subsystems/wisdom-steward/govern/handler.js +68 -0
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
|
|
5
5
|
*/
|
|
6
6
|
import { EventEmitter } from 'node:events';
|
|
7
7
|
import { FormatAdapterRegistry } from './adapters.js';
|
|
@@ -11,66 +11,104 @@ import { type SourceFetcher, type VirtualDocument } from './fetchers.js';
|
|
|
11
11
|
import { ConnectorRegistry } from './connectors.js';
|
|
12
12
|
import type { CredentialStore } from '../governance/credentials/store.js';
|
|
13
13
|
import type { EmbeddingProvider } from '../memory/sqlite/embedding.js';
|
|
14
|
-
import type { DocumentPort } from '../
|
|
15
|
-
|
|
14
|
+
import type { DocumentPort } from '../capabilities/document/types.js';
|
|
15
|
+
import { type KnowledgeFileLimitsInput } from './file-limits.js';
|
|
16
|
+
import { type EmbedSecretPolicy } from './embed-runner.js';
|
|
17
|
+
/**
|
|
18
|
+
* 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
|
|
19
|
+
*
|
|
20
|
+
* @param filePath - 本地文件
|
|
21
|
+
* @param signal - 中止信号
|
|
22
|
+
* @returns hex digest
|
|
23
|
+
*/
|
|
24
|
+
export declare function hashFileStreaming(filePath: string, signal?: AbortSignal): Promise<string>;
|
|
25
|
+
import { type IngestJobKind as JobKind } from './job-queue.js';
|
|
26
|
+
import { type KnowledgeJobControlState } from './job-control-state.js';
|
|
27
|
+
export type { EmbedSecretPolicy };
|
|
28
|
+
export type IngestJobKind = JobKind;
|
|
29
|
+
export type { KnowledgeJobControlState };
|
|
16
30
|
export interface IngestProgressEvent {
|
|
17
31
|
type: 'job' | 'source' | 'error' | 'embed';
|
|
18
32
|
sourceId: string;
|
|
19
33
|
path?: string;
|
|
20
34
|
status: string;
|
|
21
35
|
detail?: string;
|
|
36
|
+
/** SSE 过滤用:源归属(Writer 在 emit 时补齐,API 不得再 RPC 查) */
|
|
37
|
+
tenantId?: string;
|
|
38
|
+
registeredBy?: string;
|
|
39
|
+
visibility?: string;
|
|
22
40
|
}
|
|
23
41
|
export interface KnowledgeIngestOptions {
|
|
24
42
|
sourceStore: KnowledgeSourceStore;
|
|
25
43
|
indexStore?: KnowledgeIndexStore;
|
|
26
44
|
adapterRegistry?: FormatAdapterRegistry;
|
|
27
|
-
/** parse 并发(默认
|
|
45
|
+
/** parse 并发(默认 8) */
|
|
28
46
|
parseConcurrency?: number;
|
|
29
|
-
/**
|
|
47
|
+
/** 文档(PDF/Office)抽取并发(默认 2;大文件抢占 CPU/内存) */
|
|
48
|
+
documentParseConcurrency?: number;
|
|
49
|
+
/** 队列深度上限(超出只,不务) */
|
|
30
50
|
maxQueueDepth?: number;
|
|
31
|
-
/** fs watch debounce ms
|
|
51
|
+
/** fs watch debounce ms(默?2000?*/
|
|
32
52
|
debounceMs?: number;
|
|
33
|
-
/**
|
|
53
|
+
/** 单文件大小兜底上限(格式分级?fileLimits?*/
|
|
34
54
|
maxFileBytes?: number;
|
|
35
|
-
/**
|
|
55
|
+
/**
|
|
56
|
+
* 按格式分级的文件 + 部分抽取策略?
|
|
57
|
+
* 机器资源不足时下?maxBytes / hardMaxFileBytes?
|
|
58
|
+
*/
|
|
59
|
+
fileLimits?: KnowledgeFileLimitsInput;
|
|
60
|
+
/** 单文?parse 超时 ms(默?45000?*/
|
|
36
61
|
parseTimeoutMs?: number;
|
|
37
|
-
/** Phase B embedding
|
|
62
|
+
/** Phase B embedding(未配则?*/
|
|
38
63
|
embeddingProvider?: EmbeddingProvider | null;
|
|
39
|
-
/** embed
|
|
64
|
+
/** embed 批大小( 32?*/
|
|
40
65
|
embedBatch?: number;
|
|
41
|
-
/** embed
|
|
66
|
+
/** embed 小间?ms(限速, 0?*/
|
|
42
67
|
embedMinIntervalMs?: number;
|
|
43
|
-
/** embed
|
|
68
|
+
/** embed 并发(默?parseConcurrency?*/
|
|
44
69
|
embedConcurrency?: number;
|
|
45
|
-
/**
|
|
70
|
+
/** 磁盘水位(默?false;仅不丢任务?*/
|
|
46
71
|
diskWatermarkAlert?: boolean;
|
|
47
|
-
/**
|
|
72
|
+
/** 库(source.authRef 解析;可选) */
|
|
48
73
|
credentials?: CredentialStore | null;
|
|
49
|
-
/**
|
|
74
|
+
/** ?fetcher 覆盖(测?扩展?*/
|
|
50
75
|
fetchers?: Partial<Record<'local' | 'url' | 'connector', SourceFetcher>>;
|
|
51
|
-
/** connector
|
|
76
|
+
/** connector 注册?rest?*/
|
|
52
77
|
connectors?: ConnectorRegistry;
|
|
53
|
-
/** poll 调度 tick ms
|
|
78
|
+
/** poll 调度 tick ms(默?60s?*/
|
|
54
79
|
pollTickMs?: number;
|
|
55
|
-
/**
|
|
80
|
+
/** 单源?poll 间隔 ms(默?15min?*/
|
|
56
81
|
pollMinIntervalMs?: number;
|
|
57
|
-
/** 每轮 poll
|
|
82
|
+
/** 每轮 poll 理源数(成本上限,默?8?*/
|
|
58
83
|
maxPollPerTick?: number;
|
|
59
84
|
/**
|
|
60
|
-
* DocumentPort:启用后 PDF/Office 等走抽取
|
|
61
|
-
*
|
|
85
|
+
* DocumentPort:启用后 PDF/Office 等走抽取 ?Markdown 切块
|
|
86
|
+
* (缺?null = 维持 BINARY skip ?
|
|
62
87
|
*/
|
|
63
88
|
documentPort?: DocumentPort | null;
|
|
89
|
+
/**
|
|
90
|
+
* documents.* 配置 — 与 Gateway 同源;worker 抽取与 legacy/soffice 共用
|
|
91
|
+
*/
|
|
92
|
+
documentConfig?: import('../capabilities/document/factory.js').DocumentCapabilityConfig | null;
|
|
93
|
+
/**
|
|
94
|
+
* embedding 外发前敏感形态策略( redact?
|
|
95
|
+
* - allow:原发(内网/模型?
|
|
96
|
+
* - redact:命 [REDACTED:rule] ?embed
|
|
97
|
+
* - skip:命不写向量(关仍可搜)
|
|
98
|
+
*/
|
|
99
|
+
embedSecretPolicy?: EmbedSecretPolicy;
|
|
64
100
|
}
|
|
65
101
|
export declare class KnowledgeIngest extends EventEmitter {
|
|
66
102
|
private readonly sources;
|
|
67
103
|
private readonly index;
|
|
68
104
|
private readonly adapters;
|
|
69
105
|
private readonly parseConcurrency;
|
|
106
|
+
private readonly documentParseConcurrency;
|
|
70
107
|
private readonly maxQueueDepth;
|
|
71
108
|
private readonly debounceMs;
|
|
72
109
|
private readonly maxFileBytes;
|
|
73
|
-
|
|
110
|
+
private readonly fileLimits;
|
|
111
|
+
/** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
|
|
74
112
|
private readonly parseTimeoutMs;
|
|
75
113
|
private readonly embedding;
|
|
76
114
|
private readonly embedBatch;
|
|
@@ -85,103 +123,202 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
85
123
|
private readonly pollMinIntervalMs;
|
|
86
124
|
private readonly maxPollPerTick;
|
|
87
125
|
private readonly documentPort;
|
|
88
|
-
private
|
|
126
|
+
private readonly documentConfig;
|
|
127
|
+
private readonly embedSecretPolicy;
|
|
128
|
+
private readonly embedRunner;
|
|
89
129
|
private pollTimer;
|
|
90
130
|
private reconcileTimer;
|
|
91
|
-
/**
|
|
92
|
-
private
|
|
93
|
-
|
|
131
|
+
/** 源级:DB 权威 + ?AbortSignal(跨重启?*/
|
|
132
|
+
private readonly jobControl;
|
|
133
|
+
private readonly jobQueue;
|
|
134
|
+
/** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
|
|
94
135
|
onSourceSettled?: (sourceId: string) => void;
|
|
95
|
-
/**
|
|
136
|
+
/** parse 缺口时间 */
|
|
96
137
|
private lastParseGapScanAt;
|
|
138
|
+
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
139
|
+
private gapScanUrgent;
|
|
140
|
+
private gapScanTimer;
|
|
97
141
|
private runningParse;
|
|
142
|
+
private runningDocParse;
|
|
98
143
|
private runningEmbed;
|
|
99
144
|
private draining;
|
|
100
145
|
private kickAgain;
|
|
101
|
-
/** source
|
|
146
|
+
/** source 级锁:同 source 不并?walk/parse 冲突?*/
|
|
102
147
|
private sourceLocks;
|
|
103
|
-
/** walk 发现的文件数(Phase A coverage
|
|
148
|
+
/** walk 发现的文件数(Phase A coverage 分母?*/
|
|
104
149
|
private discoveredFiles;
|
|
105
150
|
private watchers;
|
|
106
151
|
private debounceTimers;
|
|
107
152
|
private disposed;
|
|
108
153
|
constructor(options: KnowledgeIngestOptions);
|
|
154
|
+
private lastCleanupAt;
|
|
109
155
|
/**
|
|
110
|
-
*
|
|
111
|
-
*
|
|
156
|
+
* 清掉「非?knowledge_files (目录曾 no_adapter?
|
|
157
|
+
*
|
|
158
|
+
* ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
|
|
159
|
+
* 60s 节流,避免每?reconcile ?stat?
|
|
160
|
+
*
|
|
161
|
+
* @param sourceId - 限定源;省略则全部源
|
|
162
|
+
* @param opts.force - 忽略节流
|
|
163
|
+
* @returns 删除行数
|
|
164
|
+
*/
|
|
165
|
+
cleanupNonFileIndexRows(sourceId?: string, opts?: {
|
|
166
|
+
force?: boolean;
|
|
167
|
+
}): number;
|
|
168
|
+
/**
|
|
169
|
+
* 无跨进程 Lease:本进程时把历史 running 收回 queued?
|
|
170
|
+
* 仅构造时调用行中同进程的 running 法的?
|
|
112
171
|
*/
|
|
113
172
|
private reclaimOrphanRunningJobs;
|
|
114
173
|
get indexStore(): KnowledgeIndexStore;
|
|
115
174
|
/**
|
|
116
|
-
*
|
|
175
|
+
* 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
|
|
117
176
|
*
|
|
118
|
-
* @param p -
|
|
119
|
-
* @returns
|
|
177
|
+
* @param p - 文件
|
|
178
|
+
* @returns 跳过
|
|
120
179
|
*/
|
|
121
180
|
private shouldSkipFile;
|
|
181
|
+
/** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
|
|
182
|
+
private readFileHead;
|
|
183
|
+
/** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
|
|
184
|
+
private identityKeyFor;
|
|
122
185
|
/**
|
|
123
|
-
*
|
|
186
|
+
* 归属:`parse_file` / 读文件只读源 root 内的?
|
|
187
|
+
* ?**realpath** 比较,防符号链接逃出 root?
|
|
188
|
+
*
|
|
189
|
+
* @param source - 知识?
|
|
190
|
+
* @param path - 请求(本地绝对路径,
|
|
191
|
+
*/
|
|
192
|
+
private sourceOwnsPath;
|
|
193
|
+
/**
|
|
194
|
+
* 全量/增量索引某源(Phase A?
|
|
195
|
+
*
|
|
196
|
+
* @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
|
|
197
|
+
* @param opts.incremental - 增量(poll):?active 任务则跳过本?
|
|
124
198
|
*/
|
|
125
199
|
ingestSource(sourceId: string, opts?: {
|
|
126
200
|
full?: boolean;
|
|
201
|
+
incremental?: boolean;
|
|
202
|
+
fromQueue?: boolean;
|
|
127
203
|
}): Promise<void>;
|
|
204
|
+
/**
|
|
205
|
+
* 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
|
|
206
|
+
* 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
|
|
207
|
+
* indexing/error/半成品必须重做,避免把未完成当成新鲜。
|
|
208
|
+
*/
|
|
209
|
+
private isUnchangedIndexed;
|
|
128
210
|
/**
|
|
129
211
|
* URL / connector 源:discover + fetch + 索引
|
|
130
212
|
*/
|
|
131
213
|
private ingestRemoteSource;
|
|
132
214
|
/**
|
|
133
|
-
*
|
|
215
|
+
* 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
|
|
134
216
|
*
|
|
135
|
-
* @returns
|
|
136
|
-
* @throws authRef
|
|
217
|
+
* @returns ?null(仅当源?authRef?
|
|
218
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方
|
|
137
219
|
*/
|
|
138
220
|
private resolveCred;
|
|
139
221
|
/**
|
|
140
|
-
*
|
|
222
|
+
* ?VirtualDocument 写入索引?/ 规范化后文本?
|
|
141
223
|
*
|
|
142
|
-
* @returns
|
|
224
|
+
* @returns 成功写入
|
|
143
225
|
*/
|
|
144
|
-
indexVirtualDoc(sourceId: string, doc: VirtualDocument): boolean
|
|
226
|
+
indexVirtualDoc(sourceId: string, doc: VirtualDocument, authRef?: string | null): Promise<boolean>;
|
|
145
227
|
/**
|
|
146
|
-
*
|
|
228
|
+
* 单文(P0 道;完成 parse,不 await 全库?
|
|
147
229
|
*/
|
|
148
230
|
ingestFileNow(sourceId: string, filePath: string): Promise<boolean>;
|
|
231
|
+
/**
|
|
232
|
+
* 重做指定文件:强制重?解析 ?分块 ?向量
|
|
233
|
+
*
|
|
234
|
+
* - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
|
|
235
|
+
* - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
|
|
236
|
+
* - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
|
|
237
|
+
* - 等非文件:直接清,不入队
|
|
238
|
+
* - 完成后走既有 embed 链路
|
|
239
|
+
*
|
|
240
|
+
* @param sourceId - ?id
|
|
241
|
+
* @param paths - 文件列表
|
|
242
|
+
* @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
|
|
243
|
+
*/
|
|
244
|
+
reprocessFiles(sourceId: string, paths: string[]): Promise<{
|
|
245
|
+
queued: number;
|
|
246
|
+
alreadyActive: number;
|
|
247
|
+
cleanedNonFiles: number;
|
|
248
|
+
resumed: boolean;
|
|
249
|
+
rejected: number;
|
|
250
|
+
}>;
|
|
251
|
+
/** 上是否已?queued/running ?parse/fetch 任务 */
|
|
252
|
+
private hasActiveParseJob;
|
|
253
|
+
/**
|
|
254
|
+
* 级任?文件UI 跟踪重做完成?
|
|
255
|
+
*
|
|
256
|
+
* @param sourceId - ?id
|
|
257
|
+
* @param paths - 列表
|
|
258
|
+
*/
|
|
259
|
+
jobStateForPaths(sourceId: string, paths: string[]): Array<{
|
|
260
|
+
path: string;
|
|
261
|
+
jobsActive: number;
|
|
262
|
+
fileStatus: string | null;
|
|
263
|
+
chunkCount: number;
|
|
264
|
+
error: string | null;
|
|
265
|
+
exists: boolean;
|
|
266
|
+
}>;
|
|
267
|
+
/**
|
|
268
|
+
* 按筛选批量重做(当前列表?status/ext/q 条件?
|
|
269
|
+
*
|
|
270
|
+
* @param sourceId - ?id
|
|
271
|
+
* @param opts - 与文件列表筛选一?
|
|
272
|
+
* @returns 入队条数
|
|
273
|
+
*/
|
|
274
|
+
reprocessByFilter(sourceId: string, opts?: {
|
|
275
|
+
status?: 'indexed' | 'skipped' | 'error' | 'all';
|
|
276
|
+
ext?: string;
|
|
277
|
+
q?: string;
|
|
278
|
+
}): Promise<{
|
|
279
|
+
queued: number;
|
|
280
|
+
alreadyActive: number;
|
|
281
|
+
cleanedNonFiles: number;
|
|
282
|
+
resumed: boolean;
|
|
283
|
+
rejected: number;
|
|
284
|
+
}>;
|
|
149
285
|
/**
|
|
150
286
|
* 处理 fs 变更(debounce 后入队)
|
|
151
287
|
*/
|
|
152
288
|
handleFsChange(sourceId: string, filePath: string, event: 'change' | 'rename'): void;
|
|
153
289
|
/**
|
|
154
|
-
*
|
|
290
|
+
* watch(幂等)
|
|
155
291
|
*/
|
|
156
292
|
startWatch(sourceId: string): void;
|
|
157
293
|
stopWatch(sourceId: string): void;
|
|
158
294
|
/**
|
|
159
|
-
*
|
|
295
|
+
* 等待队列排空(测?管理?
|
|
160
296
|
*/
|
|
161
297
|
idle(timeoutMs?: number): Promise<void>;
|
|
162
298
|
dispose(): void;
|
|
163
299
|
/**
|
|
164
|
-
*
|
|
300
|
+
* 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
|
|
165
301
|
*
|
|
166
|
-
* @param opts.sourceId -
|
|
167
|
-
* @returns
|
|
302
|
+
* @param opts.sourceId - 仅中源;省略则全部源
|
|
303
|
+
* @returns 取消
|
|
168
304
|
*/
|
|
169
305
|
abortJobs(opts?: {
|
|
170
306
|
sourceId?: string;
|
|
171
307
|
}): {
|
|
172
308
|
cancelledQueued: number;
|
|
173
309
|
abortedRunning: number;
|
|
310
|
+
runningJobs: number;
|
|
174
311
|
};
|
|
175
312
|
/**
|
|
176
|
-
*
|
|
313
|
+
* /继续索引(相轻)
|
|
177
314
|
*
|
|
178
|
-
* -
|
|
179
|
-
* -
|
|
315
|
+
* - 清除该源?
|
|
316
|
+
* - ?cancelled 任务放回 queued
|
|
180
317
|
* - 若仍缺向量则排队 embed
|
|
181
|
-
* -
|
|
318
|
+
* - 不重?walk 全量;需要全 reindex
|
|
182
319
|
*
|
|
183
|
-
* @param opts.sourceId -
|
|
184
|
-
* @returns
|
|
320
|
+
* @param opts.sourceId - 仅恢源;省略则全部源
|
|
321
|
+
* @returns
|
|
185
322
|
*/
|
|
186
323
|
resumeJobs(opts?: {
|
|
187
324
|
sourceId?: string;
|
|
@@ -189,95 +326,133 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
189
326
|
restoredCancelled: number;
|
|
190
327
|
embedQueued: number;
|
|
191
328
|
};
|
|
192
|
-
/**
|
|
329
|
+
/** 的未(resume / supersede 后用;清 DB 态) */
|
|
193
330
|
private beginAbortEpoch;
|
|
194
331
|
/**
|
|
195
|
-
* 重建前作废本源未完成任务(supersede
|
|
332
|
+
* 重建前作废本源未完成任务(supersede?
|
|
196
333
|
*
|
|
197
|
-
* - queued
|
|
198
|
-
* - running 打断(worker
|
|
199
|
-
* - 随后 beginAbortEpoch
|
|
334
|
+
* - queued ?cancelled(superseded_by_reindex?
|
|
335
|
+
* - running 打断(worker/出)
|
|
336
|
+
* - 随后 beginAbortEpoch,本建可继续?
|
|
200
337
|
*/
|
|
201
338
|
private supersedeSourceWork;
|
|
202
339
|
private ensureParseCoverage;
|
|
340
|
+
/** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
|
|
341
|
+
private requestGapScan;
|
|
203
342
|
/**
|
|
204
|
-
*
|
|
343
|
+
* 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
|
|
205
344
|
*
|
|
206
|
-
*
|
|
207
|
-
* embed
|
|
345
|
+
* 源用?discover walk ?keep 集(;失回并?stat?
|
|
346
|
+
* embed 前调避免给已移出文件的残 chunks 上向?
|
|
208
347
|
*
|
|
209
|
-
* @param sourceId -
|
|
348
|
+
* @param sourceId - 知识?id
|
|
210
349
|
* @returns 删除的文件数
|
|
211
350
|
*/
|
|
212
351
|
pruneMissingOnDisk(sourceId: string): Promise<number>;
|
|
213
|
-
/**
|
|
352
|
+
/** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
214
353
|
private pruneByStatScan;
|
|
215
354
|
private abortSignalFor;
|
|
216
355
|
private isAborted;
|
|
217
356
|
/**
|
|
218
|
-
*
|
|
357
|
+
* /继续 控制数(互斥按钮
|
|
219
358
|
*
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
|
|
223
|
-
|
|
359
|
+
* @param sourceId - ?id
|
|
360
|
+
* @returns 队列计数 + 已中?+ 仍缺向量
|
|
361
|
+
*/
|
|
362
|
+
jobControlState(sourceId: string): KnowledgeJobControlState;
|
|
363
|
+
/**
|
|
364
|
+
* 任务对账(看门狗?不依赖单?ensureEmbedJob
|
|
365
|
+
*
|
|
366
|
+
* 解决「续去重吞掉 / 进程 / partial」等静默停摆?
|
|
367
|
+
* 1. 回收超时 running(崩?孤儿?
|
|
368
|
+
* 2. 有缺向量且无 active embed ?排队
|
|
369
|
+
* 3. 无队缺口 ??coverage/status
|
|
224
370
|
*/
|
|
225
371
|
startReconciler(intervalMs?: number): void;
|
|
226
372
|
stopReconciler(): void;
|
|
227
|
-
/**
|
|
373
|
+
/** 对账入口(也工调 */
|
|
228
374
|
reconcileJobs(): Promise<void>;
|
|
229
|
-
/**
|
|
375
|
+
/** ?job 清理(done/failed/cancelled 超过保留期) */
|
|
376
|
+
private cleanupTerminalJobs;
|
|
377
|
+
/** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
|
|
230
378
|
private reclaimStaleRunningJobs;
|
|
231
379
|
private countActiveJobs;
|
|
380
|
+
private hasRunningKind;
|
|
232
381
|
/**
|
|
233
|
-
*
|
|
382
|
+
* poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
|
|
234
383
|
*/
|
|
235
384
|
startPolling(): void;
|
|
236
385
|
stopPolling(): void;
|
|
237
386
|
/**
|
|
238
|
-
*
|
|
387
|
+
* 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
|
|
239
388
|
*
|
|
240
|
-
* @returns
|
|
389
|
+
* @returns 实际触发?sourceId 列表
|
|
241
390
|
*/
|
|
242
391
|
pollDueSources(): Promise<string[]>;
|
|
243
392
|
private hasQueuedJobs;
|
|
393
|
+
/**
|
|
394
|
+
* 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
|
|
395
|
+
* 需要计数的路径请 `await enqueueStrict`。
|
|
396
|
+
*/
|
|
244
397
|
private enqueue;
|
|
398
|
+
/** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
|
|
399
|
+
*
|
|
400
|
+
* 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
|
|
401
|
+
*/
|
|
402
|
+
private enqueueStrict;
|
|
403
|
+
/** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
|
|
404
|
+
private bindJobFileId;
|
|
405
|
+
/** job 绑定 file_id:membership → identity;本地文件可 identify */
|
|
406
|
+
private resolveJobFileId;
|
|
245
407
|
private kick;
|
|
246
408
|
private drain;
|
|
409
|
+
/** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
|
|
410
|
+
private isDocParseJob;
|
|
411
|
+
private acceptParseSlot;
|
|
247
412
|
private countQueuedKinds;
|
|
248
413
|
private claimJob;
|
|
249
414
|
private runJob;
|
|
415
|
+
/**
|
|
416
|
+
* 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
|
|
417
|
+
* 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
|
|
418
|
+
*/
|
|
250
419
|
private withTimeout;
|
|
251
420
|
/**
|
|
252
|
-
*
|
|
421
|
+
* 外源单文?job:按逻辑?fetch + 索引
|
|
253
422
|
*/
|
|
254
423
|
private fetchDocJob;
|
|
255
424
|
private parseOne;
|
|
256
425
|
/**
|
|
257
|
-
* PDF/Office:DocumentPort
|
|
258
|
-
* 新鲜度以原件字节 hash
|
|
426
|
+
* PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
|
|
427
|
+
* 新鲜度以原件字节 hash 为准?
|
|
259
428
|
*
|
|
260
|
-
* @param sourceId -
|
|
261
|
-
* @param filePath -
|
|
262
|
-
* @returns
|
|
429
|
+
* @param sourceId - 知识?id
|
|
430
|
+
* @param filePath - 文件
|
|
431
|
+
* @returns 成功入索?
|
|
263
432
|
*/
|
|
264
433
|
private parseDocumentFile;
|
|
265
|
-
/**
|
|
434
|
+
/** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
|
|
266
435
|
private extractDocument;
|
|
436
|
+
/** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
|
|
437
|
+
private chunkTextInWorker;
|
|
267
438
|
/**
|
|
268
439
|
* Phase B:嵌入待处理 chunk
|
|
269
440
|
*
|
|
270
|
-
*
|
|
271
|
-
*
|
|
441
|
+
* ?job 内按 embedConcurrency ****多批调用 API?
|
|
442
|
+
* 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
|
|
272
443
|
*/
|
|
273
444
|
private embedPending;
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
445
|
+
private recordEmbedSecretHit;
|
|
446
|
+
/** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
|
|
277
447
|
private ensureEmbedJob;
|
|
278
448
|
private heartbeatJob;
|
|
449
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
|
|
450
|
+
private coverageDirty;
|
|
451
|
+
private markCoverageDirty;
|
|
452
|
+
private flushCoverageDirty;
|
|
453
|
+
private lastCoverageRefreshAt;
|
|
279
454
|
private refreshCoverage;
|
|
280
455
|
private emitProgress;
|
|
281
|
-
/**
|
|
456
|
+
/** 磁盘水位(仅,不务) */
|
|
282
457
|
private warnDiskWatermark;
|
|
283
458
|
}
|