@octopi-agent/engine 0.86.38 → 0.90.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/agent.js +12 -5
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/engine.d.ts +4 -4
- package/dist/config-schema/governance.js +3 -2
- package/dist/config-schema/models.d.ts +2 -2
- package/dist/config-schema/shared.d.ts +3 -0
- package/dist/config-schema/shared.js +12 -0
- package/dist/config-schema/substrate.d.ts +1 -1
- package/dist/config.d.ts +31 -9
- package/dist/config.js +66 -74
- package/dist/harness/agent/builder.d.ts +3 -2
- package/dist/harness/agent/builder.js +11 -7
- package/dist/harness/agent/config-bridge.js +4 -1
- package/dist/harness/agent/tool-purpose.d.ts +9 -1
- package/dist/harness/agent/tool-purpose.js +11 -2
- package/dist/harness/agent/wire-agent-builder.js +9 -2
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.d.ts +11 -1
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.js +11 -1
- package/dist/harness/collaboration/autonomous-subsystem/loader.d.ts +13 -0
- package/dist/harness/collaboration/autonomous-subsystem/loader.js +16 -1
- package/dist/harness/collaboration/autonomous-subsystem/runtime.d.ts +4 -0
- package/dist/harness/collaboration/autonomous-subsystem/runtime.js +3 -1
- package/dist/harness/context/default-context-engine.js +4 -4
- package/dist/harness/context/layer-types.d.ts +36 -0
- package/dist/harness/context/layer-types.js +38 -0
- package/dist/harness/context/root-instructions.js +1 -1
- package/dist/harness/context/system-prompt-assembler.js +10 -4
- package/dist/harness/context/tool-result-aging.js +2 -2
- package/dist/harness/context/turn-envelope.d.ts +1 -1
- package/dist/harness/context/turn-envelope.js +1 -1
- package/dist/harness/extension/execution-environment/effect-policy.d.ts +11 -1
- package/dist/harness/extension/execution-environment/effect-policy.js +24 -0
- package/dist/harness/extension/execution-environment/index.d.ts +1 -1
- package/dist/harness/extension/execution-environment/index.js +1 -1
- package/dist/harness/extension/execution-environment/isolation.js +5 -1
- package/dist/harness/extension/execution-environment/sandbox.js +36 -8
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.d.ts +6 -0
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.js +17 -0
- package/dist/harness/extension/plugin-ecosystem/tools/file-search.js +10 -17
- package/dist/harness/extension/plugin-ecosystem/tools/git-runner.d.ts +31 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-runner.js +104 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.d.ts +111 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.js +197 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-tools.d.ts +8 -8
- package/dist/harness/extension/plugin-ecosystem/tools/git-tools.js +9 -100
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +4 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +2 -0
- package/dist/harness/index.d.ts +2 -0
- package/dist/harness/index.js +1 -0
- package/dist/harness/knowledge/file-identity.d.ts +8 -0
- package/dist/harness/knowledge/file-identity.js +26 -1
- package/dist/harness/knowledge/grounding.js +5 -5
- package/dist/harness/knowledge/index-store.d.ts +31 -4
- package/dist/harness/knowledge/index-store.js +121 -39
- package/dist/harness/knowledge/ingest.d.ts +115 -115
- package/dist/harness/knowledge/ingest.js +150 -150
- package/dist/harness/knowledge/knowledge-serve-child.js +1 -1
- package/dist/harness/knowledge/purge.js +4 -1
- package/dist/harness/memory/backfill-coverage.js +3 -1
- package/dist/harness/observability/observer/hub.js +30 -1
- package/dist/harness/observability/observer/types.d.ts +23 -0
- package/dist/harness/run/runner.d.ts +16 -0
- package/dist/harness/run/runner.js +139 -26
- package/dist/harness/session/history/port.js +15 -5
- package/dist/harness/session/in-memory-store.d.ts +5 -0
- package/dist/harness/session/in-memory-store.js +33 -2
- package/dist/harness/session/title.js +25 -9
- package/dist/harness/shared/events/agent-event-map.d.ts +26 -0
- package/dist/harness/shared/events/agent-event-map.js +2 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +3 -2
- package/dist/integration/observability/run-telemetry.d.ts +8 -0
- package/dist/integration/observability/run-telemetry.js +31 -8
- package/dist/integration/observability/trace-collector.js +26 -4
- package/dist/integration/observability/trace-events.d.ts +1 -0
- package/dist/integration/observability/trace-events.js +2 -0
- package/dist/integration/observability/trace-logger.js +27 -17
- package/dist/integration/storage/jsonl.d.ts +19 -1
- package/dist/integration/storage/jsonl.js +96 -3
- package/dist/paths.d.ts +19 -0
- package/dist/paths.js +38 -1
- package/dist/subsystems/safety-guard/config.yaml +3 -1
- package/package.json +2 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
|
|
5
5
|
*/
|
|
6
6
|
import { EventEmitter } from 'node:events';
|
|
7
7
|
import { FormatAdapterRegistry } from './adapters.js';
|
|
@@ -46,44 +46,44 @@ export interface KnowledgeIngestOptions {
|
|
|
46
46
|
parseConcurrency?: number;
|
|
47
47
|
/** 文档(PDF/Office)抽取并发(默认 2;大文件抢占 CPU/内存) */
|
|
48
48
|
documentParseConcurrency?: number;
|
|
49
|
-
/**
|
|
49
|
+
/** 队列深度上限(超出只报警,不丢任务) */
|
|
50
50
|
maxQueueDepth?: number;
|
|
51
|
-
/** fs watch debounce ms
|
|
51
|
+
/** fs watch debounce ms(默认 800) */
|
|
52
52
|
debounceMs?: number;
|
|
53
|
-
/**
|
|
53
|
+
/** 单文件大小兜底上限(可选;格式分级见 fileLimits) */
|
|
54
54
|
maxFileBytes?: number;
|
|
55
55
|
/**
|
|
56
|
-
*
|
|
57
|
-
*
|
|
56
|
+
* 按格式分级的文件限额 + 部分抽取策略。
|
|
57
|
+
* 机器资源不足时下调 maxBytes / hardMaxFileBytes。
|
|
58
58
|
*/
|
|
59
59
|
fileLimits?: KnowledgeFileLimitsInput;
|
|
60
|
-
/**
|
|
60
|
+
/** 单文件 parse 超时 ms(默认 45000) */
|
|
61
61
|
parseTimeoutMs?: number;
|
|
62
|
-
/** Phase B embedding
|
|
62
|
+
/** Phase B embedding(未配则纯关键词) */
|
|
63
63
|
embeddingProvider?: EmbeddingProvider | null;
|
|
64
|
-
/** embed
|
|
64
|
+
/** embed 批大小(默认 32) */
|
|
65
65
|
embedBatch?: number;
|
|
66
|
-
/** embed
|
|
66
|
+
/** embed 最小间隔 ms(限速,默认 0) */
|
|
67
67
|
embedMinIntervalMs?: number;
|
|
68
|
-
/** embed
|
|
68
|
+
/** embed 并发(默认 4;与 parse 分槽) */
|
|
69
69
|
embedConcurrency?: number;
|
|
70
|
-
/**
|
|
70
|
+
/** 磁盘水位告警(默认 false;仅告警不丢任务) */
|
|
71
71
|
diskWatermarkAlert?: boolean;
|
|
72
|
-
/**
|
|
72
|
+
/** 凭证库(source.authRef 解析;可选) */
|
|
73
73
|
credentials?: CredentialStore | null;
|
|
74
|
-
/**
|
|
74
|
+
/** 自定义 fetcher 覆盖(测试/扩展) */
|
|
75
75
|
fetchers?: Partial<Record<'local' | 'url' | 'connector', SourceFetcher>>;
|
|
76
|
-
/** connector
|
|
76
|
+
/** connector 注册表(默认含 rest) */
|
|
77
77
|
connectors?: ConnectorRegistry;
|
|
78
|
-
/** poll 调度 tick ms
|
|
78
|
+
/** poll 调度 tick ms(默认 60s) */
|
|
79
79
|
pollTickMs?: number;
|
|
80
|
-
/**
|
|
80
|
+
/** 单源最小 poll 间隔 ms(默认 15min) */
|
|
81
81
|
pollMinIntervalMs?: number;
|
|
82
|
-
/** 每轮 poll
|
|
82
|
+
/** 每轮 poll 最多处理源数(成本上限,默认 8) */
|
|
83
83
|
maxPollPerTick?: number;
|
|
84
84
|
/**
|
|
85
|
-
* DocumentPort:启用后 PDF/Office 等走抽取
|
|
86
|
-
*
|
|
85
|
+
* DocumentPort:启用后 PDF/Office 等走抽取 → Markdown 切块
|
|
86
|
+
* (缺省 null = 维持 BINARY skip 语义)
|
|
87
87
|
*/
|
|
88
88
|
documentPort?: DocumentPort | null;
|
|
89
89
|
/**
|
|
@@ -91,10 +91,10 @@ export interface KnowledgeIngestOptions {
|
|
|
91
91
|
*/
|
|
92
92
|
documentConfig?: import('../capabilities/document/factory.js').DocumentCapabilityConfig | null;
|
|
93
93
|
/**
|
|
94
|
-
* embedding
|
|
95
|
-
* - allow
|
|
96
|
-
* - redact
|
|
97
|
-
* - skip
|
|
94
|
+
* embedding 外发前敏感形态策略(默认 redact)
|
|
95
|
+
* - allow:原文外发(内网/本地模型)
|
|
96
|
+
* - redact:命中替换为 [REDACTED:rule] 再 embed
|
|
97
|
+
* - skip:命中则不写向量(关键词仍可搜)
|
|
98
98
|
*/
|
|
99
99
|
embedSecretPolicy?: EmbedSecretPolicy;
|
|
100
100
|
}
|
|
@@ -108,7 +108,7 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
108
108
|
private readonly debounceMs;
|
|
109
109
|
private readonly maxFileBytes;
|
|
110
110
|
private readonly fileLimits;
|
|
111
|
-
/**
|
|
111
|
+
/** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
|
|
112
112
|
private readonly parseTimeoutMs;
|
|
113
113
|
private readonly embedding;
|
|
114
114
|
private readonly embedBatch;
|
|
@@ -128,12 +128,12 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
128
128
|
private readonly embedRunner;
|
|
129
129
|
private pollTimer;
|
|
130
130
|
private reconcileTimer;
|
|
131
|
-
/**
|
|
131
|
+
/** 源级中止:DB 权威 + 本进程 AbortSignal(跨重启) */
|
|
132
132
|
private readonly jobControl;
|
|
133
133
|
private readonly jobQueue;
|
|
134
|
-
/**
|
|
134
|
+
/** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
|
|
135
135
|
onSourceSettled?: (sourceId: string) => void;
|
|
136
|
-
/**
|
|
136
|
+
/** 上次 parse 缺口扫描时间 */
|
|
137
137
|
private lastParseGapScanAt;
|
|
138
138
|
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
139
139
|
private gapScanUrgent;
|
|
@@ -143,9 +143,9 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
143
143
|
private runningEmbed;
|
|
144
144
|
private draining;
|
|
145
145
|
private kickAgain;
|
|
146
|
-
/** source
|
|
146
|
+
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
147
147
|
private sourceLocks;
|
|
148
|
-
/** walk 发现的文件数(Phase A coverage
|
|
148
|
+
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
149
149
|
private discoveredFiles;
|
|
150
150
|
private watchers;
|
|
151
151
|
private debounceTimers;
|
|
@@ -153,10 +153,10 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
153
153
|
constructor(options: KnowledgeIngestOptions);
|
|
154
154
|
private lastCleanupAt;
|
|
155
155
|
/**
|
|
156
|
-
*
|
|
156
|
+
* 清掉「非普通文件」误入 knowledge_files 的行(目录曾被标 no_adapter)。
|
|
157
157
|
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
158
|
+
* 仅处理 skipped/error 且无 chunk 的行:有 chunk 的仍走 pruneMissing/磁盘对齐。
|
|
159
|
+
* 默认 60s 节流,避免每轮 reconcile 扫全表 stat。
|
|
160
160
|
*
|
|
161
161
|
* @param sourceId - 限定源;省略则全部源
|
|
162
162
|
* @param opts.force - 忽略节流
|
|
@@ -166,16 +166,16 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
166
166
|
force?: boolean;
|
|
167
167
|
}): number;
|
|
168
168
|
/**
|
|
169
|
-
* 无跨进程 Lease
|
|
170
|
-
*
|
|
169
|
+
* 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
|
|
170
|
+
* 仅构造时调用——运行中同进程的 running 是合法的。
|
|
171
171
|
*/
|
|
172
172
|
private reclaimOrphanRunningJobs;
|
|
173
173
|
get indexStore(): KnowledgeIndexStore;
|
|
174
174
|
/**
|
|
175
|
-
* 过滤:文档扩展名交给 worker
|
|
175
|
+
* 过滤:文档扩展名交给 worker 抽取,不被 BINARY 短路。
|
|
176
176
|
*
|
|
177
|
-
* @param p -
|
|
178
|
-
* @returns
|
|
177
|
+
* @param p - 文件路径
|
|
178
|
+
* @returns 是否跳过
|
|
179
179
|
*/
|
|
180
180
|
private shouldSkipFile;
|
|
181
181
|
/** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
|
|
@@ -183,18 +183,18 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
183
183
|
/** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
|
|
184
184
|
private identityKeyFor;
|
|
185
185
|
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
186
|
+
* 路径归属:`parse_file` / 本地读文件只允许读源 root 内的路径。
|
|
187
|
+
* 并做 **realpath** 比较,防符号链接逃出 root。
|
|
188
188
|
*
|
|
189
|
-
* @param source -
|
|
190
|
-
* @param path -
|
|
189
|
+
* @param source - 知识源
|
|
190
|
+
* @param path - 请求路径(本地绝对路径,或外源逻辑键)
|
|
191
191
|
*/
|
|
192
192
|
private sourceOwnsPath;
|
|
193
193
|
/**
|
|
194
|
-
* 全量/增量索引某源(Phase A
|
|
194
|
+
* 全量/增量索引某源(Phase A)
|
|
195
195
|
*
|
|
196
|
-
* @param opts.full - 显式重建(supersede
|
|
197
|
-
* @param opts.incremental -
|
|
196
|
+
* @param opts.full - 显式重建(supersede 未完成任务);poll/增量默认不打断在跑
|
|
197
|
+
* @param opts.incremental - 增量同步(poll):有 active 任务则跳过本轮
|
|
198
198
|
*/
|
|
199
199
|
ingestSource(sourceId: string, opts?: {
|
|
200
200
|
full?: boolean;
|
|
@@ -212,34 +212,34 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
212
212
|
*/
|
|
213
213
|
private ingestRemoteSource;
|
|
214
214
|
/**
|
|
215
|
-
*
|
|
215
|
+
* 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
|
|
216
216
|
*
|
|
217
|
-
* @returns
|
|
218
|
-
* @throws authRef
|
|
217
|
+
* @returns 凭证或 null(仅当源未配置 authRef)
|
|
218
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
|
|
219
219
|
*/
|
|
220
220
|
private resolveCred;
|
|
221
221
|
/**
|
|
222
|
-
*
|
|
222
|
+
* 将 VirtualDocument 写入索引(外源 / 规范化后文本)
|
|
223
223
|
*
|
|
224
|
-
* @returns
|
|
224
|
+
* @returns 是否成功写入
|
|
225
225
|
*/
|
|
226
226
|
indexVirtualDoc(sourceId: string, doc: VirtualDocument, authRef?: string | null): Promise<boolean>;
|
|
227
227
|
/**
|
|
228
|
-
*
|
|
228
|
+
* 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
|
|
229
229
|
*/
|
|
230
230
|
ingestFileNow(sourceId: string, filePath: string): Promise<boolean>;
|
|
231
231
|
/**
|
|
232
|
-
*
|
|
232
|
+
* 重做指定文件:强制重新 解析 → 分块 → 向量
|
|
233
233
|
*
|
|
234
|
-
* -
|
|
235
|
-
* - 失效 contentHash
|
|
236
|
-
* -
|
|
237
|
-
* -
|
|
234
|
+
* - 路径必须归属本源(防越权读盘);外源逻辑键走 fetch_doc 重取
|
|
235
|
+
* - 失效 contentHash(忽略 isFresh),旧 chunks 保留到 upsert 成功
|
|
236
|
+
* - **源处于中止态时自动 resume**:显式重做 = 意图要跑活,不允许静默丢弃
|
|
237
|
+
* - 目录等非文件:直接清脏行,不假装入队
|
|
238
238
|
* - 完成后走既有 embed 链路
|
|
239
239
|
*
|
|
240
|
-
* @param sourceId -
|
|
241
|
-
* @param paths -
|
|
242
|
-
* @returns queued=新入队;alreadyActive
|
|
240
|
+
* @param sourceId - 源 id
|
|
241
|
+
* @param paths - 文件路径列表
|
|
242
|
+
* @returns queued=新入队;alreadyActive=已在队列/执行;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属本源
|
|
243
243
|
*/
|
|
244
244
|
reprocessFiles(sourceId: string, paths: string[]): Promise<{
|
|
245
245
|
queued: number;
|
|
@@ -248,13 +248,13 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
248
248
|
resumed: boolean;
|
|
249
249
|
rejected: number;
|
|
250
250
|
}>;
|
|
251
|
-
/**
|
|
251
|
+
/** 路径上是否已有 queued/running 的 parse/fetch 任务 */
|
|
252
252
|
private hasActiveParseJob;
|
|
253
253
|
/**
|
|
254
|
-
*
|
|
254
|
+
* 路径级任务/文件状态(UI 跟踪重做完成)
|
|
255
255
|
*
|
|
256
|
-
* @param sourceId -
|
|
257
|
-
* @param paths -
|
|
256
|
+
* @param sourceId - 源 id
|
|
257
|
+
* @param paths - 路径列表
|
|
258
258
|
*/
|
|
259
259
|
jobStateForPaths(sourceId: string, paths: string[]): Array<{
|
|
260
260
|
path: string;
|
|
@@ -265,10 +265,10 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
265
265
|
exists: boolean;
|
|
266
266
|
}>;
|
|
267
267
|
/**
|
|
268
|
-
*
|
|
268
|
+
* 按筛选批量重做(当前列表的 status/ext/q 条件)
|
|
269
269
|
*
|
|
270
|
-
* @param sourceId -
|
|
271
|
-
* @param opts -
|
|
270
|
+
* @param sourceId - 源 id
|
|
271
|
+
* @param opts - 与文件列表筛选一致
|
|
272
272
|
* @returns 入队条数
|
|
273
273
|
*/
|
|
274
274
|
reprocessByFilter(sourceId: string, opts?: {
|
|
@@ -287,20 +287,20 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
287
287
|
*/
|
|
288
288
|
handleFsChange(sourceId: string, filePath: string, event: 'change' | 'rename'): void;
|
|
289
289
|
/**
|
|
290
|
-
*
|
|
290
|
+
* 启动目录 watch(幂等)
|
|
291
291
|
*/
|
|
292
292
|
startWatch(sourceId: string): void;
|
|
293
293
|
stopWatch(sourceId: string): void;
|
|
294
294
|
/**
|
|
295
|
-
*
|
|
295
|
+
* 等待队列排空(测试/管理)
|
|
296
296
|
*/
|
|
297
297
|
idle(timeoutMs?: number): Promise<void>;
|
|
298
298
|
dispose(): void;
|
|
299
299
|
/**
|
|
300
|
-
*
|
|
300
|
+
* 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
|
|
301
301
|
*
|
|
302
|
-
* @param opts.sourceId -
|
|
303
|
-
* @returns
|
|
302
|
+
* @param opts.sourceId - 仅中止该源;省略则全部源
|
|
303
|
+
* @returns 取消统计
|
|
304
304
|
*/
|
|
305
305
|
abortJobs(opts?: {
|
|
306
306
|
sourceId?: string;
|
|
@@ -310,15 +310,15 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
310
310
|
runningJobs: number;
|
|
311
311
|
};
|
|
312
312
|
/**
|
|
313
|
-
*
|
|
313
|
+
* 恢复/继续索引(相对「重建」更轻)
|
|
314
314
|
*
|
|
315
|
-
* -
|
|
316
|
-
* -
|
|
315
|
+
* - 清除该源中止态
|
|
316
|
+
* - 把 cancelled 任务放回 queued
|
|
317
317
|
* - 若仍缺向量则排队 embed
|
|
318
|
-
* -
|
|
318
|
+
* - 不重新 walk 全量;需要全量对齐请 reindex
|
|
319
319
|
*
|
|
320
|
-
* @param opts.sourceId -
|
|
321
|
-
* @returns
|
|
320
|
+
* @param opts.sourceId - 仅恢复该源;省略则全部源
|
|
321
|
+
* @returns 恢复统计
|
|
322
322
|
*/
|
|
323
323
|
resumeJobs(opts?: {
|
|
324
324
|
sourceId?: string;
|
|
@@ -326,67 +326,67 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
326
326
|
restoredCancelled: number;
|
|
327
327
|
embedQueued: number;
|
|
328
328
|
};
|
|
329
|
-
/**
|
|
329
|
+
/** 开启新的未中止纪元(resume / supersede 后用;清 DB 中止态) */
|
|
330
330
|
private beginAbortEpoch;
|
|
331
331
|
/**
|
|
332
|
-
* 重建前作废本源未完成任务(supersede
|
|
332
|
+
* 重建前作废本源未完成任务(supersede)
|
|
333
333
|
*
|
|
334
|
-
* - queued
|
|
335
|
-
* - running 打断(worker
|
|
336
|
-
* - 随后 beginAbortEpoch
|
|
334
|
+
* - queued → cancelled(superseded_by_reindex)
|
|
335
|
+
* - running 打断(worker/循环退出)
|
|
336
|
+
* - 随后 beginAbortEpoch,本轮重建可继续写
|
|
337
337
|
*/
|
|
338
338
|
private supersedeSourceWork;
|
|
339
339
|
private ensureParseCoverage;
|
|
340
340
|
/** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
|
|
341
341
|
private requestGapScan;
|
|
342
342
|
/**
|
|
343
|
-
*
|
|
343
|
+
* 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
|
|
344
344
|
*
|
|
345
|
-
*
|
|
346
|
-
* embed
|
|
345
|
+
* 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
|
|
346
|
+
* embed 前调用,避免给已移出文件的残 chunks 上向量。
|
|
347
347
|
*
|
|
348
|
-
* @param sourceId -
|
|
348
|
+
* @param sourceId - 知识源 id
|
|
349
349
|
* @returns 删除的文件数
|
|
350
350
|
*/
|
|
351
351
|
pruneMissingOnDisk(sourceId: string): Promise<number>;
|
|
352
|
-
/**
|
|
352
|
+
/** 逐文件 stat(file 源 / walk 失败回退);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
353
353
|
private pruneByStatScan;
|
|
354
354
|
private abortSignalFor;
|
|
355
355
|
private isAborted;
|
|
356
356
|
/**
|
|
357
|
-
*
|
|
357
|
+
* 中止/继续 控制面读数(互斥按钮用)
|
|
358
358
|
*
|
|
359
|
-
* @param sourceId -
|
|
360
|
-
* @returns 队列计数 +
|
|
359
|
+
* @param sourceId - 源 id
|
|
360
|
+
* @returns 队列计数 + 是否已中止 + 是否仍缺向量
|
|
361
361
|
*/
|
|
362
362
|
jobControlState(sourceId: string): KnowledgeJobControlState;
|
|
363
363
|
/**
|
|
364
|
-
*
|
|
364
|
+
* 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
|
|
365
365
|
*
|
|
366
|
-
*
|
|
367
|
-
* 1. 回收超时 running
|
|
368
|
-
* 2. 有缺向量且无 active embed
|
|
369
|
-
* 3.
|
|
366
|
+
* 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
|
|
367
|
+
* 1. 回收超时 running(崩溃/挂死孤儿)
|
|
368
|
+
* 2. 有缺向量且无 active embed → 自动排队
|
|
369
|
+
* 3. 无队列、无缺口 → 刷 coverage/status
|
|
370
370
|
*/
|
|
371
371
|
startReconciler(intervalMs?: number): void;
|
|
372
372
|
stopReconciler(): void;
|
|
373
|
-
/**
|
|
373
|
+
/** 对账入口(也可手工调用) */
|
|
374
374
|
reconcileJobs(): Promise<void>;
|
|
375
|
-
/**
|
|
375
|
+
/** 终态 job 清理(done/failed/cancelled 超过保留期) */
|
|
376
376
|
private cleanupTerminalJobs;
|
|
377
|
-
/** running 超时(无 heartbeat)→ 收回 queued
|
|
377
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
|
|
378
378
|
private reclaimStaleRunningJobs;
|
|
379
379
|
private countActiveJobs;
|
|
380
380
|
private hasRunningKind;
|
|
381
381
|
/**
|
|
382
|
-
*
|
|
382
|
+
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
383
383
|
*/
|
|
384
384
|
startPolling(): void;
|
|
385
385
|
stopPolling(): void;
|
|
386
386
|
/**
|
|
387
|
-
*
|
|
387
|
+
* 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
|
|
388
388
|
*
|
|
389
|
-
* @returns
|
|
389
|
+
* @returns 本轮实际触发的 sourceId 列表
|
|
390
390
|
*/
|
|
391
391
|
pollDueSources(): Promise<string[]>;
|
|
392
392
|
private hasQueuedJobs;
|
|
@@ -413,22 +413,22 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
413
413
|
private claimJob;
|
|
414
414
|
private runJob;
|
|
415
415
|
/**
|
|
416
|
-
*
|
|
417
|
-
*
|
|
416
|
+
* 带截止时间的可取消执行:超时会 abort signal,工作侧必须在副作用前检查。
|
|
417
|
+
* 禁止再用 Promise.race 包不可取消的 Promise——那会在超时后留下僵尸写入。
|
|
418
418
|
*/
|
|
419
419
|
private withTimeout;
|
|
420
420
|
/**
|
|
421
|
-
*
|
|
421
|
+
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
422
422
|
*/
|
|
423
423
|
private fetchDocJob;
|
|
424
424
|
private parseOne;
|
|
425
425
|
/**
|
|
426
|
-
* PDF/Office:DocumentPort
|
|
427
|
-
* 新鲜度以原件字节 hash
|
|
426
|
+
* PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
|
|
427
|
+
* 新鲜度以原件字节 hash 为准。
|
|
428
428
|
*
|
|
429
|
-
* @param sourceId -
|
|
430
|
-
* @param filePath -
|
|
431
|
-
* @returns
|
|
429
|
+
* @param sourceId - 知识源 id
|
|
430
|
+
* @param filePath - 本地文件路径
|
|
431
|
+
* @returns 是否成功入索引
|
|
432
432
|
*/
|
|
433
433
|
private parseDocumentFile;
|
|
434
434
|
/** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
|
|
@@ -438,21 +438,21 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
438
438
|
/**
|
|
439
439
|
* Phase B:嵌入待处理 chunk
|
|
440
440
|
*
|
|
441
|
-
*
|
|
442
|
-
*
|
|
441
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
442
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
443
443
|
*/
|
|
444
444
|
private embedPending;
|
|
445
445
|
private recordEmbedSecretHit;
|
|
446
|
-
/**
|
|
446
|
+
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
447
447
|
private ensureEmbedJob;
|
|
448
448
|
private heartbeatJob;
|
|
449
|
-
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile
|
|
449
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷新 */
|
|
450
450
|
private coverageDirty;
|
|
451
451
|
private markCoverageDirty;
|
|
452
452
|
private flushCoverageDirty;
|
|
453
453
|
private lastCoverageRefreshAt;
|
|
454
454
|
private refreshCoverage;
|
|
455
455
|
private emitProgress;
|
|
456
|
-
/**
|
|
456
|
+
/** 磁盘水位告警(仅告警,不丢任务) */
|
|
457
457
|
private warnDiskWatermark;
|
|
458
458
|
}
|