@octopi-agent/engine 0.86.38 → 0.97.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-admin/agent-presets.d.ts +70 -0
- package/dist/config-admin/agent-presets.js +243 -0
- package/dist/config-admin/discover.d.ts +30 -0
- package/dist/config-admin/discover.js +48 -0
- package/dist/config-admin/env.d.ts +22 -0
- package/dist/config-admin/env.js +41 -0
- package/dist/config-admin/index.d.ts +20 -0
- package/dist/config-admin/index.js +15 -0
- package/dist/config-admin/model-presets.d.ts +75 -0
- package/dist/config-admin/model-presets.js +249 -0
- package/dist/config-admin/name-guard.d.ts +14 -0
- package/dist/config-admin/name-guard.js +24 -0
- package/dist/config-admin/patch.d.ts +56 -0
- package/dist/config-admin/patch.js +201 -0
- package/dist/config-admin/port.d.ts +68 -0
- package/dist/config-admin/port.js +21 -0
- package/dist/config-admin/redact.d.ts +39 -0
- package/dist/config-admin/redact.js +42 -0
- package/dist/config-schema/agent.d.ts +43 -1
- package/dist/config-schema/agent.js +12 -5
- package/dist/config-schema/context.d.ts +7 -2
- package/dist/config-schema/context.js +8 -0
- package/dist/config-schema/engine.d.ts +110 -8
- package/dist/config-schema/governance.d.ts +43 -1
- package/dist/config-schema/governance.js +160 -2
- package/dist/config-schema/models.d.ts +7 -4
- package/dist/config-schema/models.js +4 -1
- package/dist/config-schema/observability.d.ts +9 -0
- package/dist/config-schema/observability.js +29 -0
- package/dist/config-schema/run.d.ts +3 -0
- package/dist/config-schema/run.js +14 -0
- package/dist/config-schema/shared.d.ts +3 -0
- package/dist/config-schema/shared.js +12 -0
- package/dist/config-schema/substrate.d.ts +1 -1
- package/dist/config.d.ts +60 -13
- package/dist/config.js +81 -78
- package/dist/harness/activation/dispatcher.d.ts +8 -2
- package/dist/harness/activation/dispatcher.js +33 -5
- package/dist/harness/activation/inbox.d.ts +87 -0
- package/dist/harness/activation/inbox.js +169 -0
- package/dist/harness/activation/runtime.d.ts +9 -8
- package/dist/harness/activation/runtime.js +100 -8
- package/dist/harness/activation/types.d.ts +19 -0
- package/dist/harness/activation/types.js +2 -0
- package/dist/harness/agent/builder.d.ts +18 -3
- package/dist/harness/agent/builder.js +42 -17
- package/dist/harness/agent/config-bridge.d.ts +5 -0
- package/dist/harness/agent/config-bridge.js +7 -1
- package/dist/harness/agent/config-integrity.d.ts +72 -0
- package/dist/harness/agent/config-integrity.js +146 -0
- package/dist/harness/agent/tool-purpose.d.ts +9 -1
- package/dist/harness/agent/tool-purpose.js +11 -2
- package/dist/harness/agent/wire-agent-builder.js +11 -3
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.d.ts +11 -1
- package/dist/harness/collaboration/autonomous-subsystem/boundary/validator.js +11 -1
- package/dist/harness/collaboration/autonomous-subsystem/loader.d.ts +13 -0
- package/dist/harness/collaboration/autonomous-subsystem/loader.js +16 -1
- package/dist/harness/collaboration/autonomous-subsystem/runtime.d.ts +4 -0
- package/dist/harness/collaboration/autonomous-subsystem/runtime.js +3 -1
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +3 -1
- package/dist/harness/collaboration/delegation/child-run.js +21 -5
- package/dist/harness/collaboration/delegation/notifier.js +4 -3
- package/dist/harness/collaboration/delegation/service.js +2 -3
- package/dist/harness/collaboration/delegation/wiring.js +6 -5
- package/dist/harness/collaboration/orchestration/reflector/llm-reflector.d.ts +1 -1
- package/dist/harness/collaboration/orchestration/reflector/llm-reflector.js +1 -1
- package/dist/harness/context/capabilities/compact/engine.d.ts +30 -0
- package/dist/harness/context/capabilities/compact/engine.js +198 -46
- package/dist/harness/context/capabilities/compact/handover.d.ts +78 -0
- package/dist/harness/context/capabilities/compact/handover.js +107 -0
- package/dist/harness/context/capabilities/compact/index.d.ts +3 -1
- package/dist/harness/context/capabilities/compact/index.js +2 -1
- package/dist/harness/context/capabilities/compact/types.d.ts +30 -1
- package/dist/harness/context/capabilities/summary/executor.js +2 -1
- package/dist/harness/context/capabilities/summary/policies.js +8 -5
- package/dist/harness/context/constitution/default-agents.md +11 -10
- package/dist/harness/context/default-context-engine.d.ts +34 -4
- package/dist/harness/context/default-context-engine.js +198 -53
- package/dist/harness/context/hybrid-compressor.js +6 -0
- package/dist/harness/context/layer-types.d.ts +36 -0
- package/dist/harness/context/layer-types.js +38 -0
- package/dist/harness/context/llm-summarizer.d.ts +2 -8
- package/dist/harness/context/llm-summarizer.js +69 -91
- package/dist/harness/context/message-selector.js +4 -3
- package/dist/harness/context/root-instructions.js +1 -1
- package/dist/harness/context/summarize.d.ts +2 -0
- package/dist/harness/context/summarize.js +7 -2
- package/dist/harness/context/system-prompt-assembler.js +10 -4
- package/dist/harness/context/tool-result-aging.js +2 -2
- package/dist/harness/context/truncate-compressor.js +7 -0
- package/dist/harness/context/turn-envelope.d.ts +4 -3
- package/dist/harness/context/turn-envelope.js +4 -3
- package/dist/harness/context/types.d.ts +33 -1
- package/dist/harness/extension/execution-environment/dangerous-commands.d.ts +28 -0
- package/dist/harness/extension/execution-environment/dangerous-commands.js +129 -0
- package/dist/harness/extension/execution-environment/effect-policy.d.ts +133 -4
- package/dist/harness/extension/execution-environment/effect-policy.js +263 -14
- package/dist/harness/extension/execution-environment/index.d.ts +4 -2
- package/dist/harness/extension/execution-environment/index.js +2 -1
- package/dist/harness/extension/execution-environment/isolation.js +5 -1
- package/dist/harness/extension/execution-environment/kill-switch.d.ts +34 -0
- package/dist/harness/extension/execution-environment/kill-switch.js +46 -0
- package/dist/harness/extension/execution-environment/ops-audit.d.ts +200 -0
- package/dist/harness/extension/execution-environment/ops-audit.js +338 -0
- package/dist/harness/extension/execution-environment/privileged-exec.d.ts +158 -0
- package/dist/harness/extension/execution-environment/privileged-exec.js +418 -0
- package/dist/harness/extension/execution-environment/process-info.d.ts +180 -0
- package/dist/harness/extension/execution-environment/process-info.js +466 -0
- package/dist/harness/extension/execution-environment/process-isolation.d.ts +7 -0
- package/dist/harness/extension/execution-environment/process-isolation.js +9 -0
- package/dist/harness/extension/execution-environment/sandbox.d.ts +109 -0
- package/dist/harness/extension/execution-environment/sandbox.js +264 -29
- package/dist/harness/extension/execution-environment/tool-capabilities.d.ts +1 -1
- package/dist/harness/extension/execution-environment/tool-capabilities.js +6 -0
- package/dist/harness/extension/execution-environment/types.d.ts +19 -0
- package/dist/harness/extension/plugin-ecosystem/channel-types.d.ts +5 -0
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.d.ts +6 -0
- package/dist/harness/extension/plugin-ecosystem/tools/allowlisted-run.js +50 -0
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.d.ts +4 -5
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +5 -15
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +3 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +53 -2
- package/dist/harness/extension/plugin-ecosystem/tools/file-search.js +10 -17
- package/dist/harness/extension/plugin-ecosystem/tools/git-runner.d.ts +31 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-runner.js +104 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.d.ts +111 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-status-snapshot.js +197 -0
- package/dist/harness/extension/plugin-ecosystem/tools/git-tools.d.ts +8 -8
- package/dist/harness/extension/plugin-ecosystem/tools/git-tools.js +9 -100
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +6 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +4 -0
- package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.d.ts +43 -0
- package/dist/harness/extension/plugin-ecosystem/tools/privileged-tool.js +350 -0
- package/dist/harness/extension/plugin-ecosystem/tools/process-tools.d.ts +30 -0
- package/dist/harness/extension/plugin-ecosystem/tools/process-tools.js +137 -0
- package/dist/harness/governance/human-in-the-loop/decision-cache.d.ts +54 -4
- package/dist/harness/governance/human-in-the-loop/decision-cache.js +42 -16
- package/dist/harness/governance/security/temp-path.d.ts +9 -0
- package/dist/harness/governance/security/temp-path.js +13 -2
- package/dist/harness/governance/session-acl/seed-roles.js +14 -0
- package/dist/harness/index.d.ts +4 -0
- package/dist/harness/index.js +3 -0
- package/dist/harness/knowledge/file-identity.d.ts +8 -0
- package/dist/harness/knowledge/file-identity.js +26 -1
- package/dist/harness/knowledge/fts.js +1 -2
- package/dist/harness/knowledge/grounding.js +5 -5
- package/dist/harness/knowledge/index-store.d.ts +31 -4
- package/dist/harness/knowledge/index-store.js +180 -48
- package/dist/harness/knowledge/ingest.d.ts +115 -115
- package/dist/harness/knowledge/ingest.js +150 -150
- package/dist/harness/knowledge/knowledge-serve-child.js +1 -1
- package/dist/harness/knowledge/purge.js +4 -1
- package/dist/harness/memory/backfill-coverage.js +3 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +1 -0
- package/dist/harness/memory/sqlite/keyword-search.js +4 -3
- package/dist/harness/observability/observer/hub.js +38 -1
- package/dist/harness/observability/observer/run-snapshot.js +1 -0
- package/dist/harness/observability/observer/types.d.ts +27 -0
- package/dist/harness/run/agent/agent.d.ts +7 -0
- package/dist/harness/run/agent/agent.js +23 -1
- package/dist/harness/run/reliability/fallback-provider.d.ts +117 -5
- package/dist/harness/run/reliability/fallback-provider.js +262 -22
- package/dist/harness/run/reliability/harness-events.d.ts +24 -1
- package/dist/harness/run/reliability/model-error-policy.d.ts +32 -0
- package/dist/harness/run/reliability/model-error-policy.js +42 -0
- package/dist/harness/run/reliability/provider-wrapper.js +29 -2
- package/dist/harness/run/reliability/refusal-guard.d.ts +20 -0
- package/dist/harness/run/reliability/refusal-guard.js +102 -0
- package/dist/harness/run/reliability/run-agent.js +231 -33
- package/dist/harness/run/run-guard/default-run-guard.js +1 -1
- package/dist/harness/run/run-scope.d.ts +31 -0
- package/dist/harness/run/runner.d.ts +132 -2
- package/dist/harness/run/runner.js +679 -60
- package/dist/harness/session/attachments/llm-intent.js +2 -1
- package/dist/harness/session/history/port.js +15 -5
- package/dist/harness/session/in-memory-store.d.ts +5 -0
- package/dist/harness/session/in-memory-store.js +33 -2
- package/dist/harness/session/tasks/render.d.ts +4 -1
- package/dist/harness/session/tasks/render.js +10 -7
- package/dist/harness/session/title.js +25 -9
- package/dist/harness/shared/events/agent-event-map.d.ts +92 -0
- package/dist/harness/shared/events/agent-event-map.js +7 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.js +4 -3
- package/dist/integration/agent-runtime/channel-message-source.js +19 -1
- package/dist/integration/index.d.ts +2 -0
- package/dist/integration/index.js +1 -0
- package/dist/integration/observability/loop-watchdog.d.ts +56 -0
- package/dist/integration/observability/loop-watchdog.js +177 -0
- package/dist/integration/observability/metrics.d.ts +3 -0
- package/dist/integration/observability/metrics.js +18 -7
- package/dist/integration/observability/run-telemetry.d.ts +8 -0
- package/dist/integration/observability/run-telemetry.js +31 -8
- package/dist/integration/observability/trace-collector.js +26 -4
- package/dist/integration/observability/trace-events.d.ts +1 -0
- package/dist/integration/observability/trace-events.js +2 -0
- package/dist/integration/observability/trace-logger.js +27 -17
- package/dist/integration/providers/anthropic.d.ts +23 -0
- package/dist/integration/providers/anthropic.js +97 -12
- package/dist/integration/providers/fetch-timeout.d.ts +42 -0
- package/dist/integration/providers/fetch-timeout.js +51 -0
- package/dist/integration/providers/openai.d.ts +10 -0
- package/dist/integration/providers/openai.js +58 -11
- package/dist/integration/providers/provider-diagnostics.d.ts +80 -0
- package/dist/integration/providers/provider-diagnostics.js +89 -0
- package/dist/integration/providers/timeout-coherence.d.ts +52 -0
- package/dist/integration/providers/timeout-coherence.js +68 -0
- package/dist/integration/providers/wire-dump.d.ts +62 -0
- package/dist/integration/providers/wire-dump.js +111 -2
- package/dist/integration/storage/jsonl.d.ts +19 -1
- package/dist/integration/storage/jsonl.js +100 -4
- package/dist/paths.d.ts +19 -0
- package/dist/paths.js +38 -1
- package/dist/subsystems/cognition-steward/conceptualize/handler.js +1 -1
- package/dist/subsystems/memory-steward/backfill/handler.js +1 -1
- package/dist/subsystems/safety-guard/config.yaml +3 -1
- package/dist/subsystems/wisdom-steward/formation/handler.js +1 -1
- package/package.json +3 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
|
|
5
5
|
*/
|
|
6
6
|
import { randomUUID, createHash } from 'node:crypto';
|
|
7
7
|
import { open, stat } from 'node:fs/promises';
|
|
@@ -65,7 +65,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
65
65
|
debounceMs;
|
|
66
66
|
maxFileBytes;
|
|
67
67
|
fileLimits;
|
|
68
|
-
/**
|
|
68
|
+
/** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
|
|
69
69
|
parseTimeoutMs;
|
|
70
70
|
embedding;
|
|
71
71
|
embedBatch;
|
|
@@ -85,12 +85,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
85
85
|
embedRunner;
|
|
86
86
|
pollTimer = null;
|
|
87
87
|
reconcileTimer = null;
|
|
88
|
-
/**
|
|
88
|
+
/** 源级中止:DB 权威 + 本进程 AbortSignal(跨重启) */
|
|
89
89
|
jobControl;
|
|
90
90
|
jobQueue;
|
|
91
|
-
/**
|
|
91
|
+
/** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
|
|
92
92
|
onSourceSettled;
|
|
93
|
-
/**
|
|
93
|
+
/** 上次 parse 缺口扫描时间 */
|
|
94
94
|
lastParseGapScanAt = new Map();
|
|
95
95
|
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
96
96
|
gapScanUrgent = new Set();
|
|
@@ -100,9 +100,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
100
100
|
runningEmbed = 0;
|
|
101
101
|
draining = false;
|
|
102
102
|
kickAgain = false;
|
|
103
|
-
/** source
|
|
103
|
+
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
104
104
|
sourceLocks = new Set();
|
|
105
|
-
/** walk 发现的文件数(Phase A coverage
|
|
105
|
+
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
106
106
|
discoveredFiles = new Map();
|
|
107
107
|
watchers = new Map();
|
|
108
108
|
debounceTimers = new Map();
|
|
@@ -129,7 +129,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
129
129
|
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
130
130
|
this.parseConcurrency = options.parseConcurrency ?? 8;
|
|
131
131
|
this.documentParseConcurrency = options.documentParseConcurrency ?? 2;
|
|
132
|
-
// watch
|
|
132
|
+
// watch 批量落盘时缩短静默窗,新增文件更快入队
|
|
133
133
|
this.debounceMs = options.debounceMs ?? 800;
|
|
134
134
|
this.fileLimits = resolveKnowledgeFileLimits({
|
|
135
135
|
...options.fileLimits,
|
|
@@ -141,7 +141,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
141
141
|
this.embedding = options.embeddingProvider ?? null;
|
|
142
142
|
this.embedBatch = options.embedBatch ?? 32;
|
|
143
143
|
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
144
|
-
// embed
|
|
144
|
+
// embed 与 parse 分槽;缺省 4 路并行打 embedding API
|
|
145
145
|
this.embedConcurrency = options.embedConcurrency ?? 4;
|
|
146
146
|
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
147
147
|
this.credentials = options.credentials ?? null;
|
|
@@ -161,7 +161,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
161
161
|
emitProgress: (evt) => this.emitProgress(evt),
|
|
162
162
|
recordSecretHit: (input) => this.recordEmbedSecretHit(input),
|
|
163
163
|
});
|
|
164
|
-
// embedding
|
|
164
|
+
// embedding 外发审计(合规可查;非权威计量)
|
|
165
165
|
this.sources.database.raw.exec(`
|
|
166
166
|
CREATE TABLE IF NOT EXISTS knowledge_embed_secret_log (
|
|
167
167
|
id TEXT PRIMARY KEY,
|
|
@@ -181,17 +181,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
181
181
|
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
182
182
|
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
183
183
|
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
184
|
-
//
|
|
184
|
+
// 崩溃/重启遗留的 running 会堵住 enqueue 去重并永久显示 indexing
|
|
185
185
|
this.reclaimOrphanRunningJobs();
|
|
186
|
-
//
|
|
186
|
+
// 历史脏行:目录等非文件曾被标成 no_adapter,启动即清
|
|
187
187
|
this.cleanupNonFileIndexRows();
|
|
188
188
|
}
|
|
189
189
|
lastCleanupAt = 0;
|
|
190
190
|
/**
|
|
191
|
-
*
|
|
191
|
+
* 清掉「非普通文件」误入 knowledge_files 的行(目录曾被标 no_adapter)。
|
|
192
192
|
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
193
|
+
* 仅处理 skipped/error 且无 chunk 的行:有 chunk 的仍走 pruneMissing/磁盘对齐。
|
|
194
|
+
* 默认 60s 节流,避免每轮 reconcile 扫全表 stat。
|
|
195
195
|
*
|
|
196
196
|
* @param sourceId - 限定源;省略则全部源
|
|
197
197
|
* @param opts.force - 忽略节流
|
|
@@ -223,15 +223,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
223
223
|
}
|
|
224
224
|
}
|
|
225
225
|
catch {
|
|
226
|
-
//
|
|
226
|
+
// 路径不存在:留给 pruneMissing(磁盘对齐),避免误删有效但暂时不可见的文件
|
|
227
227
|
}
|
|
228
228
|
}
|
|
229
229
|
}
|
|
230
230
|
return removed;
|
|
231
231
|
}
|
|
232
232
|
/**
|
|
233
|
-
* 无跨进程 Lease
|
|
234
|
-
*
|
|
233
|
+
* 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
|
|
234
|
+
* 仅构造时调用——运行中同进程的 running 是合法的。
|
|
235
235
|
*/
|
|
236
236
|
reclaimOrphanRunningJobs() {
|
|
237
237
|
this.jobQueue.reclaimOrphanRunning();
|
|
@@ -240,10 +240,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
240
240
|
return this.index;
|
|
241
241
|
}
|
|
242
242
|
/**
|
|
243
|
-
* 过滤:文档扩展名交给 worker
|
|
243
|
+
* 过滤:文档扩展名交给 worker 抽取,不被 BINARY 短路。
|
|
244
244
|
*
|
|
245
|
-
* @param p -
|
|
246
|
-
* @returns
|
|
245
|
+
* @param p - 文件路径
|
|
246
|
+
* @returns 是否跳过
|
|
247
247
|
*/
|
|
248
248
|
shouldSkipFile(p) {
|
|
249
249
|
if (isDocumentPath(p))
|
|
@@ -277,11 +277,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
277
277
|
}
|
|
278
278
|
}
|
|
279
279
|
/**
|
|
280
|
-
*
|
|
281
|
-
*
|
|
280
|
+
* 路径归属:`parse_file` / 本地读文件只允许读源 root 内的路径。
|
|
281
|
+
* 并做 **realpath** 比较,防符号链接逃出 root。
|
|
282
282
|
*
|
|
283
|
-
* @param source -
|
|
284
|
-
* @param path -
|
|
283
|
+
* @param source - 知识源
|
|
284
|
+
* @param path - 请求路径(本地绝对路径,或外源逻辑键)
|
|
285
285
|
*/
|
|
286
286
|
sourceOwnsPath(source, path) {
|
|
287
287
|
if (source.kind === 'url' || source.kind === 'connector') {
|
|
@@ -290,13 +290,13 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
290
290
|
const root = resolve(source.location);
|
|
291
291
|
const target = resolve(path);
|
|
292
292
|
const eq = (a, b) => process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b;
|
|
293
|
-
//
|
|
293
|
+
// 词法比较(路径不存在时也拒绝明显越界)
|
|
294
294
|
if (!eq(root, target)) {
|
|
295
295
|
const relLex = relative(root, target);
|
|
296
296
|
if (!relLex || relLex.startsWith('..') || isAbsolute(relLex))
|
|
297
297
|
return false;
|
|
298
298
|
}
|
|
299
|
-
// realpath
|
|
299
|
+
// realpath 比较(symlink stat 失败回词法结果)
|
|
300
300
|
try {
|
|
301
301
|
const rootReal = realpathSync(root);
|
|
302
302
|
const targetReal = realpathSync(target);
|
|
@@ -310,10 +310,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
310
310
|
}
|
|
311
311
|
}
|
|
312
312
|
/**
|
|
313
|
-
* 全量/增量索引某源(Phase A
|
|
313
|
+
* 全量/增量索引某源(Phase A)
|
|
314
314
|
*
|
|
315
|
-
* @param opts.full - 显式重建(supersede
|
|
316
|
-
* @param opts.incremental -
|
|
315
|
+
* @param opts.full - 显式重建(supersede 未完成任务);poll/增量默认不打断在跑
|
|
316
|
+
* @param opts.incremental - 增量同步(poll):有 active 任务则跳过本轮
|
|
317
317
|
*/
|
|
318
318
|
async ingestSource(sourceId, opts) {
|
|
319
319
|
// fromQueue(walk_source):不 supersede、不跳过 active;reindex 才作废队列
|
|
@@ -353,7 +353,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
353
353
|
return;
|
|
354
354
|
}
|
|
355
355
|
if (this.sourceLocks.has(sourceId)) {
|
|
356
|
-
//
|
|
356
|
+
// 已有同源任务在跑:入队 walk 即可(去重)
|
|
357
357
|
await this.enqueue(sourceId, 'walk_source', null, 1);
|
|
358
358
|
return;
|
|
359
359
|
}
|
|
@@ -362,20 +362,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
362
362
|
// 保留旧 coverage:重建期间索引仍在库中,写 0 会让 UI 误判为清空
|
|
363
363
|
this.sources.update(sourceId, { status: 'discovering' });
|
|
364
364
|
this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
|
|
365
|
-
// full 不再 clearSource
|
|
365
|
+
// full 不再 clearSource:靠本轮 keepPaths 差量 prune,discover 失败不丢库
|
|
366
366
|
if (source.kind === 'url' || source.kind === 'connector') {
|
|
367
367
|
await this.ingestRemoteSource(source);
|
|
368
368
|
return;
|
|
369
369
|
}
|
|
370
|
-
// watch 策略索引同时挂上 fs
|
|
370
|
+
// watch 策略索引同时挂上 fs 监听(幂等;否则后续变更静默丢失)
|
|
371
371
|
this.startWatch(sourceId);
|
|
372
372
|
const localDiscovery = await this.localFetcher.discover(source);
|
|
373
373
|
const files = localDiscovery.docs;
|
|
374
374
|
this.discoveredFiles.set(sourceId, files.length);
|
|
375
|
-
//
|
|
375
|
+
// 中止后不 prune、不入队(半截 keep 会误删)
|
|
376
376
|
if (this.isAborted(sourceId))
|
|
377
377
|
return;
|
|
378
|
-
//
|
|
378
|
+
// 差量 prune:仅 walk 完整时执行;出错/截断时 keep 不完整会误删
|
|
379
379
|
if (localDiscovery.complete) {
|
|
380
380
|
const localKeep = new Set(files.map((f) => f.path));
|
|
381
381
|
// walk 成功且目录为空时允许清空 Membership(§6.2)
|
|
@@ -456,7 +456,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
456
456
|
return;
|
|
457
457
|
}
|
|
458
458
|
const refs = discovered.docs;
|
|
459
|
-
// 条件 GET
|
|
459
|
+
// 条件 GET:带上已有 etag/Last-Modified,304 则整文档跳过
|
|
460
460
|
const enriched = refs.map((ref) => {
|
|
461
461
|
const existing = this.index.getFile(source.id, ref.path);
|
|
462
462
|
return {
|
|
@@ -476,7 +476,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
476
476
|
let okCount = 0;
|
|
477
477
|
let unchanged = 0;
|
|
478
478
|
for (const ref of enriched) {
|
|
479
|
-
//
|
|
479
|
+
// 中止后立刻停:不再外发、不再写索引
|
|
480
480
|
if (this.isAborted(source.id)) {
|
|
481
481
|
this.emitProgress({
|
|
482
482
|
type: 'source',
|
|
@@ -489,7 +489,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
489
489
|
try {
|
|
490
490
|
const doc = await fetcher.fetch(source, ref, cred);
|
|
491
491
|
if (!doc) {
|
|
492
|
-
// 304 /
|
|
492
|
+
// 304 / 未变:保持原索引
|
|
493
493
|
unchanged += 1;
|
|
494
494
|
okCount += 1;
|
|
495
495
|
continue;
|
|
@@ -509,8 +509,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
509
509
|
});
|
|
510
510
|
}
|
|
511
511
|
}
|
|
512
|
-
//
|
|
513
|
-
//
|
|
512
|
+
// 差量 prune:仅当 discover **完整** 时才可删;截断/walk 失败时 keep 不完整,prune 会误删
|
|
513
|
+
// 中止后不做 prune:半截 keep 会误删已入库文档
|
|
514
514
|
if (!this.isAborted(source.id) && discovered.complete) {
|
|
515
515
|
const keep = new Set(enriched.map((r) => r.path));
|
|
516
516
|
const pruned = await this.index.pruneMissingAsync(source.id, keep, { allowEmptyKeep: true });
|
|
@@ -536,10 +536,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
536
536
|
});
|
|
537
537
|
}
|
|
538
538
|
/**
|
|
539
|
-
*
|
|
539
|
+
* 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
|
|
540
540
|
*
|
|
541
|
-
* @returns
|
|
542
|
-
* @throws authRef
|
|
541
|
+
* @returns 凭证或 null(仅当源未配置 authRef)
|
|
542
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
|
|
543
543
|
*/
|
|
544
544
|
async resolveCred(source) {
|
|
545
545
|
if (!source.authRef)
|
|
@@ -557,16 +557,16 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
557
557
|
return resolved;
|
|
558
558
|
}
|
|
559
559
|
/**
|
|
560
|
-
*
|
|
560
|
+
* 将 VirtualDocument 写入索引(外源 / 规范化后文本)
|
|
561
561
|
*
|
|
562
|
-
* @returns
|
|
562
|
+
* @returns 是否成功写入
|
|
563
563
|
*/
|
|
564
564
|
async indexVirtualDoc(sourceId, doc, authRef) {
|
|
565
565
|
if (!doc.content?.trim()) {
|
|
566
566
|
this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
|
|
567
567
|
return false;
|
|
568
568
|
}
|
|
569
|
-
//
|
|
569
|
+
// 逻辑键可能无扩展名(connector id / URL path):按 MIME → 扩展名 → 内容形态回退
|
|
570
570
|
let adapter = this.adapters.match(doc.path, doc.contentType);
|
|
571
571
|
if (!adapter) {
|
|
572
572
|
if (looksLikeHtml(doc.content, doc.contentType))
|
|
@@ -615,7 +615,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
615
615
|
}
|
|
616
616
|
}
|
|
617
617
|
/**
|
|
618
|
-
*
|
|
618
|
+
* 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
|
|
619
619
|
*/
|
|
620
620
|
async ingestFileNow(sourceId, filePath) {
|
|
621
621
|
const ok = await this.parseOne(sourceId, filePath);
|
|
@@ -623,17 +623,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
623
623
|
return ok;
|
|
624
624
|
}
|
|
625
625
|
/**
|
|
626
|
-
*
|
|
626
|
+
* 重做指定文件:强制重新 解析 → 分块 → 向量
|
|
627
627
|
*
|
|
628
|
-
* -
|
|
629
|
-
* - 失效 contentHash
|
|
630
|
-
* -
|
|
631
|
-
* -
|
|
628
|
+
* - 路径必须归属本源(防越权读盘);外源逻辑键走 fetch_doc 重取
|
|
629
|
+
* - 失效 contentHash(忽略 isFresh),旧 chunks 保留到 upsert 成功
|
|
630
|
+
* - **源处于中止态时自动 resume**:显式重做 = 意图要跑活,不允许静默丢弃
|
|
631
|
+
* - 目录等非文件:直接清脏行,不假装入队
|
|
632
632
|
* - 完成后走既有 embed 链路
|
|
633
633
|
*
|
|
634
|
-
* @param sourceId -
|
|
635
|
-
* @param paths -
|
|
636
|
-
* @returns queued=新入队;alreadyActive
|
|
634
|
+
* @param sourceId - 源 id
|
|
635
|
+
* @param paths - 文件路径列表
|
|
636
|
+
* @returns queued=新入队;alreadyActive=已在队列/执行;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属本源
|
|
637
637
|
*/
|
|
638
638
|
async reprocessFiles(sourceId, paths) {
|
|
639
639
|
if (!paths.length) {
|
|
@@ -659,7 +659,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
659
659
|
continue;
|
|
660
660
|
}
|
|
661
661
|
if (isRemote) {
|
|
662
|
-
// 外源:path
|
|
662
|
+
// 外源:path 是逻辑键,重做 = 重新 fetch + 索引(不读本地盘)
|
|
663
663
|
this.index.invalidateFileForReparse(sourceId, p);
|
|
664
664
|
if (await this.enqueue(sourceId, 'fetch_doc', p, 1)) {
|
|
665
665
|
queued += 1;
|
|
@@ -669,7 +669,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
669
669
|
}
|
|
670
670
|
continue;
|
|
671
671
|
}
|
|
672
|
-
//
|
|
672
|
+
// 目录/设备节点:清脏行即可,没有「解析」可跑
|
|
673
673
|
try {
|
|
674
674
|
const st = statSync(p);
|
|
675
675
|
if (!st.isFile()) {
|
|
@@ -679,7 +679,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
679
679
|
}
|
|
680
680
|
}
|
|
681
681
|
catch {
|
|
682
|
-
//
|
|
682
|
+
// 路径不存在:仍按文件流程,让 parse 自清理
|
|
683
683
|
}
|
|
684
684
|
// 总是失效 hash:即使已在队列,跑起来也不能 isFresh 跳过
|
|
685
685
|
this.index.invalidateFileForReparse(sourceId, p);
|
|
@@ -693,15 +693,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
693
693
|
this.kick();
|
|
694
694
|
return { queued, alreadyActive, cleanedNonFiles, resumed, rejected };
|
|
695
695
|
}
|
|
696
|
-
/**
|
|
696
|
+
/** 路径上是否已有 queued/running 的 parse/fetch 任务 */
|
|
697
697
|
hasActiveParseJob(sourceId, path) {
|
|
698
698
|
return this.jobQueue.hasActivePathJob(sourceId, path);
|
|
699
699
|
}
|
|
700
700
|
/**
|
|
701
|
-
*
|
|
701
|
+
* 路径级任务/文件状态(UI 跟踪重做完成)
|
|
702
702
|
*
|
|
703
|
-
* @param sourceId -
|
|
704
|
-
* @param paths -
|
|
703
|
+
* @param sourceId - 源 id
|
|
704
|
+
* @param paths - 路径列表
|
|
705
705
|
*/
|
|
706
706
|
jobStateForPaths(sourceId, paths) {
|
|
707
707
|
return paths.map((p) => {
|
|
@@ -728,10 +728,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
728
728
|
});
|
|
729
729
|
}
|
|
730
730
|
/**
|
|
731
|
-
*
|
|
731
|
+
* 按筛选批量重做(当前列表的 status/ext/q 条件)
|
|
732
732
|
*
|
|
733
|
-
* @param sourceId -
|
|
734
|
-
* @param opts -
|
|
733
|
+
* @param sourceId - 源 id
|
|
734
|
+
* @param opts - 与文件列表筛选一致
|
|
735
735
|
* @returns 入队条数
|
|
736
736
|
*/
|
|
737
737
|
async reprocessByFilter(sourceId, opts) {
|
|
@@ -750,7 +750,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
750
750
|
clearTimeout(prev);
|
|
751
751
|
this.debounceTimers.set(key, setTimeout(() => {
|
|
752
752
|
this.debounceTimers.delete(key);
|
|
753
|
-
//
|
|
753
|
+
// 统一先 stat:目录误报 change 时不得 parse 成 no_adapter
|
|
754
754
|
void stat(filePath)
|
|
755
755
|
.then(async (st) => {
|
|
756
756
|
if (st.isFile()) {
|
|
@@ -776,7 +776,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
776
776
|
}, this.debounceMs));
|
|
777
777
|
}
|
|
778
778
|
/**
|
|
779
|
-
*
|
|
779
|
+
* 启动目录 watch(幂等)
|
|
780
780
|
*/
|
|
781
781
|
startWatch(sourceId) {
|
|
782
782
|
const source = this.sources.get(sourceId);
|
|
@@ -820,7 +820,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
820
820
|
}
|
|
821
821
|
}
|
|
822
822
|
/**
|
|
823
|
-
*
|
|
823
|
+
* 等待队列排空(测试/管理)
|
|
824
824
|
*/
|
|
825
825
|
async idle(timeoutMs = 30_000) {
|
|
826
826
|
const start = Date.now();
|
|
@@ -846,10 +846,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
846
846
|
this.stopWatch(id);
|
|
847
847
|
}
|
|
848
848
|
/**
|
|
849
|
-
*
|
|
849
|
+
* 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
|
|
850
850
|
*
|
|
851
|
-
* @param opts.sourceId -
|
|
852
|
-
* @returns
|
|
851
|
+
* @param opts.sourceId - 仅中止该源;省略则全部源
|
|
852
|
+
* @returns 取消统计
|
|
853
853
|
*/
|
|
854
854
|
abortJobs(opts) {
|
|
855
855
|
const sid = opts?.sourceId?.trim();
|
|
@@ -863,11 +863,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
863
863
|
let cancelledQueued = 0;
|
|
864
864
|
let abortedRunning = 0;
|
|
865
865
|
for (const id of unique) {
|
|
866
|
-
// 1) DB
|
|
866
|
+
// 1) DB 落中止态(跨重启)+ 打断本进程信号
|
|
867
867
|
const newly = this.jobControl.markAborted(id);
|
|
868
868
|
if (newly)
|
|
869
869
|
abortedRunning += 1;
|
|
870
|
-
// 2) 清空 queued
|
|
870
|
+
// 2) 清空 queued → cancelled(共享 File 的 parse 不杀:仍被领用则保留)
|
|
871
871
|
const res = this.sources.database.raw
|
|
872
872
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
873
873
|
WHERE source_id = ? AND status = 'queued'
|
|
@@ -882,7 +882,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
882
882
|
.run(Date.now(), id, id);
|
|
883
883
|
cancelledQueued += Number(res.changes ?? 0);
|
|
884
884
|
}
|
|
885
|
-
// 3)
|
|
885
|
+
// 3) 看门狗不要再把 aborted running 收回 queued(限定本次涉及的源)
|
|
886
886
|
if (sid) {
|
|
887
887
|
this.sources.database.raw
|
|
888
888
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
@@ -910,15 +910,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
910
910
|
return { cancelledQueued, abortedRunning, runningJobs };
|
|
911
911
|
}
|
|
912
912
|
/**
|
|
913
|
-
*
|
|
913
|
+
* 恢复/继续索引(相对「重建」更轻)
|
|
914
914
|
*
|
|
915
|
-
* -
|
|
916
|
-
* -
|
|
915
|
+
* - 清除该源中止态
|
|
916
|
+
* - 把 cancelled 任务放回 queued
|
|
917
917
|
* - 若仍缺向量则排队 embed
|
|
918
|
-
* -
|
|
918
|
+
* - 不重新 walk 全量;需要全量对齐请 reindex
|
|
919
919
|
*
|
|
920
|
-
* @param opts.sourceId -
|
|
921
|
-
* @returns
|
|
920
|
+
* @param opts.sourceId - 仅恢复该源;省略则全部源
|
|
921
|
+
* @returns 恢复统计
|
|
922
922
|
*/
|
|
923
923
|
resumeJobs(opts) {
|
|
924
924
|
const sid = opts?.sourceId?.trim();
|
|
@@ -933,15 +933,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
933
933
|
let restoredCancelled = 0;
|
|
934
934
|
let embedQueued = 0;
|
|
935
935
|
for (const id of ids) {
|
|
936
|
-
// 1)
|
|
936
|
+
// 1) 清除中止态(DB + 新纪元信号)
|
|
937
937
|
this.beginAbortEpoch(id);
|
|
938
|
-
// 2) cancelled
|
|
938
|
+
// 2) cancelled → queued(用户中止时被杀的活)
|
|
939
939
|
const res = this.sources.database.raw
|
|
940
940
|
.prepare(`UPDATE knowledge_jobs SET status = 'queued', last_error = NULL, updated_at = ?
|
|
941
941
|
WHERE source_id = ? AND status = 'cancelled' AND (last_error IS NULL OR last_error = 'aborted')`)
|
|
942
942
|
.run(Date.now(), id);
|
|
943
943
|
restoredCancelled += Number(res.changes ?? 0);
|
|
944
|
-
// 3)
|
|
944
|
+
// 3) 仍缺向量则确保 embed 在队
|
|
945
945
|
if (this.embedding) {
|
|
946
946
|
const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
|
|
947
947
|
const active = this.countActiveJobs(id);
|
|
@@ -960,23 +960,23 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
960
960
|
});
|
|
961
961
|
return { restoredCancelled, embedQueued };
|
|
962
962
|
}
|
|
963
|
-
/**
|
|
963
|
+
/** 开启新的未中止纪元(resume / supersede 后用;清 DB 中止态) */
|
|
964
964
|
beginAbortEpoch(sourceId) {
|
|
965
965
|
this.jobControl.beginEpoch(sourceId);
|
|
966
966
|
}
|
|
967
967
|
/**
|
|
968
|
-
* 重建前作废本源未完成任务(supersede
|
|
968
|
+
* 重建前作废本源未完成任务(supersede)
|
|
969
969
|
*
|
|
970
|
-
* - queued
|
|
971
|
-
* - running 打断(worker
|
|
972
|
-
* - 随后 beginAbortEpoch
|
|
970
|
+
* - queued → cancelled(superseded_by_reindex)
|
|
971
|
+
* - running 打断(worker/循环退出)
|
|
972
|
+
* - 随后 beginAbortEpoch,本轮重建可继续写
|
|
973
973
|
*/
|
|
974
974
|
supersedeSourceWork(sourceId) {
|
|
975
975
|
this.sources.database.raw
|
|
976
976
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
|
|
977
977
|
WHERE source_id = ? AND status = 'queued'`)
|
|
978
978
|
.run(Date.now(), sourceId);
|
|
979
|
-
// 打断在跑 parse/embed
|
|
979
|
+
// 打断在跑 parse/embed;再开新纪元(清中止态),避免 walk 自己被 abort 卡死
|
|
980
980
|
this.jobControl.markAborted(sourceId);
|
|
981
981
|
this.beginAbortEpoch(sourceId);
|
|
982
982
|
this.emitProgress({
|
|
@@ -1060,12 +1060,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1060
1060
|
this.gapScanTimer.unref?.();
|
|
1061
1061
|
}
|
|
1062
1062
|
/**
|
|
1063
|
-
*
|
|
1063
|
+
* 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
|
|
1064
1064
|
*
|
|
1065
|
-
*
|
|
1066
|
-
* embed
|
|
1065
|
+
* 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
|
|
1066
|
+
* embed 前调用,避免给已移出文件的残 chunks 上向量。
|
|
1067
1067
|
*
|
|
1068
|
-
* @param sourceId -
|
|
1068
|
+
* @param sourceId - 知识源 id
|
|
1069
1069
|
* @returns 删除的文件数
|
|
1070
1070
|
*/
|
|
1071
1071
|
async pruneMissingOnDisk(sourceId) {
|
|
@@ -1077,12 +1077,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1077
1077
|
try {
|
|
1078
1078
|
const found = await this.localFetcher.discover(source);
|
|
1079
1079
|
if (!found.complete) {
|
|
1080
|
-
// walk
|
|
1080
|
+
// walk 不完整:退回逐文件 stat,禁止用残缺 keep 清库
|
|
1081
1081
|
removed = await this.pruneByStatScan(sourceId);
|
|
1082
1082
|
}
|
|
1083
1083
|
else {
|
|
1084
1084
|
const keep = new Set(found.docs.map((f) => f.path));
|
|
1085
|
-
// walk
|
|
1085
|
+
// walk 完整:即使空目录也可清空(与逐文件 stat 语义一致)
|
|
1086
1086
|
removed = await this.index.pruneMissingAsync(sourceId, keep, { allowEmptyKeep: true });
|
|
1087
1087
|
}
|
|
1088
1088
|
}
|
|
@@ -1104,7 +1104,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1104
1104
|
}
|
|
1105
1105
|
return removed;
|
|
1106
1106
|
}
|
|
1107
|
-
/**
|
|
1107
|
+
/** 逐文件 stat(file 源 / walk 失败回退);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
1108
1108
|
async pruneByStatScan(sourceId, concurrency = 4) {
|
|
1109
1109
|
const files = this.index.listFiles(sourceId);
|
|
1110
1110
|
let removed = 0;
|
|
@@ -1141,10 +1141,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1141
1141
|
return this.jobControl.isAborted(sourceId);
|
|
1142
1142
|
}
|
|
1143
1143
|
/**
|
|
1144
|
-
*
|
|
1144
|
+
* 中止/继续 控制面读数(互斥按钮用)
|
|
1145
1145
|
*
|
|
1146
|
-
* @param sourceId -
|
|
1147
|
-
* @returns 队列计数 +
|
|
1146
|
+
* @param sourceId - 源 id
|
|
1147
|
+
* @returns 队列计数 + 是否已中止 + 是否仍缺向量
|
|
1148
1148
|
*/
|
|
1149
1149
|
jobControlState(sourceId) {
|
|
1150
1150
|
return readJobControlState(this.sources.database, this.index, sourceId, {
|
|
@@ -1152,12 +1152,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1152
1152
|
});
|
|
1153
1153
|
}
|
|
1154
1154
|
/**
|
|
1155
|
-
*
|
|
1155
|
+
* 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
|
|
1156
1156
|
*
|
|
1157
|
-
*
|
|
1158
|
-
* 1. 回收超时 running
|
|
1159
|
-
* 2. 有缺向量且无 active embed
|
|
1160
|
-
* 3.
|
|
1157
|
+
* 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
|
|
1158
|
+
* 1. 回收超时 running(崩溃/挂死孤儿)
|
|
1159
|
+
* 2. 有缺向量且无 active embed → 自动排队
|
|
1160
|
+
* 3. 无队列、无缺口 → 刷 coverage/status
|
|
1161
1161
|
*/
|
|
1162
1162
|
startReconciler(intervalMs = 15_000) {
|
|
1163
1163
|
if (this.reconcileTimer || this.disposed)
|
|
@@ -1180,21 +1180,21 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1180
1180
|
this.reconcileTimer = null;
|
|
1181
1181
|
}
|
|
1182
1182
|
}
|
|
1183
|
-
/**
|
|
1183
|
+
/** 对账入口(也可手工调用) */
|
|
1184
1184
|
async reconcileJobs() {
|
|
1185
1185
|
if (this.disposed)
|
|
1186
1186
|
return;
|
|
1187
|
-
// stale
|
|
1187
|
+
// stale 阈值必须 **大于** 硬超时,否则合法的长 parse 会被收回后双跑
|
|
1188
1188
|
this.reclaimStaleRunningJobs(this.fileLimits.maxParseTimeoutMs + 60_000);
|
|
1189
1189
|
// path: 与 win:/unix: 身份分裂的零认领残留
|
|
1190
1190
|
this.index.purgeOrphanPathIdentityFiles();
|
|
1191
|
-
//
|
|
1191
|
+
// 目录脏行等历史误入项(轻量;非每轮全表)
|
|
1192
1192
|
this.cleanupNonFileIndexRows();
|
|
1193
|
-
//
|
|
1193
|
+
// 终态 job 定期清理,防表无限增长
|
|
1194
1194
|
this.cleanupTerminalJobs();
|
|
1195
|
-
// 存量 FTS
|
|
1195
|
+
// 存量 FTS 后台补齐(分批让出事件循环;构造期禁止同步 rebuild)
|
|
1196
1196
|
void this.index.ensureFtsBackfill();
|
|
1197
|
-
// 存量向量回填:小批量 +
|
|
1197
|
+
// 存量向量回填:小批量 + 让出事件循环,禁止 2000 条同步写(会堵死 abort)
|
|
1198
1198
|
await this.index.backfillVecFromEmbeddings(25);
|
|
1199
1199
|
for (const s of this.sources.list()) {
|
|
1200
1200
|
if (s.status === 'removed' || s.status === 'disabled')
|
|
@@ -1208,8 +1208,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1208
1208
|
(s.kind === 'file' || s.kind === 'directory' || s.kind === 'workspace')) {
|
|
1209
1209
|
await this.ingestSource(s.id, { fromQueue: true }).catch(() => undefined);
|
|
1210
1210
|
}
|
|
1211
|
-
// 稳定态收尾(auto-describe
|
|
1212
|
-
// 先看 active,再 gap
|
|
1211
|
+
// 稳定态收尾(auto-describe 等)——纯关键词部署同样要触发
|
|
1212
|
+
// 先看 active,再 gap;避免刚入队的 gap job 挡住 settled
|
|
1213
1213
|
if (active.total === 0) {
|
|
1214
1214
|
this.refreshCoverage(s.id, { force: true });
|
|
1215
1215
|
try {
|
|
@@ -1222,7 +1222,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1222
1222
|
else {
|
|
1223
1223
|
this.flushCoverageDirty(s.id);
|
|
1224
1224
|
}
|
|
1225
|
-
// watch 漏事件 / skip
|
|
1225
|
+
// watch 漏事件 / skip 可重试时补齐 parse(与是否配置 embedding 无关)
|
|
1226
1226
|
// urgent(watch 突发)不走 60s 节流;否则批量拷贝后要等一整轮才补
|
|
1227
1227
|
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
1228
1228
|
const gapDue = this.gapScanUrgent.has(s.id) || Date.now() - lastGap > 60_000;
|
|
@@ -1245,11 +1245,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1245
1245
|
}
|
|
1246
1246
|
this.kick();
|
|
1247
1247
|
}
|
|
1248
|
-
/**
|
|
1248
|
+
/** 终态 job 清理(done/failed/cancelled 超过保留期) */
|
|
1249
1249
|
cleanupTerminalJobs(retainMs = 24 * 60 * 60_000) {
|
|
1250
1250
|
this.jobQueue.cleanupTerminal(retainMs);
|
|
1251
1251
|
}
|
|
1252
|
-
/** running 超时(无 heartbeat)→ 收回 queued
|
|
1252
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
|
|
1253
1253
|
reclaimStaleRunningJobs(staleMs) {
|
|
1254
1254
|
this.jobQueue.reclaimStaleRunning(staleMs);
|
|
1255
1255
|
}
|
|
@@ -1264,7 +1264,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1264
1264
|
return Boolean(row?.ok);
|
|
1265
1265
|
}
|
|
1266
1266
|
/**
|
|
1267
|
-
*
|
|
1267
|
+
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
1268
1268
|
*/
|
|
1269
1269
|
startPolling() {
|
|
1270
1270
|
if (this.pollTimer || this.disposed)
|
|
@@ -1279,7 +1279,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1279
1279
|
});
|
|
1280
1280
|
});
|
|
1281
1281
|
}, this.pollTickMs);
|
|
1282
|
-
//
|
|
1282
|
+
// 不阻止进程退出
|
|
1283
1283
|
this.pollTimer.unref?.();
|
|
1284
1284
|
}
|
|
1285
1285
|
stopPolling() {
|
|
@@ -1289,9 +1289,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1289
1289
|
}
|
|
1290
1290
|
}
|
|
1291
1291
|
/**
|
|
1292
|
-
*
|
|
1292
|
+
* 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
|
|
1293
1293
|
*
|
|
1294
|
-
* @returns
|
|
1294
|
+
* @returns 本轮实际触发的 sourceId 列表
|
|
1295
1295
|
*/
|
|
1296
1296
|
async pollDueSources() {
|
|
1297
1297
|
if (this.disposed)
|
|
@@ -1321,7 +1321,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1321
1321
|
status: 'poll_due',
|
|
1322
1322
|
detail: source.sync.intervalMs?.toString(),
|
|
1323
1323
|
});
|
|
1324
|
-
//
|
|
1324
|
+
// 异步入队,不阻塞 tick;增量:有 active 则跳过,禁止 supersede 打断
|
|
1325
1325
|
void this.ingestSource(source.id, { incremental: true }).catch((err) => {
|
|
1326
1326
|
this.emitProgress({
|
|
1327
1327
|
type: 'error',
|
|
@@ -1333,7 +1333,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1333
1333
|
}
|
|
1334
1334
|
return triggered;
|
|
1335
1335
|
}
|
|
1336
|
-
//
|
|
1336
|
+
// ── 内部 ──
|
|
1337
1337
|
hasQueuedJobs() {
|
|
1338
1338
|
return this.jobQueue.hasQueuedJobs();
|
|
1339
1339
|
}
|
|
@@ -1404,7 +1404,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1404
1404
|
if (this.disposed)
|
|
1405
1405
|
return;
|
|
1406
1406
|
if (this.draining) {
|
|
1407
|
-
//
|
|
1407
|
+
// 等待中的 drain 会再 kick;标记一次避免丢信号
|
|
1408
1408
|
this.kickAgain = true;
|
|
1409
1409
|
return;
|
|
1410
1410
|
}
|
|
@@ -1440,7 +1440,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1440
1440
|
}
|
|
1441
1441
|
const job = this.claimJob(kinds, (j) => this.acceptParseSlot(j));
|
|
1442
1442
|
if (!job) {
|
|
1443
|
-
//
|
|
1443
|
+
// 队列清空:合并刷新被 parse/drop 标脏的 coverage,避免每文件一次 COUNT
|
|
1444
1444
|
this.flushCoverageDirty();
|
|
1445
1445
|
return;
|
|
1446
1446
|
}
|
|
@@ -1503,7 +1503,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1503
1503
|
return;
|
|
1504
1504
|
}
|
|
1505
1505
|
try {
|
|
1506
|
-
// 归属闸门:任何会碰本地盘 /
|
|
1506
|
+
// 归属闸门:任何会碰本地盘 / 外源逻辑键的 path 任务都必须属于本源
|
|
1507
1507
|
if (job.path && (job.kind === 'parse_file' || job.kind === 'fetch_doc')) {
|
|
1508
1508
|
const owner = this.sources.get(job.source_id);
|
|
1509
1509
|
if (owner && !this.sourceOwnsPath(owner, job.path)) {
|
|
@@ -1511,7 +1511,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1511
1511
|
}
|
|
1512
1512
|
}
|
|
1513
1513
|
if (job.kind === 'parse_file' && job.path) {
|
|
1514
|
-
// 单文件解析硬超时:大 xlsx/pdf
|
|
1514
|
+
// 单文件解析硬超时:大 xlsx/pdf 同步抽取会堵死事件循环;超时随体积放大
|
|
1515
1515
|
let parseTimeoutMs = this.parseTimeoutMs;
|
|
1516
1516
|
try {
|
|
1517
1517
|
const st = await stat(job.path);
|
|
@@ -1519,7 +1519,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1519
1519
|
parseTimeoutMs = parseTimeoutForSize(st.size, this.fileLimits);
|
|
1520
1520
|
}
|
|
1521
1521
|
catch {
|
|
1522
|
-
//
|
|
1522
|
+
// 路径已删时交给 parseOne 自清理
|
|
1523
1523
|
}
|
|
1524
1524
|
this.heartbeatJob(job.id);
|
|
1525
1525
|
await this.withTimeout((signal) => this.parseOne(job.source_id, job.path, signal), parseTimeoutMs, `parse_timeout:${job.path}`, this.abortSignalFor(job.source_id));
|
|
@@ -1561,7 +1561,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1561
1561
|
if (!this.embedding) {
|
|
1562
1562
|
const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
|
|
1563
1563
|
if (stillMissing) {
|
|
1564
|
-
//
|
|
1564
|
+
// 禁止空跑标 done:迁移后缺向量却显示 queue=0
|
|
1565
1565
|
throw new Error('embedding_provider_missing');
|
|
1566
1566
|
}
|
|
1567
1567
|
return;
|
|
@@ -1571,7 +1571,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1571
1571
|
throw new Error('aborted');
|
|
1572
1572
|
}
|
|
1573
1573
|
}
|
|
1574
|
-
//
|
|
1574
|
+
// 统一收尾闸门:任何 kind 中止后都不得标 done
|
|
1575
1575
|
if (this.isAborted(job.source_id)) {
|
|
1576
1576
|
throw new Error('aborted');
|
|
1577
1577
|
}
|
|
@@ -1584,7 +1584,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1584
1584
|
path: job.path ?? undefined,
|
|
1585
1585
|
status: 'done',
|
|
1586
1586
|
});
|
|
1587
|
-
//
|
|
1587
|
+
// 标 done 后再续跑:否则 enqueue 去重会把「自己」当成 active 而丢掉下一棒
|
|
1588
1588
|
if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
|
|
1589
1589
|
this.ensureEmbedJob(job.source_id);
|
|
1590
1590
|
this.refreshCoverage(job.source_id, { force: true });
|
|
@@ -1593,7 +1593,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1593
1593
|
catch (err) {
|
|
1594
1594
|
const message = err instanceof Error ? err.message : String(err);
|
|
1595
1595
|
const aborted = message === 'aborted' || this.isAborted(job.source_id);
|
|
1596
|
-
//
|
|
1596
|
+
// 归属拒绝不是「文件解析失败」,不得写入 knowledge_files
|
|
1597
1597
|
const ownership = message.includes('not owned by source');
|
|
1598
1598
|
if (job.kind === 'parse_file' && job.path && !aborted && !ownership) {
|
|
1599
1599
|
this.index.markFileError(job.source_id, job.path, message);
|
|
@@ -1610,8 +1610,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1610
1610
|
}
|
|
1611
1611
|
}
|
|
1612
1612
|
/**
|
|
1613
|
-
*
|
|
1614
|
-
*
|
|
1613
|
+
* 带截止时间的可取消执行:超时会 abort signal,工作侧必须在副作用前检查。
|
|
1614
|
+
* 禁止再用 Promise.race 包不可取消的 Promise——那会在超时后留下僵尸写入。
|
|
1615
1615
|
*/
|
|
1616
1616
|
async withTimeout(work, ms, label, parent) {
|
|
1617
1617
|
const controller = new AbortController();
|
|
@@ -1636,7 +1636,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1636
1636
|
}
|
|
1637
1637
|
}
|
|
1638
1638
|
/**
|
|
1639
|
-
*
|
|
1639
|
+
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
1640
1640
|
*/
|
|
1641
1641
|
async fetchDocJob(sourceId, path, signal) {
|
|
1642
1642
|
const source = this.sources.get(sourceId);
|
|
@@ -1663,7 +1663,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1663
1663
|
}
|
|
1664
1664
|
async parseOne(sourceId, filePath, signal) {
|
|
1665
1665
|
const source = this.sources.get(sourceId);
|
|
1666
|
-
//
|
|
1666
|
+
// 防御深度:即使队列被污染,也不读源 root 外的本地文件
|
|
1667
1667
|
if (source && !this.sourceOwnsPath(source, filePath)) {
|
|
1668
1668
|
throw new Error(`path not owned by source: ${filePath}`);
|
|
1669
1669
|
}
|
|
@@ -1673,7 +1673,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1673
1673
|
this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
|
|
1674
1674
|
return false;
|
|
1675
1675
|
}
|
|
1676
|
-
//
|
|
1676
|
+
// 先确认是普通文件:目录/设备节点不得进 knowledge_files(历史上曾误标 no_adapter)
|
|
1677
1677
|
let st;
|
|
1678
1678
|
try {
|
|
1679
1679
|
st = await stat(filePath);
|
|
@@ -1688,7 +1688,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1688
1688
|
}
|
|
1689
1689
|
const adapter = this.adapters.match(filePath);
|
|
1690
1690
|
if (!adapter) {
|
|
1691
|
-
// worker
|
|
1691
|
+
// 文档优先 worker 抽取;注意进程内 port 仅作 worker 不可用时的回退
|
|
1692
1692
|
if (isDocumentPath(filePath)) {
|
|
1693
1693
|
return this.parseDocumentFile(sourceId, filePath, signal);
|
|
1694
1694
|
}
|
|
@@ -1749,12 +1749,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1749
1749
|
}
|
|
1750
1750
|
}
|
|
1751
1751
|
/**
|
|
1752
|
-
* PDF/Office:DocumentPort
|
|
1753
|
-
* 新鲜度以原件字节 hash
|
|
1752
|
+
* PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
|
|
1753
|
+
* 新鲜度以原件字节 hash 为准。
|
|
1754
1754
|
*
|
|
1755
|
-
* @param sourceId -
|
|
1756
|
-
* @param filePath -
|
|
1757
|
-
* @returns
|
|
1755
|
+
* @param sourceId - 知识源 id
|
|
1756
|
+
* @param filePath - 本地文件路径
|
|
1757
|
+
* @returns 是否成功入索引
|
|
1758
1758
|
*/
|
|
1759
1759
|
async parseDocumentFile(sourceId, filePath, signal) {
|
|
1760
1760
|
if (signal?.aborted)
|
|
@@ -1804,12 +1804,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1804
1804
|
}
|
|
1805
1805
|
catch (err) {
|
|
1806
1806
|
const msg = err instanceof Error ? err.message : String(err);
|
|
1807
|
-
//
|
|
1807
|
+
// 中止/超时不是「这个文件解析失败」,必须上抛让 job 标 failed,禁止静默 done
|
|
1808
1808
|
if (msg === 'aborted' || msg.startsWith('parse_timeout') || signal?.aborted) {
|
|
1809
1809
|
throw err;
|
|
1810
1810
|
}
|
|
1811
1811
|
if (isDocumentExtractError(err)) {
|
|
1812
|
-
//
|
|
1812
|
+
// 结构化跳过(密码/老格式/超大等),不挡整库
|
|
1813
1813
|
const identKey = await this.identityKeyFor(filePath);
|
|
1814
1814
|
this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase(), st.size, identKey);
|
|
1815
1815
|
return false;
|
|
@@ -1874,8 +1874,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1874
1874
|
/**
|
|
1875
1875
|
* Phase B:嵌入待处理 chunk
|
|
1876
1876
|
*
|
|
1877
|
-
*
|
|
1878
|
-
*
|
|
1877
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
1878
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
1879
1879
|
*/
|
|
1880
1880
|
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
1881
1881
|
await this.embedRunner.embedPending(sourceId, maxRounds, jobId);
|
|
@@ -1886,7 +1886,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1886
1886
|
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
1887
1887
|
.run(`kes_${randomUUID().slice(0, 12)}`, input.sourceId, input.chunkId, input.action, JSON.stringify(input.hits), Date.now());
|
|
1888
1888
|
}
|
|
1889
|
-
/**
|
|
1889
|
+
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
1890
1890
|
ensureEmbedJob(sourceId) {
|
|
1891
1891
|
if (!this.embedding)
|
|
1892
1892
|
return;
|
|
@@ -1901,7 +1901,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1901
1901
|
heartbeatJob(jobId) {
|
|
1902
1902
|
this.jobQueue.heartbeat(jobId);
|
|
1903
1903
|
}
|
|
1904
|
-
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile
|
|
1904
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷新 */
|
|
1905
1905
|
coverageDirty = new Set();
|
|
1906
1906
|
markCoverageDirty(sourceId) {
|
|
1907
1907
|
this.coverageDirty.add(sourceId);
|
|
@@ -1929,7 +1929,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1929
1929
|
if (!source)
|
|
1930
1930
|
return;
|
|
1931
1931
|
const discovered = this.discoveredFiles.get(sourceId) ?? 0;
|
|
1932
|
-
// Phase A:已落库文件(indexed+skipped+error
|
|
1932
|
+
// Phase A:已落库文件(indexed+skipped+error)/ walk 发现数
|
|
1933
1933
|
const phaseA = discovered > 0
|
|
1934
1934
|
? Math.min(1, stats.files / discovered)
|
|
1935
1935
|
: stats.files > 0
|
|
@@ -1987,7 +1987,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1987
1987
|
this.emit('progress', out);
|
|
1988
1988
|
this.emit('knowledge.index.progress', out);
|
|
1989
1989
|
}
|
|
1990
|
-
/**
|
|
1990
|
+
/** 磁盘水位告警(仅告警,不丢任务) */
|
|
1991
1991
|
async warnDiskWatermark(sourceId) {
|
|
1992
1992
|
try {
|
|
1993
1993
|
const { statfs } = await import('node:fs/promises');
|