@octopi-agent/engine 0.58.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +38 -1
  4. package/dist/config-schema/substrate.d.ts +37 -0
  5. package/dist/config-schema/substrate.js +47 -0
  6. package/dist/config.d.ts +30 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  23. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  24. package/dist/harness/capabilities/document/factory.js +37 -0
  25. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  37. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  39. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  40. package/dist/harness/capabilities/document/worker-port.js +78 -0
  41. package/dist/harness/context/capabilities/index.d.ts +2 -2
  42. package/dist/harness/context/capabilities/index.js +2 -2
  43. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  44. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  45. package/dist/harness/index.d.ts +4 -2
  46. package/dist/harness/index.js +2 -1
  47. package/dist/harness/knowledge/adapters.d.ts +8 -5
  48. package/dist/harness/knowledge/adapters.js +53 -13
  49. package/dist/harness/knowledge/client.d.ts +197 -0
  50. package/dist/harness/knowledge/client.js +237 -0
  51. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  52. package/dist/harness/knowledge/connectors.d.ts +3 -3
  53. package/dist/harness/knowledge/connectors.js +14 -8
  54. package/dist/harness/knowledge/db.d.ts +40 -5
  55. package/dist/harness/knowledge/db.js +441 -113
  56. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  57. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  58. package/dist/harness/knowledge/embed-runner.js +171 -0
  59. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  60. package/dist/harness/knowledge/engine-thread.js +302 -0
  61. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  62. package/dist/harness/knowledge/extract-document.js +32 -8
  63. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  64. package/dist/harness/knowledge/fetchers.js +52 -16
  65. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  66. package/dist/harness/knowledge/file-identity.js +172 -0
  67. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  68. package/dist/harness/knowledge/file-index-store.js +132 -0
  69. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  70. package/dist/harness/knowledge/file-limits.js +175 -0
  71. package/dist/harness/knowledge/fts.d.ts +60 -0
  72. package/dist/harness/knowledge/fts.js +218 -0
  73. package/dist/harness/knowledge/grounding.d.ts +3 -2
  74. package/dist/harness/knowledge/http-app.d.ts +50 -0
  75. package/dist/harness/knowledge/http-app.js +652 -0
  76. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  77. package/dist/harness/knowledge/http-bridge.js +17 -0
  78. package/dist/harness/knowledge/index-store.d.ts +158 -93
  79. package/dist/harness/knowledge/index-store.js +982 -441
  80. package/dist/harness/knowledge/index.d.ts +45 -5
  81. package/dist/harness/knowledge/index.js +23 -2
  82. package/dist/harness/knowledge/ingest.d.ts +262 -87
  83. package/dist/harness/knowledge/ingest.js +1072 -480
  84. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  85. package/dist/harness/knowledge/job-control-state.js +44 -0
  86. package/dist/harness/knowledge/job-control.d.ts +35 -0
  87. package/dist/harness/knowledge/job-control.js +86 -0
  88. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  89. package/dist/harness/knowledge/job-queue.js +173 -0
  90. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  91. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  92. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  93. package/dist/harness/knowledge/local-stack.js +63 -0
  94. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  95. package/dist/harness/knowledge/membership-store.js +198 -0
  96. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  97. package/dist/harness/knowledge/network-guard.js +130 -85
  98. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  99. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  100. package/dist/harness/knowledge/query-service.d.ts +272 -0
  101. package/dist/harness/knowledge/query-service.js +326 -0
  102. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  103. package/dist/harness/knowledge/query-worker-client.js +207 -0
  104. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  105. package/dist/harness/knowledge/query-worker.js +186 -0
  106. package/dist/harness/knowledge/read-http.d.ts +35 -0
  107. package/dist/harness/knowledge/read-http.js +373 -0
  108. package/dist/harness/knowledge/retriever.d.ts +22 -5
  109. package/dist/harness/knowledge/retriever.js +66 -35
  110. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  111. package/dist/harness/knowledge/secret-scan.js +25 -2
  112. package/dist/harness/knowledge/serve.d.ts +48 -0
  113. package/dist/harness/knowledge/serve.js +336 -0
  114. package/dist/harness/knowledge/source-store.d.ts +80 -17
  115. package/dist/harness/knowledge/source-store.js +160 -89
  116. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  117. package/dist/harness/knowledge/start-service-process.js +158 -0
  118. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  119. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  120. package/dist/harness/knowledge/types.d.ts +1 -1
  121. package/dist/harness/knowledge/types.js +1 -1
  122. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  123. package/dist/harness/knowledge/vector-ann.js +68 -0
  124. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  125. package/dist/harness/knowledge/writer-lock.js +71 -0
  126. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  127. package/dist/harness/knowledge/writer-service.js +355 -0
  128. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  129. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  130. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  131. package/dist/harness/knowledge/writer-worker.js +189 -0
  132. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  133. package/dist/harness/memory/sqlite/embedding.js +109 -16
  134. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  135. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  136. package/dist/harness/session/attachments/service.d.ts +1 -1
  137. package/dist/harness/session/attachments/service.js +1 -1
  138. package/dist/index.d.ts +4 -2
  139. package/dist/index.js +2 -1
  140. package/dist/worker-path.d.ts +8 -0
  141. package/dist/worker-path.js +15 -0
  142. package/package.json +2 -2
@@ -1,30 +1,71 @@
1
1
  /**
2
- * KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
2
+ * KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
3
3
  *
4
- * 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
4
+ * 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
5
5
  */
6
- import { randomUUID } from 'node:crypto';
7
- import { readFile, stat } from 'node:fs/promises';
6
+ import { randomUUID, createHash } from 'node:crypto';
7
+ import { open, stat } from 'node:fs/promises';
8
+ import { createReadStream, statSync, realpathSync } from 'node:fs';
8
9
  import { watch } from 'node:fs';
9
- import { join, sep } from 'node:path';
10
+ import { isAbsolute, join, relative, resolve, sep } from 'node:path';
10
11
  import { EventEmitter } from 'node:events';
11
- import { FormatAdapterRegistry, htmlAdapter, markdownAdapter, textAdapter } from './adapters.js';
12
+ import { FormatAdapterRegistry, htmlAdapter, looksLikeAxureExportJs, markdownAdapter, textAdapter, } from './adapters.js';
12
13
  import { looksLikeHtml } from './html.js';
13
14
  import { hashContent, KnowledgeIndexStore } from './index-store.js';
14
- import { LocalFsFetcher, UrlFetcher } from './fetchers.js';
15
+ import { identifyLocalFile, urlIdentityKey, normalizePathLexical, } from './file-identity.js';
16
+ import { KnowledgeJobControl } from './job-control.js';
17
+ import { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
15
18
  import { ConnectorRegistry } from './connectors.js';
16
19
  import { ConnectorFetcher } from './connector-fetcher.js';
17
- import { isDocumentPath } from '../context/capabilities/document/format.js';
18
- import { isDocumentExtractError } from '../context/capabilities/document/errors.js';
20
+ import { isDocumentPath } from '../capabilities/document/format.js';
21
+ import { isDocumentExtractError } from '../capabilities/document/errors.js';
22
+ import { classifyFileKind, decideBySize, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
23
+ import { EmbedRunner } from './embed-runner.js';
24
+ /**
25
+ * 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
26
+ *
27
+ * @param filePath - 本地文件
28
+ * @param signal - 中止信号
29
+ * @returns hex digest
30
+ */
31
+ export async function hashFileStreaming(filePath, signal) {
32
+ const hash = createHash('sha256');
33
+ await new Promise((resolve, reject) => {
34
+ if (signal?.aborted) {
35
+ reject(new Error('aborted'));
36
+ return;
37
+ }
38
+ const stream = createReadStream(filePath);
39
+ const onAbort = () => {
40
+ stream.destroy();
41
+ reject(new Error('aborted'));
42
+ };
43
+ signal?.addEventListener('abort', onAbort, { once: true });
44
+ stream.on('data', (chunk) => hash.update(chunk));
45
+ stream.once('end', () => {
46
+ signal?.removeEventListener('abort', onAbort);
47
+ resolve();
48
+ });
49
+ stream.once('error', (err) => {
50
+ signal?.removeEventListener('abort', onAbort);
51
+ reject(err);
52
+ });
53
+ });
54
+ return hash.digest('hex');
55
+ }
56
+ import { JobQueue } from './job-queue.js';
57
+ import { readJobControlState } from './job-control-state.js';
19
58
  export class KnowledgeIngest extends EventEmitter {
20
59
  sources;
21
60
  index;
22
61
  adapters;
23
62
  parseConcurrency;
63
+ documentParseConcurrency;
24
64
  maxQueueDepth;
25
65
  debounceMs;
26
66
  maxFileBytes;
27
- /** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
67
+ fileLimits;
68
+ /** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
28
69
  parseTimeoutMs;
29
70
  embedding;
30
71
  embedBatch;
@@ -39,22 +80,29 @@ export class KnowledgeIngest extends EventEmitter {
39
80
  pollMinIntervalMs;
40
81
  maxPollPerTick;
41
82
  documentPort;
42
- lastEmbedAt = 0;
83
+ documentConfig;
84
+ embedSecretPolicy;
85
+ embedRunner;
43
86
  pollTimer = null;
44
87
  reconcileTimer = null;
45
- /** 源级中止控制器(手动 abort / 重建前清场) */
46
- abortBySource = new Map();
47
- /** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
88
+ /** 源级:DB 权威 + ?AbortSignal(跨重启?*/
89
+ jobControl;
90
+ jobQueue;
91
+ /** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
48
92
  onSourceSettled;
49
- /** 上次 parse 缺口扫描时间 */
93
+ /** parse 缺口时间 */
50
94
  lastParseGapScanAt = new Map();
95
+ /** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
96
+ gapScanUrgent = new Set();
97
+ gapScanTimer = null;
51
98
  runningParse = 0;
99
+ runningDocParse = 0;
52
100
  runningEmbed = 0;
53
101
  draining = false;
54
102
  kickAgain = false;
55
- /** source 级锁:同一 source 不并行 walk/parse 冲突写 */
103
+ /** source 级锁:同 source 不并?walk/parse 冲突?*/
56
104
  sourceLocks = new Set();
57
- /** walk 发现的文件数(Phase A coverage 分母) */
105
+ /** walk 发现的文件数(Phase A coverage 分母?*/
58
106
  discoveredFiles = new Map();
59
107
  watchers = new Map();
60
108
  debounceTimers = new Map();
@@ -62,22 +110,68 @@ export class KnowledgeIngest extends EventEmitter {
62
110
  constructor(options) {
63
111
  super();
64
112
  this.sources = options.sourceStore;
113
+ this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
114
+ this.jobControl = new KnowledgeJobControl(options.sourceStore.database);
115
+ this.jobQueue = new JobQueue({
116
+ db: options.sourceStore.database,
117
+ maxQueueDepth: this.maxQueueDepth,
118
+ isAborted: (id) => this.isAborted(id),
119
+ onBackpressure: (sourceId, queued) => {
120
+ this.emitProgress({
121
+ type: 'error',
122
+ sourceId,
123
+ status: 'queue_backpressure',
124
+ detail: `queued=${queued} (delay only, jobs kept)`,
125
+ });
126
+ },
127
+ });
65
128
  this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
66
129
  this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
67
130
  this.parseConcurrency = options.parseConcurrency ?? 8;
68
- this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
69
- // watch 批量落盘时缩短静默窗,新增文件更快入队
131
+ this.documentParseConcurrency = options.documentParseConcurrency ?? 2;
132
+ // watch 批量落盘时缩默窗,新增文件更?
70
133
  this.debounceMs = options.debounceMs ?? 800;
71
- this.maxFileBytes = options.maxFileBytes ?? 5_000_000;
72
- this.parseTimeoutMs = options.parseTimeoutMs ?? 45_000;
134
+ this.fileLimits = resolveKnowledgeFileLimits({
135
+ ...options.fileLimits,
136
+ maxFileBytes: options.fileLimits?.maxFileBytes ?? options.maxFileBytes,
137
+ parseTimeoutMs: options.fileLimits?.parseTimeoutMs ?? options.parseTimeoutMs,
138
+ });
139
+ this.maxFileBytes = this.fileLimits.maxFileBytes;
140
+ this.parseTimeoutMs = this.fileLimits.parseTimeoutMs;
73
141
  this.embedding = options.embeddingProvider ?? null;
74
142
  this.embedBatch = options.embedBatch ?? 32;
75
143
  this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
76
- // embed 与 parse 分槽;缺省 4 路并行打 embedding API
144
+ // embed ?parse 分槽;缺?4 行打 embedding API
77
145
  this.embedConcurrency = options.embedConcurrency ?? 4;
78
146
  this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
79
147
  this.credentials = options.credentials ?? null;
80
148
  this.documentPort = options.documentPort ?? null;
149
+ this.documentConfig = options.documentConfig ?? null;
150
+ this.embedSecretPolicy = options.embedSecretPolicy ?? 'redact';
151
+ this.embedRunner = new EmbedRunner({
152
+ index: this.index,
153
+ embedding: this.embedding,
154
+ embedBatch: this.embedBatch,
155
+ embedConcurrency: this.embedConcurrency,
156
+ embedMinIntervalMs: this.embedMinIntervalMs,
157
+ embedSecretPolicy: this.embedSecretPolicy,
158
+ isAborted: (id) => this.isAborted(id),
159
+ heartbeatJob: (id) => this.heartbeatJob(id),
160
+ refreshCoverage: (id) => this.refreshCoverage(id),
161
+ emitProgress: (evt) => this.emitProgress(evt),
162
+ recordSecretHit: (input) => this.recordEmbedSecretHit(input),
163
+ });
164
+ // embedding 外发(合规可查;非权量)
165
+ this.sources.database.raw.exec(`
166
+ CREATE TABLE IF NOT EXISTS knowledge_embed_secret_log (
167
+ id TEXT PRIMARY KEY,
168
+ source_id TEXT NOT NULL,
169
+ chunk_id TEXT NOT NULL,
170
+ action TEXT NOT NULL,
171
+ hits_json TEXT NOT NULL,
172
+ created_at INTEGER NOT NULL
173
+ );
174
+ `);
81
175
  this.localFetcher =
82
176
  options.fetchers?.local ?? new LocalFsFetcher((p) => this.shouldSkipFile(p));
83
177
  this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
@@ -87,38 +181,157 @@ export class KnowledgeIngest extends EventEmitter {
87
181
  this.pollTickMs = options.pollTickMs ?? 60_000;
88
182
  this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
89
183
  this.maxPollPerTick = options.maxPollPerTick ?? 8;
90
- // 崩溃/重启遗留的 running 会堵住 enqueue 去重并永久显示 indexing
184
+ // 崩溃/重启遗留?running 会堵?enqueue 去重并永久显?indexing
91
185
  this.reclaimOrphanRunningJobs();
186
+ // 历史:目录等非文件曾?no_adapter,启动即?
187
+ this.cleanupNonFileIndexRows();
188
+ }
189
+ lastCleanupAt = 0;
190
+ /**
191
+ * 清掉「非?knowledge_files (目录曾 no_adapter?
192
+ *
193
+ * ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
194
+ * 60s 节流,避免每?reconcile ?stat?
195
+ *
196
+ * @param sourceId - 限定源;省略则全部源
197
+ * @param opts.force - 忽略节流
198
+ * @returns 删除行数
199
+ */
200
+ cleanupNonFileIndexRows(sourceId, opts) {
201
+ const now = Date.now();
202
+ if (!opts?.force && sourceId == null && now - this.lastCleanupAt < 60_000) {
203
+ return 0;
204
+ }
205
+ this.lastCleanupAt = now;
206
+ const ids = sourceId
207
+ ? [sourceId]
208
+ : this.sources.list().map((s) => s.id);
209
+ let removed = 0;
210
+ for (const id of ids) {
211
+ const rows = this.sources.database.raw
212
+ .prepare(`SELECT m.logical_path AS path
213
+ FROM knowledge_memberships m
214
+ JOIN knowledge_files f ON f.id = m.file_id
215
+ WHERE m.source_id = ? AND f.status IN ('skipped', 'error') AND f.chunk_count = 0`)
216
+ .all(id);
217
+ for (const r of rows) {
218
+ try {
219
+ const st = statSync(r.path);
220
+ if (!st.isFile()) {
221
+ this.index.removeFile(id, r.path);
222
+ removed += 1;
223
+ }
224
+ }
225
+ catch {
226
+ // 不存留给 pruneMissing齐),避删有效但暂时不可见的文件
227
+ }
228
+ }
229
+ }
230
+ return removed;
92
231
  }
93
232
  /**
94
- * 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
95
- * 仅构造时调用——运行中同进程的 running 是合法的。
233
+ * 无跨进程 Lease:本进程时把历史 running 收回 queued?
234
+ * 仅构造时调用行中同进程的 running 法的?
96
235
  */
97
236
  reclaimOrphanRunningJobs() {
98
- this.sources.database.raw
99
- .prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ? WHERE status = 'running'`)
100
- .run(Date.now());
237
+ this.jobQueue.reclaimOrphanRunning();
101
238
  }
102
239
  get indexStore() {
103
240
  return this.index;
104
241
  }
105
242
  /**
106
- * 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
243
+ * 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
107
244
  *
108
- * @param p - 文件路径
109
- * @returns 是否跳过
245
+ * @param p - 文件
246
+ * @returns 跳过
110
247
  */
111
248
  shouldSkipFile(p) {
112
- if (this.documentPort && isDocumentPath(p))
249
+ if (isDocumentPath(p))
113
250
  return false;
114
251
  return this.adapters.shouldSkipPath(p);
115
252
  }
253
+ /** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
254
+ async readFileHead(filePath, bytes = 256) {
255
+ try {
256
+ const fh = await open(filePath, 'r');
257
+ try {
258
+ const buf = Buffer.alloc(bytes);
259
+ const { bytesRead } = await fh.read(buf, 0, bytes, 0);
260
+ return buf.subarray(0, bytesRead).toString('utf8');
261
+ }
262
+ finally {
263
+ await fh.close();
264
+ }
265
+ }
266
+ catch {
267
+ return null;
268
+ }
269
+ }
270
+ /** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
271
+ async identityKeyFor(filePath) {
272
+ try {
273
+ return (await identifyLocalFile(filePath)).key;
274
+ }
275
+ catch {
276
+ return undefined;
277
+ }
278
+ }
116
279
  /**
117
- * 全量/增量索引某源(Phase A)
280
+ * 归属:`parse_file` / 读文件只读源 root 内的?
281
+ * ?**realpath** 比较,防符号链接逃出 root?
282
+ *
283
+ * @param source - 知识?
284
+ * @param path - 请求(本地绝对路径,
285
+ */
286
+ sourceOwnsPath(source, path) {
287
+ if (source.kind === 'url' || source.kind === 'connector') {
288
+ return Boolean(this.index.getFile(source.id, path));
289
+ }
290
+ const root = resolve(source.location);
291
+ const target = resolve(path);
292
+ const eq = (a, b) => process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b;
293
+ // 词法(路径不存在时也拒绝明显越界?
294
+ if (!eq(root, target)) {
295
+ const relLex = relative(root, target);
296
+ if (!relLex || relLex.startsWith('..') || isAbsolute(relLex))
297
+ return false;
298
+ }
299
+ // realpath ?symlink stat 失败回词法结果)
300
+ try {
301
+ const rootReal = realpathSync(root);
302
+ const targetReal = realpathSync(target);
303
+ if (eq(rootReal, targetReal))
304
+ return true;
305
+ const rel = relative(rootReal, targetReal);
306
+ return Boolean(rel) && !rel.startsWith('..') && !isAbsolute(rel);
307
+ }
308
+ catch {
309
+ return true; // 词法;文件尚在时交给后续 stat
310
+ }
311
+ }
312
+ /**
313
+ * 全量/增量索引某源(Phase A?
314
+ *
315
+ * @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
316
+ * @param opts.incremental - 增量(poll):?active 任务则跳过本?
118
317
  */
119
318
  async ingestSource(sourceId, opts) {
120
- // 重建 = supersede:以本次磁盘扫描为准,作废本源未完成任务
121
- this.supersedeSourceWork(sourceId);
319
+ // fromQueue(walk_source):不 supersede、不跳过 active;reindex 才作废队列
320
+ if (opts?.fromQueue) {
321
+ // fall through to walk
322
+ }
323
+ else if (opts?.full !== false && !opts?.incremental) {
324
+ this.supersedeSourceWork(sourceId);
325
+ }
326
+ else if (this.countActiveJobs(sourceId).total > 0) {
327
+ this.emitProgress({
328
+ type: 'source',
329
+ sourceId,
330
+ status: 'poll_skipped',
331
+ detail: 'active jobs in flight',
332
+ });
333
+ return;
334
+ }
122
335
  const source = this.sources.get(sourceId);
123
336
  if (!source || source.status === 'removed') {
124
337
  throw new Error(`knowledge source not found: ${sourceId}`);
@@ -140,24 +353,34 @@ export class KnowledgeIngest extends EventEmitter {
140
353
  return;
141
354
  }
142
355
  if (this.sourceLocks.has(sourceId)) {
143
- // 已有同源任务在跑:入队 walk 即可(去重)
144
- this.enqueue(sourceId, 'walk_source', null, 1, source.location);
356
+ // 已有同源任务在跑:入?walk 即可(去重)
357
+ await this.enqueue(sourceId, 'walk_source', null, 1);
145
358
  return;
146
359
  }
147
360
  this.sourceLocks.add(sourceId);
148
361
  try {
149
- this.sources.update(sourceId, { status: 'discovering', coverage: 0 });
362
+ // 保留旧 coverage:重建期间索引仍在库中,写 0 会让 UI 误判为清空
363
+ this.sources.update(sourceId, { status: 'discovering' });
150
364
  this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
151
- // full 不再 clearSource:靠本轮 keepPaths 差量 prune,discover 失败不丢库
365
+ // full 不再 clearSource:靠 keepPaths prune,discover 失败不丢?
152
366
  if (source.kind === 'url' || source.kind === 'connector') {
153
367
  await this.ingestRemoteSource(source);
154
368
  return;
155
369
  }
156
- const files = await this.localFetcher.discover(source);
370
+ // watch 策略索引同时挂上 fs 监听(幂等;否则后续变更静默丢失?
371
+ this.startWatch(sourceId);
372
+ const localDiscovery = await this.localFetcher.discover(source);
373
+ const files = localDiscovery.docs;
157
374
  this.discoveredFiles.set(sourceId, files.length);
158
- // 差量 prune:磁盘上已消失的 path
159
- const localKeep = new Set(files.map((f) => f.path));
160
- this.index.pruneMissing(sourceId, localKeep);
375
+ // 后不 prune、不入队(半?keep 删)
376
+ if (this.isAborted(sourceId))
377
+ return;
378
+ // prune:仅 walk 完整时执行;出错/?keep 不完整会
379
+ if (localDiscovery.complete) {
380
+ const localKeep = new Set(files.map((f) => f.path));
381
+ // walk 成功且目录为空时允许清空 Membership(§6.2)
382
+ await this.index.pruneMissingAsync(sourceId, localKeep, { allowEmptyKeep: true });
383
+ }
161
384
  this.sources.update(sourceId, { status: 'partial', coverage: 0 });
162
385
  this.emitProgress({
163
386
  type: 'source',
@@ -166,10 +389,14 @@ export class KnowledgeIngest extends EventEmitter {
166
389
  detail: `${files.length} files`,
167
390
  });
168
391
  for (const file of files) {
169
- this.enqueue(sourceId, 'parse_file', file.path, 2, file.path);
392
+ if (this.isAborted(sourceId))
393
+ return;
394
+ if (this.isUnchangedIndexed(sourceId, file))
395
+ continue;
396
+ await this.enqueue(sourceId, 'parse_file', file.path, 2);
170
397
  }
171
- if (this.embedding) {
172
- this.enqueue(sourceId, 'embed_source', null, 3);
398
+ if (this.embedding && !this.isAborted(sourceId)) {
399
+ await this.enqueue(sourceId, 'embed_source', null, 3);
173
400
  }
174
401
  this.kick();
175
402
  }
@@ -177,6 +404,25 @@ export class KnowledgeIngest extends EventEmitter {
177
404
  this.sourceLocks.delete(sourceId);
178
405
  }
179
406
  }
407
+ /**
408
+ * 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
409
+ * 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
410
+ * indexing/error/半成品必须重做,避免把未完成当成新鲜。
411
+ */
412
+ isUnchangedIndexed(sourceId, ref) {
413
+ if (ref.size == null || ref.mtime == null)
414
+ return false;
415
+ const existing = this.index.getFile(sourceId, ref.path);
416
+ if (!existing)
417
+ return false;
418
+ if (existing.status !== 'indexed')
419
+ return false;
420
+ if (existing.chunkCount <= 0)
421
+ return false;
422
+ if (!existing.contentHash)
423
+ return false;
424
+ return existing.size === ref.size && existing.mtime === ref.mtime;
425
+ }
180
426
  /**
181
427
  * URL / connector 源:discover + fetch + 索引
182
428
  */
@@ -196,9 +442,9 @@ export class KnowledgeIngest extends EventEmitter {
196
442
  return;
197
443
  }
198
444
  const fetcher = source.kind === 'connector' ? this.connectorFetcher : this.urlFetcher;
199
- let refs;
445
+ let discovered;
200
446
  try {
201
- refs = await fetcher.discover(source, cred);
447
+ discovered = await fetcher.discover(source, cred);
202
448
  }
203
449
  catch (err) {
204
450
  const msg = err instanceof Error ? err.message : String(err);
@@ -209,7 +455,8 @@ export class KnowledgeIngest extends EventEmitter {
209
455
  this.emitProgress({ type: 'error', sourceId: source.id, status: 'discover_failed', detail: msg });
210
456
  return;
211
457
  }
212
- // 条件 GET:带上已有 etag/Last-Modified,304 则整文档跳过
458
+ const refs = discovered.docs;
459
+ // 条件 GET:带上已?etag/Last-Modified?04 则整文档跳过
213
460
  const enriched = refs.map((ref) => {
214
461
  const existing = this.index.getFile(source.id, ref.path);
215
462
  return {
@@ -229,15 +476,25 @@ export class KnowledgeIngest extends EventEmitter {
229
476
  let okCount = 0;
230
477
  let unchanged = 0;
231
478
  for (const ref of enriched) {
479
+ // 后立刻停:不再写索引
480
+ if (this.isAborted(source.id)) {
481
+ this.emitProgress({
482
+ type: 'source',
483
+ sourceId: source.id,
484
+ status: 'aborted',
485
+ detail: `stopped after ${okCount + unchanged}/${enriched.length}`,
486
+ });
487
+ return;
488
+ }
232
489
  try {
233
490
  const doc = await fetcher.fetch(source, ref, cred);
234
491
  if (!doc) {
235
- // 304 / 未变:保持原索引
492
+ // 304 / :保持原索引
236
493
  unchanged += 1;
237
494
  okCount += 1;
238
495
  continue;
239
496
  }
240
- if (this.indexVirtualDoc(source.id, doc)) {
497
+ if (await this.indexVirtualDoc(source.id, doc, source.authRef)) {
241
498
  okCount += 1;
242
499
  }
243
500
  }
@@ -252,22 +509,25 @@ export class KnowledgeIngest extends EventEmitter {
252
509
  });
253
510
  }
254
511
  }
255
- // 差量 prune:discover 成功后,源上已消失的文档才删
256
- const keep = new Set(enriched.map((r) => r.path));
257
- const pruned = this.index.pruneMissing(source.id, keep);
258
- if (pruned > 0) {
259
- this.emitProgress({
260
- type: 'source',
261
- sourceId: source.id,
262
- status: 'pruned',
263
- detail: `removed=${pruned}`,
264
- });
512
+ // prune:仅?discover **完整** 时才;截?walk 失败?keep 不完整,prune ?
513
+ // 后不?prune:半?keep 删已入库文档
514
+ if (!this.isAborted(source.id) && discovered.complete) {
515
+ const keep = new Set(enriched.map((r) => r.path));
516
+ const pruned = await this.index.pruneMissingAsync(source.id, keep, { allowEmptyKeep: true });
517
+ if (pruned > 0) {
518
+ this.emitProgress({
519
+ type: 'source',
520
+ sourceId: source.id,
521
+ status: 'pruned',
522
+ detail: `removed=${pruned}`,
523
+ });
524
+ }
265
525
  }
266
- if (this.embedding) {
267
- this.enqueue(source.id, 'embed_source', null, 3);
526
+ if (this.embedding && !this.isAborted(source.id)) {
527
+ await this.enqueue(source.id, 'embed_source', null, 3);
268
528
  this.kick();
269
529
  }
270
- this.refreshCoverage(source.id);
530
+ this.refreshCoverage(source.id, { force: true });
271
531
  this.emitProgress({
272
532
  type: 'source',
273
533
  sourceId: source.id,
@@ -276,10 +536,10 @@ export class KnowledgeIngest extends EventEmitter {
276
536
  });
277
537
  }
278
538
  /**
279
- * 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
539
+ * 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
280
540
  *
281
- * @returns 凭证或 null(仅当源未配置 authRef)
282
- * @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
541
+ * @returns ?null(仅当源?authRef?
542
+ * @throws authRef 配置了但无法解析时抛错,由调用方
283
543
  */
284
544
  async resolveCred(source) {
285
545
  if (!source.authRef)
@@ -294,16 +554,16 @@ export class KnowledgeIngest extends EventEmitter {
294
554
  return resolved;
295
555
  }
296
556
  /**
297
- * 将 VirtualDocument 写入索引(外源 / 规范化后文本)
557
+ * ?VirtualDocument 写入索引?/ 规范化后文本?
298
558
  *
299
- * @returns 是否成功写入
559
+ * @returns 成功写入
300
560
  */
301
- indexVirtualDoc(sourceId, doc) {
561
+ async indexVirtualDoc(sourceId, doc, authRef) {
302
562
  if (!doc.content?.trim()) {
303
563
  this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
304
564
  return false;
305
565
  }
306
- // 逻辑键可能无扩展名(connector id / URL path):按 MIME → 扩展名 → 内容形态回退
566
+ // 逻辑能无扩展名(connector id / URL path):按 MIME / 内容启发式
307
567
  let adapter = this.adapters.match(doc.path, doc.contentType);
308
568
  if (!adapter) {
309
569
  if (looksLikeHtml(doc.content, doc.contentType))
@@ -318,8 +578,20 @@ export class KnowledgeIngest extends EventEmitter {
318
578
  return true;
319
579
  }
320
580
  try {
321
- const chunks = adapter.chunk(doc.content, doc.path);
322
- this.index.upsertFile({
581
+ const chunks = await this.chunkTextInWorker(doc.content, doc.path, undefined, adapter.id);
582
+ let identityKey;
583
+ if (doc.externalUrl) {
584
+ identityKey = urlIdentityKey(doc.externalUrl, authRef);
585
+ }
586
+ else {
587
+ try {
588
+ identityKey = `path:${normalizePathLexical(doc.path)}`;
589
+ }
590
+ catch {
591
+ identityKey = `path:${doc.path.replace(/\\/g, '/')}`;
592
+ }
593
+ }
594
+ await this.index.upsertFile({
323
595
  sourceId,
324
596
  path: doc.path,
325
597
  contentHash,
@@ -327,6 +599,7 @@ export class KnowledgeIngest extends EventEmitter {
327
599
  mtime: Math.floor(Date.now() / 1000),
328
600
  adapterId: adapter.id,
329
601
  chunks,
602
+ identityKey,
330
603
  externalUrl: doc.externalUrl,
331
604
  etag: doc.etag,
332
605
  lastModified: doc.lastModified,
@@ -339,13 +612,129 @@ export class KnowledgeIngest extends EventEmitter {
339
612
  }
340
613
  }
341
614
  /**
342
- * 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
615
+ * 单文(P0 道;完成 parse,不 await 全库?
343
616
  */
344
617
  async ingestFileNow(sourceId, filePath) {
345
618
  const ok = await this.parseOne(sourceId, filePath);
346
619
  this.refreshCoverage(sourceId);
347
620
  return ok;
348
621
  }
622
+ /**
623
+ * 重做指定文件:强制重?解析 ?分块 ?向量
624
+ *
625
+ * - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
626
+ * - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
627
+ * - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
628
+ * - 等非文件:直接清,不入队
629
+ * - 完成后走既有 embed 链路
630
+ *
631
+ * @param sourceId - ?id
632
+ * @param paths - 文件列表
633
+ * @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
634
+ */
635
+ async reprocessFiles(sourceId, paths) {
636
+ if (!paths.length) {
637
+ return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: 0 };
638
+ }
639
+ const source = this.sources.get(sourceId);
640
+ if (!source) {
641
+ return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: paths.length };
642
+ }
643
+ let resumed = false;
644
+ if (this.isAborted(sourceId)) {
645
+ this.beginAbortEpoch(sourceId);
646
+ resumed = true;
647
+ }
648
+ let queued = 0;
649
+ let alreadyActive = 0;
650
+ let cleanedNonFiles = 0;
651
+ let rejected = 0;
652
+ const isRemote = source.kind === 'url' || source.kind === 'connector';
653
+ for (const p of paths) {
654
+ if (!this.sourceOwnsPath(source, p)) {
655
+ rejected += 1;
656
+ continue;
657
+ }
658
+ if (isRemote) {
659
+ // 外源:path 重做 = 重新 fetch + 索引(不读本地盘?
660
+ this.index.invalidateFileForReparse(sourceId, p);
661
+ if (await this.enqueue(sourceId, 'fetch_doc', p, 1)) {
662
+ queued += 1;
663
+ }
664
+ else if (this.hasActiveParseJob(sourceId, p)) {
665
+ alreadyActive += 1;
666
+ }
667
+ continue;
668
+ }
669
+ // /节点:清即可,没?
670
+ try {
671
+ const st = statSync(p);
672
+ if (!st.isFile()) {
673
+ this.index.removeFile(sourceId, p);
674
+ cleanedNonFiles += 1;
675
+ continue;
676
+ }
677
+ }
678
+ catch {
679
+ // 不存仍按文件流程 parse ?
680
+ }
681
+ // 总是失效 hash:即使已在队列,跑起来也不能 isFresh 跳过
682
+ this.index.invalidateFileForReparse(sourceId, p);
683
+ if (await this.enqueue(sourceId, 'parse_file', p, 1)) {
684
+ queued += 1;
685
+ }
686
+ else if (this.hasActiveParseJob(sourceId, p)) {
687
+ alreadyActive += 1;
688
+ }
689
+ }
690
+ this.kick();
691
+ return { queued, alreadyActive, cleanedNonFiles, resumed, rejected };
692
+ }
693
+ /** 上是否已?queued/running ?parse/fetch 任务 */
694
+ hasActiveParseJob(sourceId, path) {
695
+ return this.jobQueue.hasActivePathJob(sourceId, path);
696
+ }
697
+ /**
698
+ * 级任?文件UI 跟踪重做完成?
699
+ *
700
+ * @param sourceId - ?id
701
+ * @param paths - 列表
702
+ */
703
+ jobStateForPaths(sourceId, paths) {
704
+ return paths.map((p) => {
705
+ const jobs = this.sources.database.raw
706
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs
707
+ WHERE source_id = ? AND path = ? AND status IN ('queued','running')`)
708
+ .get(sourceId, p);
709
+ const f = this.index.getFile(sourceId, p);
710
+ let exists = false;
711
+ try {
712
+ exists = statSync(p).isFile();
713
+ }
714
+ catch {
715
+ exists = false;
716
+ }
717
+ return {
718
+ path: p,
719
+ jobsActive: Number(jobs?.n ?? 0),
720
+ fileStatus: f?.status ?? null,
721
+ chunkCount: f?.chunkCount ?? 0,
722
+ error: f?.error ?? null,
723
+ exists,
724
+ };
725
+ });
726
+ }
727
+ /**
728
+ * 按筛选批量重做(当前列表?status/ext/q 条件?
729
+ *
730
+ * @param sourceId - ?id
731
+ * @param opts - 与文件列表筛选一?
732
+ * @returns 入队条数
733
+ */
734
+ async reprocessByFilter(sourceId, opts) {
735
+ const paths = this.index.listFilePathsFiltered(sourceId, opts);
736
+ return this.reprocessFiles(sourceId, paths);
737
+ }
349
738
  /**
350
739
  * 处理 fs 变更(debounce 后入队)
351
740
  */
@@ -358,30 +747,33 @@ export class KnowledgeIngest extends EventEmitter {
358
747
  clearTimeout(prev);
359
748
  this.debounceTimers.set(key, setTimeout(() => {
360
749
  this.debounceTimers.delete(key);
361
- if (event === 'rename') {
362
- // inotify:create/move 也报 rename — 按存在性分支,勿一律 drop
363
- void stat(filePath)
364
- .then((st) => {
365
- if (st.isFile()) {
366
- this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
367
- }
368
- else {
369
- this.enqueue(sourceId, 'drop_file', filePath, 1, filePath);
370
- }
750
+ // 统一?stat:目?change 时不?parse ?no_adapter
751
+ void stat(filePath)
752
+ .then(async (st) => {
753
+ if (st.isFile()) {
754
+ await this.enqueue(sourceId, 'parse_file', filePath, 1);
755
+ }
756
+ else if (event === 'rename' || !st.isDirectory()) {
757
+ await this.enqueue(sourceId, 'drop_file', filePath, 1);
758
+ }
759
+ else {
760
+ this.index.removeFile(sourceId, filePath);
761
+ }
762
+ this.requestGapScan(sourceId);
763
+ this.kick();
764
+ })
765
+ .catch(() => {
766
+ void this.enqueue(sourceId, 'drop_file', filePath, 1)
767
+ .then(() => {
768
+ this.requestGapScan(sourceId);
371
769
  this.kick();
372
770
  })
373
- .catch(() => {
374
- this.enqueue(sourceId, 'drop_file', filePath, 1, filePath);
375
- this.kick();
376
- });
377
- return;
378
- }
379
- this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
380
- this.kick();
771
+ .catch(() => undefined);
772
+ });
381
773
  }, this.debounceMs));
382
774
  }
383
775
  /**
384
- * 启动目录 watch(幂等)
776
+ * watch(幂等)
385
777
  */
386
778
  startWatch(sourceId) {
387
779
  const source = this.sources.get(sourceId);
@@ -425,7 +817,7 @@ export class KnowledgeIngest extends EventEmitter {
425
817
  }
426
818
  }
427
819
  /**
428
- * 等待队列排空(测试/管理)
820
+ * 等待队列排空(测?管理?
429
821
  */
430
822
  async idle(timeoutMs = 30_000) {
431
823
  const start = Date.now();
@@ -441,81 +833,112 @@ export class KnowledgeIngest extends EventEmitter {
441
833
  this.disposed = true;
442
834
  this.stopPolling();
443
835
  this.stopReconciler();
836
+ if (this.gapScanTimer) {
837
+ clearTimeout(this.gapScanTimer);
838
+ this.gapScanTimer = null;
839
+ }
840
+ this.gapScanUrgent.clear();
444
841
  void this.abortJobs();
445
842
  for (const id of this.watchers.keys())
446
843
  this.stopWatch(id);
447
844
  }
448
845
  /**
449
- * 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
846
+ * 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
450
847
  *
451
- * @param opts.sourceId - 仅中止该源;省略则全部源
452
- * @returns 取消统计
848
+ * @param opts.sourceId - 仅中源;省略则全部源
849
+ * @returns 取消
453
850
  */
454
851
  abortJobs(opts) {
455
852
  const sid = opts?.sourceId?.trim();
456
- const ids = sid ? [sid] : [...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)];
853
+ const ids = sid
854
+ ? [sid]
855
+ : [
856
+ ...this.jobControl.listAbortedSourceIds(),
857
+ ...this.sources.list().map((s) => s.id),
858
+ ];
457
859
  const unique = [...new Set(ids)];
458
860
  let cancelledQueued = 0;
459
861
  let abortedRunning = 0;
460
- // 1) 清空 queued → cancelled
461
862
  for (const id of unique) {
863
+ // 1) DB 落中跨重+ 打断程信?
864
+ const newly = this.jobControl.markAborted(id);
865
+ if (newly)
866
+ abortedRunning += 1;
867
+ // 2) 清空 queued ?cancelled(共?File ?parse 不杀:仍领则保留?
462
868
  const res = this.sources.database.raw
463
869
  .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
464
- WHERE source_id = ? AND status = 'queued'`)
465
- .run(Date.now(), id);
870
+ WHERE source_id = ? AND status = 'queued'
871
+ AND (
872
+ file_id IS NULL
873
+ OR kind IN ('walk_source', 'embed_source')
874
+ OR NOT EXISTS (
875
+ SELECT 1 FROM knowledge_memberships m
876
+ WHERE m.file_id = knowledge_jobs.file_id AND m.source_id != ?
877
+ )
878
+ )`)
879
+ .run(Date.now(), id, id);
466
880
  cancelledQueued += Number(res.changes ?? 0);
467
- // 2) 打断在跑(worker terminate / embed 循环退出);保持 aborted 直到下次 reindex
468
- let ctrl = this.abortBySource.get(id);
469
- if (!ctrl) {
470
- ctrl = new AbortController();
471
- this.abortBySource.set(id, ctrl);
472
- }
473
- if (!ctrl.signal.aborted) {
474
- ctrl.abort();
475
- abortedRunning += 1;
476
- }
477
881
  }
478
- // 3) 看门狗不要再把 aborted running 收回 queued
479
- this.sources.database.raw
480
- .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
481
- WHERE status = 'running' AND last_error = 'aborted'`)
482
- .run(Date.now());
882
+ // 3) 看门狗不要再?aborted running 收回 queued(限定本次涉及的源)
883
+ if (sid) {
884
+ this.sources.database.raw
885
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
886
+ WHERE source_id = ? AND status = 'running' AND last_error = 'aborted'`)
887
+ .run(Date.now(), sid);
888
+ }
889
+ else {
890
+ this.sources.database.raw
891
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
892
+ WHERE status = 'running' AND last_error = 'aborted'`)
893
+ .run(Date.now());
894
+ }
895
+ const runningRow = this.sources.database.raw
896
+ .prepare(sid
897
+ ? `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE source_id = ? AND status = 'running'`
898
+ : `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'running'`)
899
+ .get(...(sid ? [sid] : []));
900
+ const runningJobs = Number(runningRow?.n ?? 0);
483
901
  this.emitProgress({
484
902
  type: 'source',
485
903
  sourceId: sid ?? '*',
486
904
  status: 'aborted',
487
- detail: `cancelledQueued=${cancelledQueued}`,
905
+ detail: `cancelledQueued=${cancelledQueued} runningTail=${runningJobs}`,
488
906
  });
489
- return { cancelledQueued, abortedRunning };
907
+ return { cancelledQueued, abortedRunning, runningJobs };
490
908
  }
491
909
  /**
492
- * 恢复/继续索引(相对「重建」更轻)
910
+ * /继续索引(相轻)
493
911
  *
494
- * - 清除该源中止态
495
- * - 把 cancelled 任务放回 queued
912
+ * - 清除该源?
913
+ * - ?cancelled 任务放回 queued
496
914
  * - 若仍缺向量则排队 embed
497
- * - 不重新 walk 全量;需要全量对齐请 reindex
915
+ * - 不重?walk 全量;需要全 reindex
498
916
  *
499
- * @param opts.sourceId - 仅恢复该源;省略则全部源
500
- * @returns 恢复统计
917
+ * @param opts.sourceId - 仅恢源;省略则全部源
918
+ * @returns
501
919
  */
502
920
  resumeJobs(opts) {
503
921
  const sid = opts?.sourceId?.trim();
504
922
  const ids = sid
505
923
  ? [sid]
506
- : [...new Set([...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)])];
924
+ : [
925
+ ...new Set([
926
+ ...this.jobControl.listAbortedSourceIds(),
927
+ ...this.sources.list().map((s) => s.id),
928
+ ]),
929
+ ];
507
930
  let restoredCancelled = 0;
508
931
  let embedQueued = 0;
509
932
  for (const id of ids) {
510
- // 1) 清除中止纪元
933
+ // 1) 清除态(DB + 新纪元信号)
511
934
  this.beginAbortEpoch(id);
512
- // 2) cancelled → queued(用户中止时被杀的活)
935
+ // 2) cancelled ?queued(用户中止时的活?
513
936
  const res = this.sources.database.raw
514
- .prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
515
- WHERE source_id = ? AND status = 'cancelled'`)
937
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', last_error = NULL, updated_at = ?
938
+ WHERE source_id = ? AND status = 'cancelled' AND (last_error IS NULL OR last_error = 'aborted')`)
516
939
  .run(Date.now(), id);
517
940
  restoredCancelled += Number(res.changes ?? 0);
518
- // 3) 仍缺向量则确保 embed 在队
941
+ // 3) 仍缺向量则确?embed 在队
519
942
  if (this.embedding) {
520
943
  const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
521
944
  const active = this.countActiveJobs(id);
@@ -534,31 +957,24 @@ export class KnowledgeIngest extends EventEmitter {
534
957
  });
535
958
  return { restoredCancelled, embedQueued };
536
959
  }
537
- /** 开启新的中止纪元(abort 后新任务不受旧信号影响) */
960
+ /** 的未(resume / supersede 后用;清 DB 态) */
538
961
  beginAbortEpoch(sourceId) {
539
- this.abortBySource.set(sourceId, new AbortController());
962
+ this.jobControl.beginEpoch(sourceId);
540
963
  }
541
964
  /**
542
- * 重建前作废本源未完成任务(supersede)
965
+ * 重建前作废本源未完成任务(supersede?
543
966
  *
544
- * - queued → cancelled(superseded_by_reindex)
545
- * - running 打断(worker/循环退出)
546
- * - 随后 beginAbortEpoch,本轮重建可继续写
967
+ * - queued ?cancelled(superseded_by_reindex?
968
+ * - running 打断(worker/出)
969
+ * - 随后 beginAbortEpoch,本建可继续?
547
970
  */
548
971
  supersedeSourceWork(sourceId) {
549
972
  this.sources.database.raw
550
973
  .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
551
974
  WHERE source_id = ? AND status = 'queued'`)
552
975
  .run(Date.now(), sourceId);
553
- // 打断在跑 parse/embed;再开新纪元,避免 walk 自己被 abort 卡死
554
- let ctrl = this.abortBySource.get(sourceId);
555
- if (!ctrl) {
556
- ctrl = new AbortController();
557
- this.abortBySource.set(sourceId, ctrl);
558
- }
559
- if (!ctrl.signal.aborted) {
560
- ctrl.abort();
561
- }
976
+ // 打断在跑 parse/embed;再新纪元(清中,避?walk ?abort
977
+ this.jobControl.markAborted(sourceId);
562
978
  this.beginAbortEpoch(sourceId);
563
979
  this.emitProgress({
564
980
  type: 'source',
@@ -575,8 +991,10 @@ export class KnowledgeIngest extends EventEmitter {
575
991
  return 0;
576
992
  if (this.isAborted(sourceId))
577
993
  return 0;
578
- // 已有 parse 积压则不重复扫
579
- if (this.countQueuedKinds(['parse_file', 'walk_source']) > 0)
994
+ // 仅跳过「全量 walk 在途」:individual parse_file 排队 **不得** 挡住缺口补扫。
995
+ // 否则 watch 漏事件后,当前批次 parse 会一直占着门闩,排空才一次入队数百文件(84 → 726)。
996
+ // 不用 sourceLocks:与 reconcile 同轮 fire-and-forget ingestSource 竞态,会误伤补扫。
997
+ if (this.countQueuedKinds(['walk_source'], sourceId) > 0)
580
998
  return 0;
581
999
  let found;
582
1000
  try {
@@ -585,12 +1003,28 @@ export class KnowledgeIngest extends EventEmitter {
585
1003
  catch {
586
1004
  return 0;
587
1005
  }
588
- const indexed = new Set(this.index.listFiles(sourceId).map((f) => f.path));
1006
+ const byPath = new Map(this.index.listFiles(sourceId).map((f) => [f.path.replace(/\\/g, '/'), f]));
589
1007
  let added = 0;
590
- for (const f of found) {
591
- if (!indexed.has(f.path)) {
592
- this.enqueue(sourceId, 'parse_file', f.path, 2, f.path);
1008
+ let retriedSkipped = 0;
1009
+ let retriedIncomplete = 0;
1010
+ for (const f of found.docs) {
1011
+ const logical = f.path.replace(/\\/g, '/');
1012
+ const existing = byPath.get(logical);
1013
+ // 缺文件必补;indexing=半写入/崩溃残留必补;skipped 且原因可重试时补
1014
+ const need = !existing ||
1015
+ existing.status === 'indexing' ||
1016
+ (existing.status === 'skipped' &&
1017
+ isRetryableSkipReason(existing.error ?? null, existing.size, this.fileLimits));
1018
+ if (!need)
1019
+ continue;
1020
+ if (await this.enqueue(sourceId, 'parse_file', f.path, 2)) {
593
1021
  added += 1;
1022
+ if (!existing)
1023
+ continue;
1024
+ if (existing.status === 'indexing')
1025
+ retriedIncomplete += 1;
1026
+ else
1027
+ retriedSkipped += 1;
594
1028
  }
595
1029
  }
596
1030
  if (added > 0) {
@@ -598,19 +1032,37 @@ export class KnowledgeIngest extends EventEmitter {
598
1032
  type: 'source',
599
1033
  sourceId,
600
1034
  status: 'parse_gap_found',
601
- detail: `missing_files=${added}`,
1035
+ detail: `missing_files=${added - retriedSkipped - retriedIncomplete} retry_skipped=${retriedSkipped} retry_indexing=${retriedIncomplete}`,
602
1036
  });
603
1037
  this.kick();
604
1038
  }
605
1039
  return added;
606
1040
  }
1041
+ /** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
1042
+ requestGapScan(sourceId, delayMs = 3_000) {
1043
+ if (this.disposed || this.isAborted(sourceId))
1044
+ return;
1045
+ this.gapScanUrgent.add(sourceId);
1046
+ if (this.gapScanTimer)
1047
+ return;
1048
+ this.gapScanTimer = setTimeout(() => {
1049
+ this.gapScanTimer = null;
1050
+ const ids = [...this.gapScanUrgent];
1051
+ this.gapScanUrgent.clear();
1052
+ for (const id of ids) {
1053
+ this.lastParseGapScanAt.set(id, Date.now());
1054
+ void this.ensureParseCoverage(id).catch(() => undefined);
1055
+ }
1056
+ }, delayMs);
1057
+ this.gapScanTimer.unref?.();
1058
+ }
607
1059
  /**
608
- * 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
1060
+ * 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
609
1061
  *
610
- * 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
611
- * embed 前调用,避免给已移出文件的残 chunks 上向量。
1062
+ * 源用?discover walk ?keep 集(;失回并?stat?
1063
+ * embed 前调避免给已移出文件的残 chunks 上向?
612
1064
  *
613
- * @param sourceId - 知识源 id
1065
+ * @param sourceId - 知识?id
614
1066
  * @returns 删除的文件数
615
1067
  */
616
1068
  async pruneMissingOnDisk(sourceId) {
@@ -621,9 +1073,15 @@ export class KnowledgeIngest extends EventEmitter {
621
1073
  if (source.kind === 'directory' || source.kind === 'workspace') {
622
1074
  try {
623
1075
  const found = await this.localFetcher.discover(source);
624
- const keep = new Set(found.map((f) => f.path));
625
- // walk 成功:即使空目录也可清空(与逐文件 stat 语义一致)
626
- removed = this.index.pruneMissing(sourceId, keep, { allowEmptyKeep: true });
1076
+ if (!found.complete) {
1077
+ // walk 不完整:?stat止用残缺 keep 清库
1078
+ removed = await this.pruneByStatScan(sourceId);
1079
+ }
1080
+ else {
1081
+ const keep = new Set(found.docs.map((f) => f.path));
1082
+ // walk 完整:即使空也可清空(与逐文?stat 致)
1083
+ removed = await this.index.pruneMissingAsync(sourceId, keep, { allowEmptyKeep: true });
1084
+ }
627
1085
  }
628
1086
  catch {
629
1087
  removed = await this.pruneByStatScan(sourceId);
@@ -639,12 +1097,12 @@ export class KnowledgeIngest extends EventEmitter {
639
1097
  status: 'pruned_missing',
640
1098
  detail: `removed_files=${removed}`,
641
1099
  });
642
- this.refreshCoverage(sourceId);
1100
+ this.refreshCoverage(sourceId, { force: true });
643
1101
  }
644
1102
  return removed;
645
1103
  }
646
- /** 逐文件 stat(file 源 / walk 失败回退);并行限流 */
647
- async pruneByStatScan(sourceId, concurrency = 32) {
1104
+ /** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
1105
+ async pruneByStatScan(sourceId, concurrency = 4) {
648
1106
  const files = this.index.listFiles(sourceId);
649
1107
  let removed = 0;
650
1108
  let i = 0;
@@ -665,7 +1123,7 @@ export class KnowledgeIngest extends EventEmitter {
665
1123
  exists = false;
666
1124
  }
667
1125
  if (!exists) {
668
- this.index.removePathTree(sourceId, f.path);
1126
+ await this.index.removeFileAsync(sourceId, f.path);
669
1127
  removed += 1;
670
1128
  }
671
1129
  }
@@ -674,23 +1132,29 @@ export class KnowledgeIngest extends EventEmitter {
674
1132
  return removed;
675
1133
  }
676
1134
  abortSignalFor(sourceId) {
677
- let ac = this.abortBySource.get(sourceId);
678
- if (!ac) {
679
- ac = new AbortController();
680
- this.abortBySource.set(sourceId, ac);
681
- }
682
- return ac.signal;
1135
+ return this.jobControl.signalFor(sourceId);
683
1136
  }
684
1137
  isAborted(sourceId) {
685
- return this.abortSignalFor(sourceId).aborted;
1138
+ return this.jobControl.isAborted(sourceId);
686
1139
  }
687
1140
  /**
688
- * 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
1141
+ * /继续 控制数(互斥按钮
689
1142
  *
690
- * 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
691
- * 1. 回收超时 running(崩溃/挂死孤儿)
692
- * 2. 有缺向量且无 active embed → 自动排队
693
- * 3. 无队列、无缺口 → 刷 coverage/status
1143
+ * @param sourceId - ?id
1144
+ * @returns 队列计数 + 已中?+ 仍缺向量
1145
+ */
1146
+ jobControlState(sourceId) {
1147
+ return readJobControlState(this.sources.database, this.index, sourceId, {
1148
+ embeddingEnabled: Boolean(this.embedding),
1149
+ });
1150
+ }
1151
+ /**
1152
+ * 任务对账(看门狗?不依赖单?ensureEmbedJob
1153
+ *
1154
+ * 解决「续去重吞掉 / 进程 / partial」等静默停摆?
1155
+ * 1. 回收超时 running(崩?孤儿?
1156
+ * 2. 有缺向量且无 active embed ?排队
1157
+ * 3. 无队缺口 ??coverage/status
694
1158
  */
695
1159
  startReconciler(intervalMs = 15_000) {
696
1160
  if (this.reconcileTimer || this.disposed)
@@ -713,38 +1177,38 @@ export class KnowledgeIngest extends EventEmitter {
713
1177
  this.reconcileTimer = null;
714
1178
  }
715
1179
  }
716
- /** 对账入口(也可手工调用) */
1180
+ /** 对账入口(也工调 */
717
1181
  async reconcileJobs() {
718
1182
  if (this.disposed)
719
1183
  return;
720
- this.reclaimStaleRunningJobs(5 * 60_000);
721
- if (!this.embedding) {
722
- for (const s of this.sources.list()) {
723
- if (s.status === 'removed' || s.status === 'disabled')
724
- continue;
725
- this.refreshCoverage(s.id);
726
- }
727
- return;
728
- }
1184
+ // stale ?**大于** 时,否则合法的长 parse 收回后双?
1185
+ this.reclaimStaleRunningJobs(this.fileLimits.maxParseTimeoutMs + 60_000);
1186
+ // path: 与 win:/unix: 身份分裂的零认领残留
1187
+ this.index.purgeOrphanPathIdentityFiles();
1188
+ // 等历入项(轻量;非每
1189
+ this.cleanupNonFileIndexRows();
1190
+ // ?job 定期清理,防表无?
1191
+ this.cleanupTerminalJobs();
1192
+ // 存量 FTS 后台补齐(分出事件循 rebuild?
1193
+ void this.index.ensureFtsBackfill();
1194
+ // 存量向量回填:小批量 + 让出事件?2000 条同步写(会 abort?
1195
+ await this.index.backfillVecFromEmbeddings(25);
729
1196
  for (const s of this.sources.list()) {
730
1197
  if (s.status === 'removed' || s.status === 'disabled')
731
1198
  continue;
732
1199
  if (this.isAborted(s.id))
733
1200
  continue;
734
- // watch 漏事件时补齐 parse:磁盘有、索引无 → 入队(解析优先于 embed)
735
- const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
736
- if (Date.now() - lastGap > 60_000) {
737
- this.lastParseGapScanAt.set(s.id, Date.now());
738
- await this.ensureParseCoverage(s.id);
739
- }
740
- const missing = this.index.listChunksMissingEmbedding(s.id, 1).length > 0;
741
1201
  const active = this.countActiveJobs(s.id);
742
- if (missing && active.embed === 0) {
743
- this.ensureEmbedJob(s.id);
1202
+ // discovering 卡死(注册 kick 失败 / file 源 walk 中断):无任务则 re-kick,不 supersede
1203
+ if (active.total === 0 &&
1204
+ (s.status === 'pending' || s.status === 'discovering') &&
1205
+ (s.kind === 'file' || s.kind === 'directory' || s.kind === 'workspace')) {
1206
+ await this.ingestSource(s.id, { fromQueue: true }).catch(() => undefined);
744
1207
  }
745
- else if (!missing && active.total === 0) {
746
- this.refreshCoverage(s.id);
747
- // 稳定态收尾(auto-describe 等);回调不得抛出打断对账
1208
+ // 稳定态收尾(auto-describe 等)关键词部署同触发
1209
+ // 先看 active,再 gap ;避免刚入队?gap job 挡住 settled
1210
+ if (active.total === 0) {
1211
+ this.refreshCoverage(s.id, { force: true });
748
1212
  try {
749
1213
  this.onSourceSettled?.(s.id);
750
1214
  }
@@ -752,34 +1216,52 @@ export class KnowledgeIngest extends EventEmitter {
752
1216
  /* ignore settle hook errors */
753
1217
  }
754
1218
  }
1219
+ else {
1220
+ this.flushCoverageDirty(s.id);
1221
+ }
1222
+ // watch 漏事件 / skip 重试时补齐 parse(与配置 embedding 无关)
1223
+ // urgent(watch 突发)不走 60s 节流;否则批量拷贝后要等一整轮才补
1224
+ const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
1225
+ const gapDue = this.gapScanUrgent.has(s.id) || Date.now() - lastGap > 60_000;
1226
+ if (gapDue) {
1227
+ this.lastParseGapScanAt.set(s.id, Date.now());
1228
+ this.gapScanUrgent.delete(s.id);
1229
+ await this.ensureParseCoverage(s.id);
1230
+ // drop_file 在途时勿再 prune:双路径会同时删同一棵子树,放大写压力
1231
+ const dropsInFlight = this.countQueuedKinds(['drop_file'], s.id) > 0 || this.hasRunningKind(s.id, 'drop_file');
1232
+ if (!dropsInFlight) {
1233
+ await this.pruneMissingOnDisk(s.id);
1234
+ }
1235
+ }
1236
+ const missingEmbed = this.embedding
1237
+ ? this.index.hasChunksMissingEmbedding(s.id)
1238
+ : false;
1239
+ if (this.embedding && missingEmbed && active.embed === 0) {
1240
+ this.ensureEmbedJob(s.id);
1241
+ }
755
1242
  }
756
1243
  this.kick();
757
1244
  }
758
- /** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
1245
+ /** ?job 清理(done/failed/cancelled 超过保留期) */
1246
+ cleanupTerminalJobs(retainMs = 24 * 60 * 60_000) {
1247
+ this.jobQueue.cleanupTerminal(retainMs);
1248
+ }
1249
+ /** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
759
1250
  reclaimStaleRunningJobs(staleMs) {
760
- const cutoff = Date.now() - staleMs;
761
- // 先放弃重试过多的
762
- this.sources.database.raw
763
- .prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
764
- WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
765
- .run(Date.now(), cutoff);
766
- this.sources.database.raw
767
- .prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
768
- WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
769
- .run(Date.now(), cutoff);
1251
+ this.jobQueue.reclaimStaleRunning(staleMs);
770
1252
  }
771
1253
  countActiveJobs(sourceId) {
1254
+ return this.jobQueue.countActiveJobs(sourceId);
1255
+ }
1256
+ hasRunningKind(sourceId, kind) {
772
1257
  const row = this.sources.database.raw
773
- .prepare(`SELECT
774
- COUNT(*) AS total,
775
- SUM(CASE WHEN kind = 'embed_source' THEN 1 ELSE 0 END) AS embed
776
- FROM knowledge_jobs
777
- WHERE source_id = ? AND status IN ('queued', 'running')`)
778
- .get(sourceId);
779
- return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
1258
+ .prepare(`SELECT 1 AS ok FROM knowledge_jobs
1259
+ WHERE source_id = ? AND kind = ? AND status = 'running' LIMIT 1`)
1260
+ .get(sourceId, kind);
1261
+ return Boolean(row?.ok);
780
1262
  }
781
1263
  /**
782
- * 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
1264
+ * poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
783
1265
  */
784
1266
  startPolling() {
785
1267
  if (this.pollTimer || this.disposed)
@@ -794,7 +1276,7 @@ export class KnowledgeIngest extends EventEmitter {
794
1276
  });
795
1277
  });
796
1278
  }, this.pollTickMs);
797
- // 不阻止进程退出
1279
+ // 不阻止进?
798
1280
  this.pollTimer.unref?.();
799
1281
  }
800
1282
  stopPolling() {
@@ -804,9 +1286,9 @@ export class KnowledgeIngest extends EventEmitter {
804
1286
  }
805
1287
  }
806
1288
  /**
807
- * 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
1289
+ * 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
808
1290
  *
809
- * @returns 本轮实际触发的 sourceId 列表
1291
+ * @returns 实际触发?sourceId 列表
810
1292
  */
811
1293
  async pollDueSources() {
812
1294
  if (this.disposed)
@@ -836,8 +1318,8 @@ export class KnowledgeIngest extends EventEmitter {
836
1318
  status: 'poll_due',
837
1319
  detail: source.sync.intervalMs?.toString(),
838
1320
  });
839
- // 异步入队,不阻塞 tick
840
- void this.ingestSource(source.id).catch((err) => {
1321
+ // 入队,不 tick量:?active 则跳过, supersede 打断
1322
+ void this.ingestSource(source.id, { incremental: true }).catch((err) => {
841
1323
  this.emitProgress({
842
1324
  type: 'error',
843
1325
  sourceId: source.id,
@@ -848,61 +1330,78 @@ export class KnowledgeIngest extends EventEmitter {
848
1330
  }
849
1331
  return triggered;
850
1332
  }
851
- // ── 内部 ──
1333
+ // 内部
852
1334
  hasQueuedJobs() {
853
- const row = this.sources.database.raw
854
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
855
- .get();
856
- return (row?.n ?? 0) > 0;
1335
+ return this.jobQueue.hasQueuedJobs();
857
1336
  }
858
- enqueue(sourceId, kind, path, priority, _detail) {
859
- if (this.isAborted(sourceId))
860
- return;
861
- // 同 source+kind 去重(queued + running,避免并发重复 embed)
862
- const activeStatuses = ['queued', 'running'];
863
- if (path != null) {
864
- const placeholders = activeStatuses.map(() => '?').join(',');
865
- const dup = this.sources.database.raw
866
- .prepare(`SELECT id FROM knowledge_jobs
867
- WHERE source_id = ? AND kind = ? AND path = ? AND status IN (${placeholders})`)
868
- .get(sourceId, kind, path, ...activeStatuses);
869
- if (dup?.id)
870
- return;
871
- }
872
- else {
873
- const placeholders = activeStatuses.map(() => '?').join(',');
874
- const dup = this.sources.database.raw
1337
+ /**
1338
+ * 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
1339
+ * 需要计数的路径请 `await enqueueStrict`。
1340
+ */
1341
+ enqueue(sourceId, kind, path, priority, fileIdHint) {
1342
+ return this.enqueueStrict(sourceId, kind, path, priority, fileIdHint).catch(() => false);
1343
+ }
1344
+ /** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
1345
+ *
1346
+ * 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
1347
+ */
1348
+ async enqueueStrict(sourceId, kind, path, priority, fileIdHint) {
1349
+ if (this.disposed)
1350
+ return false;
1351
+ const fileId = fileIdHint ?? (await this.resolveJobFileId(sourceId, kind, path));
1352
+ if (fileId && (kind === 'parse_file' || kind === 'fetch_doc')) {
1353
+ // 共享 File 已有在途 parse:补认领后跳过,避免双写
1354
+ const busy = this.sources.database.raw
875
1355
  .prepare(`SELECT id FROM knowledge_jobs
876
- WHERE source_id = ? AND kind = ? AND path IS NULL AND status IN (${placeholders})`)
877
- .get(sourceId, kind, ...activeStatuses);
878
- if (dup?.id)
879
- return;
880
- }
881
- const now = Date.now();
882
- const depth = this.sources.database.raw
883
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
884
- .get();
885
- if ((depth?.n ?? 0) >= this.maxQueueDepth) {
886
- this.emitProgress({
887
- type: 'error',
888
- sourceId,
889
- status: 'queue_backpressure',
890
- detail: `queued=${depth.n} (delay only, jobs kept)`,
891
- });
1356
+ WHERE file_id = ? AND kind IN ('parse_file','fetch_doc') AND status IN ('queued','running')
1357
+ LIMIT 1`)
1358
+ .get(fileId);
1359
+ if (busy?.id && path) {
1360
+ this.index.attachMembership(sourceId, fileId, path);
1361
+ return false;
1362
+ }
892
1363
  }
893
- if (this.diskWatermarkAlert) {
1364
+ const ok = this.jobQueue.enqueue(sourceId, kind, path, priority, fileId);
1365
+ if (ok && this.diskWatermarkAlert) {
894
1366
  void this.warnDiskWatermark(sourceId);
895
1367
  }
1368
+ return ok;
1369
+ }
1370
+ /** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
1371
+ bindJobFileId(job) {
1372
+ if (job.file_id || !job.path)
1373
+ return;
1374
+ const rec = this.index.getFile(job.source_id, job.path);
1375
+ if (!rec?.id)
1376
+ return;
896
1377
  this.sources.database.raw
897
- .prepare(`INSERT INTO knowledge_jobs (id, source_id, kind, path, priority, status, attempts, created_at, updated_at)
898
- VALUES (?, ?, ?, ?, ?, 'queued', 0, ?, ?)`)
899
- .run(`kj_${randomUUID().slice(0, 12)}`, sourceId, kind, path, priority, now, now);
1378
+ .prepare(`UPDATE knowledge_jobs SET file_id = ? WHERE id = ? AND file_id IS NULL`)
1379
+ .run(rec.id, job.id);
1380
+ job.file_id = rec.id;
1381
+ }
1382
+ /** job 绑定 file_id:membership → identity;本地文件可 identify */
1383
+ async resolveJobFileId(sourceId, kind, path) {
1384
+ if (!path)
1385
+ return null;
1386
+ if (kind !== 'parse_file' && kind !== 'fetch_doc' && kind !== 'drop_file')
1387
+ return null;
1388
+ const logical = path.replace(/\\/g, '/');
1389
+ const existing = this.index.getFile(sourceId, logical);
1390
+ if (existing?.id)
1391
+ return existing.id;
1392
+ try {
1393
+ const ident = await identifyLocalFile(path);
1394
+ return this.index.findFileIdByIdentity(ident.key);
1395
+ }
1396
+ catch {
1397
+ return null;
1398
+ }
900
1399
  }
901
1400
  kick() {
902
1401
  if (this.disposed)
903
1402
  return;
904
1403
  if (this.draining) {
905
- // 等待中的 drain 会再 kick;标记一次避免丢信号
1404
+ // 等待 drain 会再 kick;标记一次避免丢信号
906
1405
  this.kickAgain = true;
907
1406
  return;
908
1407
  }
@@ -917,39 +1416,40 @@ export class KnowledgeIngest extends EventEmitter {
917
1416
  });
918
1417
  }
919
1418
  drain() {
1419
+ if (this.disposed)
1420
+ return;
920
1421
  for (;;) {
1422
+ if (this.disposed)
1423
+ return;
921
1424
  const allowParse = this.runningParse < this.parseConcurrency;
922
1425
  const allowEmbed = this.runningEmbed < this.embedConcurrency;
923
1426
  if (!allowParse && !allowEmbed)
924
1427
  return;
925
- const parsePending = this.countQueuedKinds([
926
- 'parse_file',
927
- 'walk_source',
928
- 'drop_file',
929
- 'fetch_doc',
930
- ]);
931
1428
  const kinds = [];
932
1429
  if (allowParse)
933
1430
  kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
934
- // 业务优先级:解析/分块做完即可关键词搜;embedding 仅在无 parse 积压时占槽
935
- if (allowEmbed && parsePending === 0)
1431
+ // parse 优先(priority 2 < 3),但 **不得** 等全部 parse 结束才 embed:
1432
+ // 大库 parse 可能跑数小时,否则向量永远 0。embed 与 parse 并行,仅受 embed 槽限制。
1433
+ if (allowEmbed)
936
1434
  kinds.push('embed_source');
937
1435
  if (kinds.length === 0) {
938
- if (!allowParse && allowEmbed && parsePending === 0)
939
- return;
940
- // parse 槽满但仍有 parse 积压 → 等 parse,不抢 embed
941
- if (parsePending > 0 && !allowParse)
942
- return;
943
1436
  return;
944
1437
  }
945
- const job = this.claimJob(kinds);
946
- if (!job)
1438
+ const job = this.claimJob(kinds, (j) => this.acceptParseSlot(j));
1439
+ if (!job) {
1440
+ // 队列清空:合并刷 parse/drop 标脏?coverage,避免每文件?COUNT
1441
+ this.flushCoverageDirty();
947
1442
  return;
1443
+ }
948
1444
  const isEmbed = job.kind === 'embed_source';
1445
+ const isDocParse = this.isDocParseJob(job);
949
1446
  if (isEmbed)
950
1447
  this.runningEmbed += 1;
951
- else
1448
+ else {
952
1449
  this.runningParse += 1;
1450
+ if (isDocParse)
1451
+ this.runningDocParse += 1;
1452
+ }
953
1453
  void this.runJob(job)
954
1454
  .catch((err) => {
955
1455
  this.emitProgress({
@@ -963,40 +1463,32 @@ export class KnowledgeIngest extends EventEmitter {
963
1463
  .finally(() => {
964
1464
  if (isEmbed)
965
1465
  this.runningEmbed -= 1;
966
- else
1466
+ else {
967
1467
  this.runningParse -= 1;
1468
+ if (isDocParse)
1469
+ this.runningDocParse -= 1;
1470
+ }
968
1471
  this.kick();
969
1472
  });
970
1473
  }
971
1474
  }
972
- countQueuedKinds(kinds) {
973
- if (kinds.length === 0)
1475
+ /** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
1476
+ isDocParseJob(job) {
1477
+ return job.kind === 'parse_file' && !!job.path && isDocumentPath(job.path);
1478
+ }
1479
+ acceptParseSlot(job) {
1480
+ if (this.isDocParseJob(job)) {
1481
+ return this.runningDocParse < this.documentParseConcurrency;
1482
+ }
1483
+ return true;
1484
+ }
1485
+ countQueuedKinds(kinds, sourceId) {
1486
+ if (this.disposed)
974
1487
  return 0;
975
- const placeholders = kinds.map(() => '?').join(',');
976
- const row = this.sources.database.raw
977
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`)
978
- .get(...kinds);
979
- return row?.n ?? 0;
1488
+ return this.jobQueue.countQueuedKinds(kinds, sourceId);
980
1489
  }
981
- claimJob(kinds) {
982
- if (kinds.length === 0)
983
- return null;
984
- const placeholders = kinds.map(() => '?').join(',');
985
- const row = this.sources.database.raw
986
- .prepare(`SELECT * FROM knowledge_jobs
987
- WHERE status = 'queued' AND kind IN (${placeholders})
988
- ORDER BY priority ASC, created_at ASC
989
- LIMIT 1`)
990
- .get(...kinds);
991
- if (!row)
992
- return null;
993
- const res = this.sources.database.raw
994
- .prepare(`UPDATE knowledge_jobs SET status = 'running', updated_at = ?
995
- WHERE id = ? AND status = 'queued'`)
996
- .run(Date.now(), row.id);
997
- if (Number(res.changes ?? 0) === 0)
998
- return null;
999
- return row;
1490
+ claimJob(kinds, accept) {
1491
+ return this.jobQueue.claimJob(kinds, accept);
1000
1492
  }
1001
1493
  async runJob(job) {
1002
1494
  const now = Date.now();
@@ -1008,44 +1500,77 @@ export class KnowledgeIngest extends EventEmitter {
1008
1500
  return;
1009
1501
  }
1010
1502
  try {
1503
+ // 归属闸门:任何会碰本地盘 / 外源逻辑 path 任务都必须属于本?
1504
+ if (job.path && (job.kind === 'parse_file' || job.kind === 'fetch_doc')) {
1505
+ const owner = this.sources.get(job.source_id);
1506
+ if (owner && !this.sourceOwnsPath(owner, job.path)) {
1507
+ throw new Error(`path not owned by source: ${job.path}`);
1508
+ }
1509
+ }
1011
1510
  if (job.kind === 'parse_file' && job.path) {
1012
- // 单文件解析硬超时:大 xlsx/pdf 同步抽取会堵死事件循环
1013
- await this.withTimeout(this.parseOne(job.source_id, job.path), this.parseTimeoutMs, `parse_timeout:${job.path}`);
1511
+ // 单文件解析硬超时:大 xlsx/pdf 抽取会堵死事件循超时随体?
1512
+ let parseTimeoutMs = this.parseTimeoutMs;
1513
+ try {
1514
+ const st = await stat(job.path);
1515
+ if (st.isFile())
1516
+ parseTimeoutMs = parseTimeoutForSize(st.size, this.fileLimits);
1517
+ }
1518
+ catch {
1519
+ // 已删时交?parseOne ?
1520
+ }
1521
+ this.heartbeatJob(job.id);
1522
+ await this.withTimeout((signal) => this.parseOne(job.source_id, job.path, signal), parseTimeoutMs, `parse_timeout:${job.path}`, this.abortSignalFor(job.source_id));
1523
+ this.bindJobFileId(job);
1014
1524
  if (this.isAborted(job.source_id)) {
1015
1525
  throw new Error('aborted');
1016
1526
  }
1017
- this.refreshCoverage(job.source_id);
1527
+ this.markCoverageDirty(job.source_id);
1018
1528
  if (this.embedding) {
1019
- this.enqueue(job.source_id, 'embed_source', null, 3);
1529
+ await this.enqueue(job.source_id, 'embed_source', null, 3);
1020
1530
  this.kick();
1021
1531
  }
1022
1532
  }
1023
1533
  else if (job.kind === 'fetch_doc' && job.path) {
1024
- await this.withTimeout(this.fetchDocJob(job.source_id, job.path), this.parseTimeoutMs, `fetch_timeout:${job.path}`);
1025
- this.refreshCoverage(job.source_id);
1534
+ this.heartbeatJob(job.id);
1535
+ await this.withTimeout((signal) => this.fetchDocJob(job.source_id, job.path, signal), this.parseTimeoutMs, `fetch_timeout:${job.path}`, this.abortSignalFor(job.source_id));
1536
+ this.bindJobFileId(job);
1537
+ if (this.isAborted(job.source_id)) {
1538
+ throw new Error('aborted');
1539
+ }
1540
+ this.markCoverageDirty(job.source_id);
1026
1541
  if (this.embedding) {
1027
- this.enqueue(job.source_id, 'embed_source', null, 3);
1542
+ await this.enqueue(job.source_id, 'embed_source', null, 3);
1028
1543
  this.kick();
1029
1544
  }
1030
1545
  }
1031
1546
  else if (job.kind === 'drop_file' && job.path) {
1032
- // 目录移出/删除:连同子路径一并清(否则只剩空目录节点)
1033
- this.index.removePathTree(job.source_id, job.path);
1034
- this.refreshCoverage(job.source_id);
1547
+ // 移出/删除:连同子树异步清(分批让出,禁止同步连环 DELETE 堵死 Engine)
1548
+ await this.index.removePathTreeAsync(job.source_id, job.path);
1549
+ this.markCoverageDirty(job.source_id);
1035
1550
  }
1036
1551
  else if (job.kind === 'walk_source') {
1037
- await this.ingestSource(job.source_id);
1552
+ await this.ingestSource(job.source_id, { fromQueue: true });
1553
+ if (this.isAborted(job.source_id)) {
1554
+ throw new Error('aborted');
1555
+ }
1038
1556
  }
1039
1557
  else if (job.kind === 'embed_source') {
1040
1558
  if (!this.embedding) {
1041
1559
  const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
1042
1560
  if (stillMissing) {
1043
- // 禁止空跑标 done:迁移后缺向量却显示 queue=0
1561
+ // 空跑?done:迁移后缺向量却显示 queue=0
1044
1562
  throw new Error('embedding_provider_missing');
1045
1563
  }
1046
1564
  return;
1047
1565
  }
1048
1566
  await this.embedPending(job.source_id, 50, job.id);
1567
+ if (this.isAborted(job.source_id)) {
1568
+ throw new Error('aborted');
1569
+ }
1570
+ }
1571
+ // 统一收尾闸门:任?kind 后都不得?done
1572
+ if (this.isAborted(job.source_id)) {
1573
+ throw new Error('aborted');
1049
1574
  }
1050
1575
  this.sources.database.raw
1051
1576
  .prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
@@ -1056,16 +1581,18 @@ export class KnowledgeIngest extends EventEmitter {
1056
1581
  path: job.path ?? undefined,
1057
1582
  status: 'done',
1058
1583
  });
1059
- // 标 done 后再续跑:否则 enqueue 去重会把「自己」当成 active 而丢掉下一棒
1584
+ // ?done 后再:否?enqueue 去重会把「自?active 而丢掉下?
1060
1585
  if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
1061
1586
  this.ensureEmbedJob(job.source_id);
1062
- this.refreshCoverage(job.source_id);
1587
+ this.refreshCoverage(job.source_id, { force: true });
1063
1588
  }
1064
1589
  }
1065
1590
  catch (err) {
1066
1591
  const message = err instanceof Error ? err.message : String(err);
1067
1592
  const aborted = message === 'aborted' || this.isAborted(job.source_id);
1068
- if (job.kind === 'parse_file' && job.path && !aborted) {
1593
+ // 归属拒绝不是「文件解析失不得写入 knowledge_files
1594
+ const ownership = message.includes('not owned by source');
1595
+ if (job.kind === 'parse_file' && job.path && !aborted && !ownership) {
1069
1596
  this.index.markFileError(job.source_id, job.path, message);
1070
1597
  }
1071
1598
  this.sources.database.raw
@@ -1079,28 +1606,41 @@ export class KnowledgeIngest extends EventEmitter {
1079
1606
  throw err;
1080
1607
  }
1081
1608
  }
1082
- async withTimeout(p, ms, label) {
1083
- let timer;
1609
+ /**
1610
+ * 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
1611
+ * 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
1612
+ */
1613
+ async withTimeout(work, ms, label, parent) {
1614
+ const controller = new AbortController();
1615
+ const onParentAbort = () => controller.abort();
1616
+ parent?.addEventListener('abort', onParentAbort, { once: true });
1617
+ const timer = setTimeout(() => controller.abort(), ms);
1084
1618
  try {
1085
- return await Promise.race([
1086
- p,
1087
- new Promise((_, reject) => {
1088
- timer = setTimeout(() => reject(new Error(label)), ms);
1089
- }),
1090
- ]);
1619
+ return await work(controller.signal);
1620
+ }
1621
+ catch (err) {
1622
+ if (controller.signal.aborted) {
1623
+ const msg = err instanceof Error ? err.message : String(err);
1624
+ if (msg === 'aborted' || msg.includes('abort'))
1625
+ throw new Error(label);
1626
+ throw new Error(label);
1627
+ }
1628
+ throw err;
1091
1629
  }
1092
1630
  finally {
1093
- if (timer)
1094
- clearTimeout(timer);
1631
+ clearTimeout(timer);
1632
+ parent?.removeEventListener('abort', onParentAbort);
1095
1633
  }
1096
1634
  }
1097
1635
  /**
1098
- * 外源单文档 job:按逻辑键 fetch + 索引
1636
+ * 外源单文?job:按逻辑?fetch + 索引
1099
1637
  */
1100
- async fetchDocJob(sourceId, path) {
1638
+ async fetchDocJob(sourceId, path, signal) {
1101
1639
  const source = this.sources.get(sourceId);
1102
1640
  if (!source || source.status === 'removed')
1103
1641
  return;
1642
+ if (signal?.aborted)
1643
+ throw new Error('aborted');
1104
1644
  const cred = await this.resolveCred(source);
1105
1645
  const existing = this.index.getFile(sourceId, path);
1106
1646
  const ref = {
@@ -1116,21 +1656,21 @@ export class KnowledgeIngest extends EventEmitter {
1116
1656
  : this.localFetcher;
1117
1657
  const doc = await fetcher.fetch(source, ref, cred);
1118
1658
  if (doc)
1119
- this.indexVirtualDoc(sourceId, doc);
1659
+ await this.indexVirtualDoc(sourceId, doc, source.authRef);
1120
1660
  }
1121
- async parseOne(sourceId, filePath) {
1661
+ async parseOne(sourceId, filePath, signal) {
1662
+ const source = this.sources.get(sourceId);
1663
+ // 防御深度:即使队污染,也?root 外的文件
1664
+ if (source && !this.sourceOwnsPath(source, filePath)) {
1665
+ throw new Error(`path not owned by source: ${filePath}`);
1666
+ }
1667
+ if (signal?.aborted)
1668
+ throw new Error('aborted');
1122
1669
  if (this.shouldSkipFile(filePath)) {
1123
1670
  this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
1124
1671
  return false;
1125
1672
  }
1126
- const adapter = this.adapters.match(filePath);
1127
- if (!adapter) {
1128
- if (this.documentPort && isDocumentPath(filePath)) {
1129
- return this.parseDocumentFile(sourceId, filePath);
1130
- }
1131
- this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
1132
- return false;
1133
- }
1673
+ // 先确认是件:/节点不得?knowledge_files(历史上?no_adapter?
1134
1674
  let st;
1135
1675
  try {
1136
1676
  st = await stat(filePath);
@@ -1139,47 +1679,83 @@ export class KnowledgeIngest extends EventEmitter {
1139
1679
  this.index.removeFile(sourceId, filePath);
1140
1680
  return false;
1141
1681
  }
1142
- if (!st.isFile())
1682
+ if (!st.isFile()) {
1683
+ this.index.removeFile(sourceId, filePath);
1143
1684
  return false;
1144
- if (st.size > this.maxFileBytes) {
1145
- this.index.markFileSkipped(sourceId, filePath, 'oversize');
1685
+ }
1686
+ const adapter = this.adapters.match(filePath);
1687
+ if (!adapter) {
1688
+ // worker DocumentPort;注?port 仅作 worker 不可用时的回
1689
+ if (isDocumentPath(filePath)) {
1690
+ return this.parseDocumentFile(sourceId, filePath, signal);
1691
+ }
1692
+ this.index.markFileSkipped(sourceId, filePath, 'no_adapter', st.size, await this.identityKeyFor(filePath));
1146
1693
  return false;
1147
1694
  }
1148
- const content = await readFile(filePath, 'utf8');
1149
- const contentHash = hashContent(content);
1150
- if (this.index.isFresh(sourceId, filePath, contentHash)) {
1151
- return true;
1695
+ // Axure 导出 data.js/document.js:内容嗅探后 skip,避免误伤普通 data.js
1696
+ if (/\.(js|cjs|mjs)$/i.test(filePath)) {
1697
+ const head = await this.readFileHead(filePath, 256);
1698
+ if (head && looksLikeAxureExportJs(head)) {
1699
+ this.index.markFileSkipped(sourceId, filePath, 'axure_export_sidecar', st.size, await this.identityKeyFor(filePath));
1700
+ return false;
1701
+ }
1152
1702
  }
1703
+ const kind = classifyFileKind(filePath);
1704
+ const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
1705
+ if (sizeDecision.action === 'skip') {
1706
+ this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size, await this.identityKeyFor(filePath));
1707
+ return false;
1708
+ }
1709
+ // 读盘 + hash + 切块 + FTS token 全在 worker(Service 事件循环只做写库)
1710
+ const maxTextChars = this.fileLimits.partial.maxTextChars;
1711
+ if (signal?.aborted)
1712
+ throw new Error('aborted');
1153
1713
  try {
1154
- const chunks = adapter.chunk(content, filePath);
1155
- this.index.upsertFile({
1714
+ const { parseTextInWorker } = await import('./parse-text-in-worker.js');
1715
+ const parsed = await parseTextInWorker(filePath, {
1716
+ sizeDecision: sizeDecision.action === 'partial' ? 'partial' : 'ok',
1717
+ maxTextChars,
1718
+ timeoutMs: parseTimeoutForSize(st.size, this.fileLimits),
1719
+ signal,
1720
+ });
1721
+ if (this.index.isFresh(sourceId, filePath, parsed.contentHash)) {
1722
+ return true;
1723
+ }
1724
+ if (signal?.aborted)
1725
+ throw new Error('aborted');
1726
+ const ident = await identifyLocalFile(filePath);
1727
+ await this.index.upsertFile({
1156
1728
  sourceId,
1157
1729
  path: filePath,
1158
- contentHash,
1730
+ contentHash: parsed.contentHash,
1159
1731
  size: st.size,
1160
1732
  mtime: Math.floor(st.mtimeMs),
1161
- adapterId: adapter.id,
1162
- chunks,
1163
- });
1733
+ adapterId: parsed.adapterId,
1734
+ chunks: parsed.chunks,
1735
+ identityKey: ident.key,
1736
+ }, { signal });
1164
1737
  return true;
1165
1738
  }
1166
1739
  catch (err) {
1167
- this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
1740
+ const msg = err instanceof Error ? err.message : String(err);
1741
+ if (msg === 'aborted' || msg.startsWith('text_parse_timeout') || signal?.aborted) {
1742
+ throw err;
1743
+ }
1744
+ this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
1168
1745
  return false;
1169
1746
  }
1170
1747
  }
1171
1748
  /**
1172
- * PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
1173
- * 新鲜度以原件字节 hash 为准。
1749
+ * PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
1750
+ * 新鲜度以原件字节 hash 为准?
1174
1751
  *
1175
- * @param sourceId - 知识源 id
1176
- * @param filePath - 本地文件路径
1177
- * @returns 是否成功入索引
1752
+ * @param sourceId - 知识?id
1753
+ * @param filePath - 文件
1754
+ * @returns 成功入索?
1178
1755
  */
1179
- async parseDocumentFile(sourceId, filePath) {
1180
- const port = this.documentPort;
1181
- if (!port)
1182
- return false;
1756
+ async parseDocumentFile(sourceId, filePath, signal) {
1757
+ if (signal?.aborted)
1758
+ throw new Error('aborted');
1183
1759
  let st;
1184
1760
  try {
1185
1761
  st = await stat(filePath);
@@ -1190,53 +1766,88 @@ export class KnowledgeIngest extends EventEmitter {
1190
1766
  }
1191
1767
  if (!st.isFile())
1192
1768
  return false;
1193
- if (st.size > this.maxFileBytes) {
1194
- this.index.markFileSkipped(sourceId, filePath, 'oversize');
1769
+ const kind = classifyFileKind(filePath);
1770
+ const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
1771
+ if (sizeDecision.action === 'skip') {
1772
+ this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size);
1195
1773
  return false;
1196
1774
  }
1197
- const raw = await readFile(filePath);
1198
- const contentHash = hashContent(raw);
1199
- if (this.index.isFresh(sourceId, filePath, contentHash)) {
1200
- return true;
1201
- }
1202
1775
  try {
1203
- // worker 抽取:同步 xlsx/pdf 不堵主循环;超时/中止可 terminate
1204
- const result = await this.extractDocument(filePath, sourceId);
1205
- const markdown = result.markdown?.trim();
1206
- if (!markdown) {
1207
- this.index.markFileSkipped(sourceId, filePath, 'empty_content');
1776
+ // worker 抽取 + 流式 hash + 切块 + ftsToks:xlsx/pdf 不堵 Service 事件循环
1777
+ const extracted = await this.extractDocument(filePath, sourceId, st.size, signal);
1778
+ const contentHash = extracted.contentHash;
1779
+ if (this.index.isFresh(sourceId, filePath, contentHash)) {
1780
+ return true;
1781
+ }
1782
+ const markdown = extracted.result.markdown?.trim();
1783
+ if (!markdown || extracted.chunks.length === 0) {
1784
+ this.index.markFileSkipped(sourceId, filePath, 'empty_content', st.size);
1208
1785
  return false;
1209
1786
  }
1210
- const chunks = markdownAdapter.chunk(markdown, filePath);
1211
- this.index.upsertFile({
1787
+ if (signal?.aborted)
1788
+ throw new Error('aborted');
1789
+ const ident = await identifyLocalFile(filePath);
1790
+ await this.index.upsertFile({
1212
1791
  sourceId,
1213
1792
  path: filePath,
1214
1793
  contentHash,
1215
1794
  size: st.size,
1216
1795
  mtime: Math.floor(st.mtimeMs),
1217
- adapterId: `document:${result.backend}`,
1218
- chunks,
1219
- });
1796
+ adapterId: `document:${extracted.result.backend}`,
1797
+ chunks: extracted.chunks,
1798
+ identityKey: ident.key,
1799
+ }, { signal });
1220
1800
  return true;
1221
1801
  }
1222
1802
  catch (err) {
1803
+ const msg = err instanceof Error ? err.message : String(err);
1804
+ // /超时不是「这件解析失必须上抛?job ?failed止静?done
1805
+ if (msg === 'aborted' || msg.startsWith('parse_timeout') || signal?.aborted) {
1806
+ throw err;
1807
+ }
1223
1808
  if (isDocumentExtractError(err)) {
1224
- // 结构化跳过(密码/老格式/超大等),不挡整库
1225
- this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase());
1809
+ // 结构化跳过(密码/老格?超大等),不挡整?
1810
+ const identKey = await this.identityKeyFor(filePath);
1811
+ this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase(), st.size, identKey);
1226
1812
  return false;
1227
1813
  }
1228
- this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
1814
+ this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
1229
1815
  return false;
1230
1816
  }
1231
1817
  }
1232
- /** 优先 worker 抽取;worker 不可用时退回进程内 port */
1233
- async extractDocument(filePath, sourceId) {
1234
- const signal = this.abortSignalFor(sourceId);
1818
+ /** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
1819
+ async extractDocument(filePath, sourceId, fileSize = 0, extraSignal) {
1820
+ const sourceSignal = this.abortSignalFor(sourceId);
1821
+ const controller = new AbortController();
1822
+ const abort = () => controller.abort();
1823
+ sourceSignal.addEventListener('abort', abort, { once: true });
1824
+ extraSignal?.addEventListener('abort', abort, { once: true });
1825
+ const signal = controller.signal;
1826
+ const timeoutMs = parseTimeoutForSize(fileSize, this.fileLimits);
1827
+ const extractOpts = {
1828
+ timeoutMs,
1829
+ // knowledge hardMax:软超限仍可 partial 抽取
1830
+ maxFileBytes: this.fileLimits.hardMaxFileBytes,
1831
+ maxSheets: this.fileLimits.partial.maxSheets,
1832
+ maxRowsPerSheet: this.fileLimits.partial.maxRowsPerSheet,
1833
+ maxPages: this.fileLimits.partial.maxPdfPages,
1834
+ maxTextChars: this.fileLimits.partial.maxTextChars,
1835
+ };
1235
1836
  try {
1837
+ // 注入的 documentPort 是调用方选定的抽取实现(测试 stub / 自定义后端)
1838
+ if (this.documentPort) {
1839
+ const result = await this.documentPort.extract({ path: filePath, name: filePath }, { ...extractOpts, signal });
1840
+ const markdown = result.markdown?.trim() ?? '';
1841
+ const chunks = markdown
1842
+ ? await this.chunkTextInWorker(markdown, filePath, signal)
1843
+ : [];
1844
+ const contentHash = await hashFileStreaming(filePath, signal);
1845
+ return { result, chunks, contentHash };
1846
+ }
1236
1847
  const { extractDocumentInWorker } = await import('./extract-document.js');
1237
1848
  return await extractDocumentInWorker(filePath, {
1238
- timeoutMs: this.parseTimeoutMs,
1239
- maxFileBytes: this.maxFileBytes,
1849
+ ...extractOpts,
1850
+ documentConfig: this.documentConfig,
1240
1851
  signal,
1241
1852
  });
1242
1853
  }
@@ -1245,93 +1856,34 @@ export class KnowledgeIngest extends EventEmitter {
1245
1856
  if (msg === 'extract_aborted' || signal.aborted) {
1246
1857
  throw new Error('aborted');
1247
1858
  }
1248
- if (msg.includes('Cannot find module') || msg.includes('ERR_MODULE')) {
1249
- return this.documentPort.extract({ path: filePath, name: filePath }, { timeoutMs: this.parseTimeoutMs });
1250
- }
1251
1859
  throw err;
1252
1860
  }
1861
+ finally {
1862
+ sourceSignal.removeEventListener('abort', abort);
1863
+ extraSignal?.removeEventListener('abort', abort);
1864
+ }
1865
+ }
1866
+ /** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
1867
+ async chunkTextInWorker(text, path, signal, adapterId) {
1868
+ const { chunkTextWithFtsToks } = await import('./parse-text-in-worker.js');
1869
+ return chunkTextWithFtsToks(text, path, signal, adapterId);
1253
1870
  }
1254
1871
  /**
1255
1872
  * Phase B:嵌入待处理 chunk
1256
1873
  *
1257
- * 单 job 内按 embedConcurrency **并行**多批调用 API;
1258
- * 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
1874
+ * ?job 内按 embedConcurrency ****多批调用 API?
1875
+ * 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
1259
1876
  */
1260
1877
  async embedPending(sourceId, maxRounds = 20, jobId) {
1261
- if (!this.embedding) {
1262
- if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
1263
- throw new Error('embedding_provider_missing');
1264
- }
1265
- return;
1266
- }
1267
- // 磁盘对齐在 reindex/walk 做;embed 任务内不再全量 prune(曾拖慢解析)
1268
- const lanes = Math.max(1, this.embedConcurrency);
1269
- const batchSize = Math.max(1, this.embedBatch);
1270
- for (let round = 0; round < maxRounds; round++) {
1271
- if (this.isAborted(sourceId))
1272
- return;
1273
- const pending = this.index.listChunksMissingEmbedding(sourceId, batchSize * lanes);
1274
- if (pending.length === 0) {
1275
- this.refreshCoverage(sourceId);
1276
- return;
1277
- }
1278
- if (jobId)
1279
- this.heartbeatJob(jobId);
1280
- // 轮次限速(并行片共享同一闸门)
1281
- if (this.embedMinIntervalMs > 0) {
1282
- const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
1283
- if (wait > 0)
1284
- await new Promise((r) => setTimeout(r, wait));
1285
- }
1286
- const slices = [];
1287
- for (let i = 0; i < pending.length; i += batchSize) {
1288
- slices.push(pending.slice(i, i + batchSize));
1289
- }
1290
- const settled = await Promise.allSettled(slices.map((slice) => this.embedOneSlice(sourceId, slice)));
1291
- let ok = 0;
1292
- let failed = false;
1293
- for (const s of settled) {
1294
- if (s.status === 'fulfilled')
1295
- ok += s.value;
1296
- else
1297
- failed = true;
1298
- }
1299
- this.lastEmbedAt = Date.now();
1300
- this.emitProgress({
1301
- type: 'embed',
1302
- sourceId,
1303
- status: 'batch_done',
1304
- detail: `embedded=${ok}/${pending.length} lanes=${slices.length}`,
1305
- });
1306
- if (failed) {
1307
- // 至少一片失败:交给 done 后的 ensureEmbedJob 续跑,不把整 job 打成 failed
1308
- return;
1309
- }
1310
- }
1311
- this.refreshCoverage(sourceId);
1878
+ await this.embedRunner.embedPending(sourceId, maxRounds, jobId);
1312
1879
  }
1313
- /** 单片 embedding;返回成功写入条数 */
1314
- async embedOneSlice(sourceId, slice) {
1315
- if (slice.length === 0)
1316
- return 0;
1317
- const vectors = await Promise.race([
1318
- this.embedding.embedBatch(slice.map((p) => p.text)),
1319
- new Promise((_, reject) => setTimeout(() => reject(new Error('embed_batch_timeout')), 120_000)),
1320
- ]);
1321
- let ok = 0;
1322
- const batch = [];
1323
- for (let i = 0; i < slice.length; i++) {
1324
- const vec = vectors[i];
1325
- if (vec?.length) {
1326
- batch.push([slice[i].id, vec]);
1327
- ok += 1;
1328
- }
1329
- }
1330
- if (batch.length > 0)
1331
- this.index.setChunkEmbeddings(batch);
1332
- return ok;
1880
+ recordEmbedSecretHit(input) {
1881
+ this.sources.database.raw
1882
+ .prepare(`INSERT INTO knowledge_embed_secret_log (id, source_id, chunk_id, action, hits_json, created_at)
1883
+ VALUES (?, ?, ?, ?, ?, ?)`)
1884
+ .run(`kes_${randomUUID().slice(0, 12)}`, input.sourceId, input.chunkId, input.action, JSON.stringify(input.hits), Date.now());
1333
1885
  }
1334
- /** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
1886
+ /** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
1335
1887
  ensureEmbedJob(sourceId) {
1336
1888
  if (!this.embedding)
1337
1889
  return;
@@ -1339,21 +1891,42 @@ export class KnowledgeIngest extends EventEmitter {
1339
1891
  return;
1340
1892
  if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
1341
1893
  return;
1342
- this.enqueue(sourceId, 'embed_source', null, 3);
1343
- this.kick();
1894
+ void this.enqueue(sourceId, 'embed_source', null, 3)
1895
+ .then(() => this.kick())
1896
+ .catch(() => undefined);
1344
1897
  }
1345
1898
  heartbeatJob(jobId) {
1346
- this.sources.database.raw
1347
- .prepare(`UPDATE knowledge_jobs SET updated_at = ? WHERE id = ? AND status = 'running'`)
1348
- .run(Date.now(), jobId);
1899
+ this.jobQueue.heartbeat(jobId);
1900
+ }
1901
+ /** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
1902
+ coverageDirty = new Set();
1903
+ markCoverageDirty(sourceId) {
1904
+ this.coverageDirty.add(sourceId);
1905
+ }
1906
+ flushCoverageDirty(sourceId) {
1907
+ const ids = sourceId ? [sourceId] : [...this.coverageDirty];
1908
+ if (sourceId)
1909
+ this.coverageDirty.delete(sourceId);
1910
+ else
1911
+ this.coverageDirty.clear();
1912
+ for (const id of ids)
1913
+ this.refreshCoverage(id, { force: true });
1349
1914
  }
1350
- refreshCoverage(sourceId) {
1915
+ lastCoverageRefreshAt = new Map();
1916
+ refreshCoverage(sourceId, opts) {
1917
+ const now = Date.now();
1918
+ const last = this.lastCoverageRefreshAt.get(sourceId) ?? 0;
1919
+ // 大库 sourceStats/embeddingCoverage 是多表 COUNT;解析/嵌入高频调用时节流,
1920
+ // 避免 Engine 线程被统计查询堵住导致 HTTP list 超时。收尾路径 force。
1921
+ if (!opts?.force && now - last < 2_000)
1922
+ return;
1923
+ this.lastCoverageRefreshAt.set(sourceId, now);
1351
1924
  const stats = this.index.sourceStats(sourceId);
1352
1925
  const source = this.sources.get(sourceId);
1353
1926
  if (!source)
1354
1927
  return;
1355
1928
  const discovered = this.discoveredFiles.get(sourceId) ?? 0;
1356
- // Phase A:已落库文件(indexed+skipped+error)/ walk 发现数
1929
+ // Phase A:已落库文件(indexed+skipped+error? walk 发现?
1357
1930
  const phaseA = discovered > 0
1358
1931
  ? Math.min(1, stats.files / discovered)
1359
1932
  : stats.files > 0
@@ -1389,10 +1962,29 @@ export class KnowledgeIngest extends EventEmitter {
1389
1962
  });
1390
1963
  }
1391
1964
  emitProgress(evt) {
1392
- this.emit('progress', evt);
1393
- this.emit('knowledge.index.progress', evt);
1965
+ let out = evt;
1966
+ if (evt.sourceId && evt.registeredBy === undefined) {
1967
+ try {
1968
+ const row = this.sources.database.raw
1969
+ .prepare('SELECT tenant_id, registered_by, visibility FROM knowledge_sources WHERE id = ?')
1970
+ .get(evt.sourceId);
1971
+ if (row) {
1972
+ out = {
1973
+ ...evt,
1974
+ tenantId: row.tenant_id,
1975
+ registeredBy: row.registered_by,
1976
+ visibility: row.visibility,
1977
+ };
1978
+ }
1979
+ }
1980
+ catch {
1981
+ /* 源已删时保持原事件;SSE 侧按无归属放行到 fallback */
1982
+ }
1983
+ }
1984
+ this.emit('progress', out);
1985
+ this.emit('knowledge.index.progress', out);
1394
1986
  }
1395
- /** 磁盘水位告警(仅告警,不丢任务) */
1987
+ /** 磁盘水位(仅,不务) */
1396
1988
  async warnDiskWatermark(sourceId) {
1397
1989
  try {
1398
1990
  const { statfs } = await import('node:fs/promises');
@@ -1405,7 +1997,7 @@ export class KnowledgeIngest extends EventEmitter {
1405
1997
  type: 'error',
1406
1998
  sourceId,
1407
1999
  status: 'disk_watermark',
1408
- detail: `free≈${Math.round(freeMb)}MB (jobs kept)`,
2000
+ detail: `free?{Math.round(freeMb)}MB (jobs kept)`,
1409
2001
  });
1410
2002
  }
1411
2003
  }