@octopi-agent/engine 0.56.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +43 -1
  4. package/dist/config-schema/substrate.d.ts +42 -0
  5. package/dist/config-schema/substrate.js +58 -0
  6. package/dist/config.d.ts +39 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/agent/config-bridge.js +3 -0
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  23. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  24. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  25. package/dist/harness/capabilities/document/factory.js +37 -0
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  37. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  39. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  40. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  41. package/dist/harness/capabilities/document/worker-port.js +78 -0
  42. package/dist/harness/context/capabilities/index.d.ts +2 -2
  43. package/dist/harness/context/capabilities/index.js +2 -2
  44. package/dist/harness/context/constitution/default-agents.md +1 -0
  45. package/dist/harness/context/index.d.ts +2 -2
  46. package/dist/harness/context/index.js +1 -1
  47. package/dist/harness/context/layer-types.d.ts +26 -2
  48. package/dist/harness/context/layer-types.js +59 -11
  49. package/dist/harness/context/layers.d.ts +3 -0
  50. package/dist/harness/context/layers.js +38 -12
  51. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  52. package/dist/harness/context/system-prompt-assembler.js +2 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/index.d.ts +5 -3
  57. package/dist/harness/index.js +3 -2
  58. package/dist/harness/knowledge/adapters.d.ts +8 -5
  59. package/dist/harness/knowledge/adapters.js +53 -13
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/client.d.ts +197 -0
  62. package/dist/harness/knowledge/client.js +237 -0
  63. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  64. package/dist/harness/knowledge/connectors.d.ts +3 -3
  65. package/dist/harness/knowledge/connectors.js +14 -8
  66. package/dist/harness/knowledge/db.d.ts +42 -3
  67. package/dist/harness/knowledge/db.js +466 -83
  68. package/dist/harness/knowledge/describe.d.ts +2 -0
  69. package/dist/harness/knowledge/describe.js +5 -1
  70. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  71. package/dist/harness/knowledge/document-extract-worker.js +68 -0
  72. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  73. package/dist/harness/knowledge/embed-runner.js +171 -0
  74. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  75. package/dist/harness/knowledge/engine-thread.js +302 -0
  76. package/dist/harness/knowledge/extract-document.d.ts +33 -0
  77. package/dist/harness/knowledge/extract-document.js +86 -0
  78. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  79. package/dist/harness/knowledge/fetchers.js +52 -16
  80. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  81. package/dist/harness/knowledge/file-identity.js +172 -0
  82. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  83. package/dist/harness/knowledge/file-index-store.js +132 -0
  84. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  85. package/dist/harness/knowledge/file-limits.js +175 -0
  86. package/dist/harness/knowledge/fts.d.ts +60 -0
  87. package/dist/harness/knowledge/fts.js +218 -0
  88. package/dist/harness/knowledge/grounding.d.ts +3 -2
  89. package/dist/harness/knowledge/http-app.d.ts +50 -0
  90. package/dist/harness/knowledge/http-app.js +652 -0
  91. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  92. package/dist/harness/knowledge/http-bridge.js +17 -0
  93. package/dist/harness/knowledge/index-store.d.ts +166 -63
  94. package/dist/harness/knowledge/index-store.js +1025 -282
  95. package/dist/harness/knowledge/index.d.ts +47 -5
  96. package/dist/harness/knowledge/index.js +24 -2
  97. package/dist/harness/knowledge/ingest.d.ts +322 -49
  98. package/dist/harness/knowledge/ingest.js +1383 -267
  99. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  100. package/dist/harness/knowledge/job-control-state.js +44 -0
  101. package/dist/harness/knowledge/job-control.d.ts +35 -0
  102. package/dist/harness/knowledge/job-control.js +86 -0
  103. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  104. package/dist/harness/knowledge/job-queue.js +173 -0
  105. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  106. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  107. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  108. package/dist/harness/knowledge/local-stack.js +63 -0
  109. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  110. package/dist/harness/knowledge/membership-store.js +198 -0
  111. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  112. package/dist/harness/knowledge/network-guard.js +130 -85
  113. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  114. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  115. package/dist/harness/knowledge/query-service.d.ts +272 -0
  116. package/dist/harness/knowledge/query-service.js +326 -0
  117. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  118. package/dist/harness/knowledge/query-worker-client.js +207 -0
  119. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  120. package/dist/harness/knowledge/query-worker.js +186 -0
  121. package/dist/harness/knowledge/read-http.d.ts +35 -0
  122. package/dist/harness/knowledge/read-http.js +373 -0
  123. package/dist/harness/knowledge/retriever.d.ts +30 -5
  124. package/dist/harness/knowledge/retriever.js +98 -35
  125. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  126. package/dist/harness/knowledge/secret-scan.js +25 -2
  127. package/dist/harness/knowledge/serve.d.ts +48 -0
  128. package/dist/harness/knowledge/serve.js +336 -0
  129. package/dist/harness/knowledge/source-store.d.ts +83 -16
  130. package/dist/harness/knowledge/source-store.js +205 -78
  131. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  132. package/dist/harness/knowledge/start-service-process.js +158 -0
  133. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  134. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  135. package/dist/harness/knowledge/topics.d.ts +13 -0
  136. package/dist/harness/knowledge/topics.js +48 -0
  137. package/dist/harness/knowledge/types.d.ts +1 -1
  138. package/dist/harness/knowledge/types.js +1 -1
  139. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  140. package/dist/harness/knowledge/vector-ann.js +68 -0
  141. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  142. package/dist/harness/knowledge/writer-lock.js +71 -0
  143. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  144. package/dist/harness/knowledge/writer-service.js +355 -0
  145. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  146. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  147. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  148. package/dist/harness/knowledge/writer-worker.js +189 -0
  149. package/dist/harness/memory/index.d.ts +3 -2
  150. package/dist/harness/memory/index.js +3 -1
  151. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  152. package/dist/harness/memory/retrieval-rank.js +60 -0
  153. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  154. package/dist/harness/memory/sqlite/embedding.js +109 -16
  155. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  156. package/dist/harness/memory/sqlite/index.js +1 -1
  157. package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
  158. package/dist/harness/memory/sqlite/keyword-search.js +104 -17
  159. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  160. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  161. package/dist/harness/memory/store.js +39 -21
  162. package/dist/harness/memory/types.d.ts +7 -0
  163. package/dist/harness/run/runner.d.ts +1 -0
  164. package/dist/harness/run/runner.js +4 -2
  165. package/dist/harness/session/attachments/service.d.ts +1 -1
  166. package/dist/harness/session/attachments/service.js +1 -1
  167. package/dist/harness/session/tasks/index.d.ts +1 -1
  168. package/dist/harness/session/tasks/index.js +1 -1
  169. package/dist/harness/session/tasks/render.d.ts +10 -0
  170. package/dist/harness/session/tasks/render.js +27 -0
  171. package/dist/index.d.ts +4 -2
  172. package/dist/index.js +2 -1
  173. package/dist/worker-path.d.ts +8 -0
  174. package/dist/worker-path.js +15 -0
  175. package/package.json +2 -2
@@ -1,29 +1,72 @@
1
1
  /**
2
- * KnowledgeIngest — 解析/切块/索引队列(P2 Phase A)
2
+ * KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
3
3
  *
4
- * 原则:ingest 与对话并发,不作 turn 同步前置;队列背压不丢任务;source 级锁。
4
+ * 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
5
5
  */
6
- import { randomUUID } from 'node:crypto';
7
- import { readFile, stat } from 'node:fs/promises';
6
+ import { randomUUID, createHash } from 'node:crypto';
7
+ import { open, stat } from 'node:fs/promises';
8
+ import { createReadStream, statSync, realpathSync } from 'node:fs';
8
9
  import { watch } from 'node:fs';
9
- import { join, sep } from 'node:path';
10
+ import { isAbsolute, join, relative, resolve, sep } from 'node:path';
10
11
  import { EventEmitter } from 'node:events';
11
- import { FormatAdapterRegistry, htmlAdapter, markdownAdapter, textAdapter } from './adapters.js';
12
+ import { FormatAdapterRegistry, htmlAdapter, looksLikeAxureExportJs, markdownAdapter, textAdapter, } from './adapters.js';
12
13
  import { looksLikeHtml } from './html.js';
13
14
  import { hashContent, KnowledgeIndexStore } from './index-store.js';
14
- import { LocalFsFetcher, UrlFetcher } from './fetchers.js';
15
+ import { identifyLocalFile, urlIdentityKey, normalizePathLexical, } from './file-identity.js';
16
+ import { KnowledgeJobControl } from './job-control.js';
17
+ import { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
15
18
  import { ConnectorRegistry } from './connectors.js';
16
19
  import { ConnectorFetcher } from './connector-fetcher.js';
17
- import { isDocumentPath } from '../context/capabilities/document/format.js';
18
- import { isDocumentExtractError } from '../context/capabilities/document/errors.js';
20
+ import { isDocumentPath } from '../capabilities/document/format.js';
21
+ import { isDocumentExtractError } from '../capabilities/document/errors.js';
22
+ import { classifyFileKind, decideBySize, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
23
+ import { EmbedRunner } from './embed-runner.js';
24
+ /**
25
+ * 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
26
+ *
27
+ * @param filePath - 本地文件
28
+ * @param signal - 中止信号
29
+ * @returns hex digest
30
+ */
31
+ export async function hashFileStreaming(filePath, signal) {
32
+ const hash = createHash('sha256');
33
+ await new Promise((resolve, reject) => {
34
+ if (signal?.aborted) {
35
+ reject(new Error('aborted'));
36
+ return;
37
+ }
38
+ const stream = createReadStream(filePath);
39
+ const onAbort = () => {
40
+ stream.destroy();
41
+ reject(new Error('aborted'));
42
+ };
43
+ signal?.addEventListener('abort', onAbort, { once: true });
44
+ stream.on('data', (chunk) => hash.update(chunk));
45
+ stream.once('end', () => {
46
+ signal?.removeEventListener('abort', onAbort);
47
+ resolve();
48
+ });
49
+ stream.once('error', (err) => {
50
+ signal?.removeEventListener('abort', onAbort);
51
+ reject(err);
52
+ });
53
+ });
54
+ return hash.digest('hex');
55
+ }
56
+ import { JobQueue } from './job-queue.js';
57
+ import { readJobControlState } from './job-control-state.js';
19
58
  export class KnowledgeIngest extends EventEmitter {
20
59
  sources;
21
60
  index;
22
61
  adapters;
23
62
  parseConcurrency;
63
+ documentParseConcurrency;
24
64
  maxQueueDepth;
25
65
  debounceMs;
26
66
  maxFileBytes;
67
+ fileLimits;
68
+ /** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
69
+ parseTimeoutMs;
27
70
  embedding;
28
71
  embedBatch;
29
72
  embedMinIntervalMs;
@@ -37,14 +80,29 @@ export class KnowledgeIngest extends EventEmitter {
37
80
  pollMinIntervalMs;
38
81
  maxPollPerTick;
39
82
  documentPort;
40
- lastEmbedAt = 0;
83
+ documentConfig;
84
+ embedSecretPolicy;
85
+ embedRunner;
41
86
  pollTimer = null;
42
- running = 0;
87
+ reconcileTimer = null;
88
+ /** 源级:DB 权威 + ?AbortSignal(跨重启?*/
89
+ jobControl;
90
+ jobQueue;
91
+ /** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
92
+ onSourceSettled;
93
+ /** parse 缺口时间 */
94
+ lastParseGapScanAt = new Map();
95
+ /** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
96
+ gapScanUrgent = new Set();
97
+ gapScanTimer = null;
98
+ runningParse = 0;
99
+ runningDocParse = 0;
43
100
  runningEmbed = 0;
44
101
  draining = false;
45
- /** source 级锁:同一 source 不并行 walk/parse 冲突写 */
102
+ kickAgain = false;
103
+ /** source 级锁:同 source 不并?walk/parse 冲突?*/
46
104
  sourceLocks = new Set();
47
- /** walk 发现的文件数(Phase A coverage 分母) */
105
+ /** walk 发现的文件数(Phase A coverage 分母?*/
48
106
  discoveredFiles = new Map();
49
107
  watchers = new Map();
50
108
  debounceTimers = new Map();
@@ -52,19 +110,68 @@ export class KnowledgeIngest extends EventEmitter {
52
110
  constructor(options) {
53
111
  super();
54
112
  this.sources = options.sourceStore;
113
+ this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
114
+ this.jobControl = new KnowledgeJobControl(options.sourceStore.database);
115
+ this.jobQueue = new JobQueue({
116
+ db: options.sourceStore.database,
117
+ maxQueueDepth: this.maxQueueDepth,
118
+ isAborted: (id) => this.isAborted(id),
119
+ onBackpressure: (sourceId, queued) => {
120
+ this.emitProgress({
121
+ type: 'error',
122
+ sourceId,
123
+ status: 'queue_backpressure',
124
+ detail: `queued=${queued} (delay only, jobs kept)`,
125
+ });
126
+ },
127
+ });
55
128
  this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
56
129
  this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
57
- this.parseConcurrency = options.parseConcurrency ?? 4;
58
- this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
59
- this.debounceMs = options.debounceMs ?? 2000;
60
- this.maxFileBytes = options.maxFileBytes ?? 5_000_000;
130
+ this.parseConcurrency = options.parseConcurrency ?? 8;
131
+ this.documentParseConcurrency = options.documentParseConcurrency ?? 2;
132
+ // watch 批量落盘时缩默窗,新增文件更?
133
+ this.debounceMs = options.debounceMs ?? 800;
134
+ this.fileLimits = resolveKnowledgeFileLimits({
135
+ ...options.fileLimits,
136
+ maxFileBytes: options.fileLimits?.maxFileBytes ?? options.maxFileBytes,
137
+ parseTimeoutMs: options.fileLimits?.parseTimeoutMs ?? options.parseTimeoutMs,
138
+ });
139
+ this.maxFileBytes = this.fileLimits.maxFileBytes;
140
+ this.parseTimeoutMs = this.fileLimits.parseTimeoutMs;
61
141
  this.embedding = options.embeddingProvider ?? null;
62
142
  this.embedBatch = options.embedBatch ?? 32;
63
143
  this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
64
- this.embedConcurrency = options.embedConcurrency ?? options.parseConcurrency ?? 2;
144
+ // embed ?parse 分槽;缺?4 行打 embedding API
145
+ this.embedConcurrency = options.embedConcurrency ?? 4;
65
146
  this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
66
147
  this.credentials = options.credentials ?? null;
67
148
  this.documentPort = options.documentPort ?? null;
149
+ this.documentConfig = options.documentConfig ?? null;
150
+ this.embedSecretPolicy = options.embedSecretPolicy ?? 'redact';
151
+ this.embedRunner = new EmbedRunner({
152
+ index: this.index,
153
+ embedding: this.embedding,
154
+ embedBatch: this.embedBatch,
155
+ embedConcurrency: this.embedConcurrency,
156
+ embedMinIntervalMs: this.embedMinIntervalMs,
157
+ embedSecretPolicy: this.embedSecretPolicy,
158
+ isAborted: (id) => this.isAborted(id),
159
+ heartbeatJob: (id) => this.heartbeatJob(id),
160
+ refreshCoverage: (id) => this.refreshCoverage(id),
161
+ emitProgress: (evt) => this.emitProgress(evt),
162
+ recordSecretHit: (input) => this.recordEmbedSecretHit(input),
163
+ });
164
+ // embedding 外发(合规可查;非权量)
165
+ this.sources.database.raw.exec(`
166
+ CREATE TABLE IF NOT EXISTS knowledge_embed_secret_log (
167
+ id TEXT PRIMARY KEY,
168
+ source_id TEXT NOT NULL,
169
+ chunk_id TEXT NOT NULL,
170
+ action TEXT NOT NULL,
171
+ hits_json TEXT NOT NULL,
172
+ created_at INTEGER NOT NULL
173
+ );
174
+ `);
68
175
  this.localFetcher =
69
176
  options.fetchers?.local ?? new LocalFsFetcher((p) => this.shouldSkipFile(p));
70
177
  this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
@@ -74,25 +181,157 @@ export class KnowledgeIngest extends EventEmitter {
74
181
  this.pollTickMs = options.pollTickMs ?? 60_000;
75
182
  this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
76
183
  this.maxPollPerTick = options.maxPollPerTick ?? 8;
184
+ // 崩溃/重启遗留?running 会堵?enqueue 去重并永久显?indexing
185
+ this.reclaimOrphanRunningJobs();
186
+ // 历史:目录等非文件曾?no_adapter,启动即?
187
+ this.cleanupNonFileIndexRows();
188
+ }
189
+ lastCleanupAt = 0;
190
+ /**
191
+ * 清掉「非?knowledge_files (目录曾 no_adapter?
192
+ *
193
+ * ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
194
+ * 60s 节流,避免每?reconcile ?stat?
195
+ *
196
+ * @param sourceId - 限定源;省略则全部源
197
+ * @param opts.force - 忽略节流
198
+ * @returns 删除行数
199
+ */
200
+ cleanupNonFileIndexRows(sourceId, opts) {
201
+ const now = Date.now();
202
+ if (!opts?.force && sourceId == null && now - this.lastCleanupAt < 60_000) {
203
+ return 0;
204
+ }
205
+ this.lastCleanupAt = now;
206
+ const ids = sourceId
207
+ ? [sourceId]
208
+ : this.sources.list().map((s) => s.id);
209
+ let removed = 0;
210
+ for (const id of ids) {
211
+ const rows = this.sources.database.raw
212
+ .prepare(`SELECT m.logical_path AS path
213
+ FROM knowledge_memberships m
214
+ JOIN knowledge_files f ON f.id = m.file_id
215
+ WHERE m.source_id = ? AND f.status IN ('skipped', 'error') AND f.chunk_count = 0`)
216
+ .all(id);
217
+ for (const r of rows) {
218
+ try {
219
+ const st = statSync(r.path);
220
+ if (!st.isFile()) {
221
+ this.index.removeFile(id, r.path);
222
+ removed += 1;
223
+ }
224
+ }
225
+ catch {
226
+ // 不存留给 pruneMissing齐),避删有效但暂时不可见的文件
227
+ }
228
+ }
229
+ }
230
+ return removed;
231
+ }
232
+ /**
233
+ * 无跨进程 Lease:本进程时把历史 running 收回 queued?
234
+ * 仅构造时调用行中同进程的 running 法的?
235
+ */
236
+ reclaimOrphanRunningJobs() {
237
+ this.jobQueue.reclaimOrphanRunning();
77
238
  }
78
239
  get indexStore() {
79
240
  return this.index;
80
241
  }
81
242
  /**
82
- * 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
243
+ * 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
83
244
  *
84
- * @param p - 文件路径
85
- * @returns 是否跳过
245
+ * @param p - 文件
246
+ * @returns 跳过
86
247
  */
87
248
  shouldSkipFile(p) {
88
- if (this.documentPort && isDocumentPath(p))
249
+ if (isDocumentPath(p))
89
250
  return false;
90
251
  return this.adapters.shouldSkipPath(p);
91
252
  }
253
+ /** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
254
+ async readFileHead(filePath, bytes = 256) {
255
+ try {
256
+ const fh = await open(filePath, 'r');
257
+ try {
258
+ const buf = Buffer.alloc(bytes);
259
+ const { bytesRead } = await fh.read(buf, 0, bytes, 0);
260
+ return buf.subarray(0, bytesRead).toString('utf8');
261
+ }
262
+ finally {
263
+ await fh.close();
264
+ }
265
+ }
266
+ catch {
267
+ return null;
268
+ }
269
+ }
270
+ /** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
271
+ async identityKeyFor(filePath) {
272
+ try {
273
+ return (await identifyLocalFile(filePath)).key;
274
+ }
275
+ catch {
276
+ return undefined;
277
+ }
278
+ }
279
+ /**
280
+ * 归属:`parse_file` / 读文件只读源 root 内的?
281
+ * ?**realpath** 比较,防符号链接逃出 root?
282
+ *
283
+ * @param source - 知识?
284
+ * @param path - 请求(本地绝对路径,
285
+ */
286
+ sourceOwnsPath(source, path) {
287
+ if (source.kind === 'url' || source.kind === 'connector') {
288
+ return Boolean(this.index.getFile(source.id, path));
289
+ }
290
+ const root = resolve(source.location);
291
+ const target = resolve(path);
292
+ const eq = (a, b) => process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b;
293
+ // 词法(路径不存在时也拒绝明显越界?
294
+ if (!eq(root, target)) {
295
+ const relLex = relative(root, target);
296
+ if (!relLex || relLex.startsWith('..') || isAbsolute(relLex))
297
+ return false;
298
+ }
299
+ // realpath ?symlink stat 失败回词法结果)
300
+ try {
301
+ const rootReal = realpathSync(root);
302
+ const targetReal = realpathSync(target);
303
+ if (eq(rootReal, targetReal))
304
+ return true;
305
+ const rel = relative(rootReal, targetReal);
306
+ return Boolean(rel) && !rel.startsWith('..') && !isAbsolute(rel);
307
+ }
308
+ catch {
309
+ return true; // 词法;文件尚在时交给后续 stat
310
+ }
311
+ }
92
312
  /**
93
- * 全量/增量索引某源(Phase A)
313
+ * 全量/增量索引某源(Phase A?
314
+ *
315
+ * @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
316
+ * @param opts.incremental - 增量(poll):?active 任务则跳过本?
94
317
  */
95
318
  async ingestSource(sourceId, opts) {
319
+ // fromQueue(walk_source):不 supersede、不跳过 active;reindex 才作废队列
320
+ if (opts?.fromQueue) {
321
+ // fall through to walk
322
+ }
323
+ else if (opts?.full !== false && !opts?.incremental) {
324
+ this.supersedeSourceWork(sourceId);
325
+ }
326
+ else if (this.countActiveJobs(sourceId).total > 0) {
327
+ this.emitProgress({
328
+ type: 'source',
329
+ sourceId,
330
+ status: 'poll_skipped',
331
+ detail: 'active jobs in flight',
332
+ });
333
+ return;
334
+ }
96
335
  const source = this.sources.get(sourceId);
97
336
  if (!source || source.status === 'removed') {
98
337
  throw new Error(`knowledge source not found: ${sourceId}`);
@@ -114,24 +353,34 @@ export class KnowledgeIngest extends EventEmitter {
114
353
  return;
115
354
  }
116
355
  if (this.sourceLocks.has(sourceId)) {
117
- // 已有同源任务在跑:入队 walk 即可(去重)
118
- this.enqueue(sourceId, 'walk_source', null, 1, source.location);
356
+ // 已有同源任务在跑:入?walk 即可(去重)
357
+ await this.enqueue(sourceId, 'walk_source', null, 1);
119
358
  return;
120
359
  }
121
360
  this.sourceLocks.add(sourceId);
122
361
  try {
123
- this.sources.update(sourceId, { status: 'discovering', coverage: 0 });
362
+ // 保留旧 coverage:重建期间索引仍在库中,写 0 会让 UI 误判为清空
363
+ this.sources.update(sourceId, { status: 'discovering' });
124
364
  this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
125
- // full 不再 clearSource:靠本轮 keepPaths 差量 prune,discover 失败不丢库
365
+ // full 不再 clearSource:靠 keepPaths prune,discover 失败不丢?
126
366
  if (source.kind === 'url' || source.kind === 'connector') {
127
367
  await this.ingestRemoteSource(source);
128
368
  return;
129
369
  }
130
- const files = await this.localFetcher.discover(source);
370
+ // watch 策略索引同时挂上 fs 监听(幂等;否则后续变更静默丢失?
371
+ this.startWatch(sourceId);
372
+ const localDiscovery = await this.localFetcher.discover(source);
373
+ const files = localDiscovery.docs;
131
374
  this.discoveredFiles.set(sourceId, files.length);
132
- // 差量 prune:磁盘上已消失的 path
133
- const localKeep = new Set(files.map((f) => f.path));
134
- this.index.pruneMissing(sourceId, localKeep);
375
+ // 后不 prune、不入队(半?keep 删)
376
+ if (this.isAborted(sourceId))
377
+ return;
378
+ // prune:仅 walk 完整时执行;出错/?keep 不完整会
379
+ if (localDiscovery.complete) {
380
+ const localKeep = new Set(files.map((f) => f.path));
381
+ // walk 成功且目录为空时允许清空 Membership(§6.2)
382
+ await this.index.pruneMissingAsync(sourceId, localKeep, { allowEmptyKeep: true });
383
+ }
135
384
  this.sources.update(sourceId, { status: 'partial', coverage: 0 });
136
385
  this.emitProgress({
137
386
  type: 'source',
@@ -140,10 +389,14 @@ export class KnowledgeIngest extends EventEmitter {
140
389
  detail: `${files.length} files`,
141
390
  });
142
391
  for (const file of files) {
143
- this.enqueue(sourceId, 'parse_file', file.path, 2, file.path);
392
+ if (this.isAborted(sourceId))
393
+ return;
394
+ if (this.isUnchangedIndexed(sourceId, file))
395
+ continue;
396
+ await this.enqueue(sourceId, 'parse_file', file.path, 2);
144
397
  }
145
- if (this.embedding) {
146
- this.enqueue(sourceId, 'embed_source', null, 3);
398
+ if (this.embedding && !this.isAborted(sourceId)) {
399
+ await this.enqueue(sourceId, 'embed_source', null, 3);
147
400
  }
148
401
  this.kick();
149
402
  }
@@ -151,6 +404,25 @@ export class KnowledgeIngest extends EventEmitter {
151
404
  this.sourceLocks.delete(sourceId);
152
405
  }
153
406
  }
407
+ /**
408
+ * 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
409
+ * 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
410
+ * indexing/error/半成品必须重做,避免把未完成当成新鲜。
411
+ */
412
+ isUnchangedIndexed(sourceId, ref) {
413
+ if (ref.size == null || ref.mtime == null)
414
+ return false;
415
+ const existing = this.index.getFile(sourceId, ref.path);
416
+ if (!existing)
417
+ return false;
418
+ if (existing.status !== 'indexed')
419
+ return false;
420
+ if (existing.chunkCount <= 0)
421
+ return false;
422
+ if (!existing.contentHash)
423
+ return false;
424
+ return existing.size === ref.size && existing.mtime === ref.mtime;
425
+ }
154
426
  /**
155
427
  * URL / connector 源:discover + fetch + 索引
156
428
  */
@@ -170,9 +442,9 @@ export class KnowledgeIngest extends EventEmitter {
170
442
  return;
171
443
  }
172
444
  const fetcher = source.kind === 'connector' ? this.connectorFetcher : this.urlFetcher;
173
- let refs;
445
+ let discovered;
174
446
  try {
175
- refs = await fetcher.discover(source, cred);
447
+ discovered = await fetcher.discover(source, cred);
176
448
  }
177
449
  catch (err) {
178
450
  const msg = err instanceof Error ? err.message : String(err);
@@ -183,7 +455,8 @@ export class KnowledgeIngest extends EventEmitter {
183
455
  this.emitProgress({ type: 'error', sourceId: source.id, status: 'discover_failed', detail: msg });
184
456
  return;
185
457
  }
186
- // 条件 GET:带上已有 etag/Last-Modified,304 则整文档跳过
458
+ const refs = discovered.docs;
459
+ // 条件 GET:带上已?etag/Last-Modified?04 则整文档跳过
187
460
  const enriched = refs.map((ref) => {
188
461
  const existing = this.index.getFile(source.id, ref.path);
189
462
  return {
@@ -203,15 +476,25 @@ export class KnowledgeIngest extends EventEmitter {
203
476
  let okCount = 0;
204
477
  let unchanged = 0;
205
478
  for (const ref of enriched) {
479
+ // 后立刻停:不再写索引
480
+ if (this.isAborted(source.id)) {
481
+ this.emitProgress({
482
+ type: 'source',
483
+ sourceId: source.id,
484
+ status: 'aborted',
485
+ detail: `stopped after ${okCount + unchanged}/${enriched.length}`,
486
+ });
487
+ return;
488
+ }
206
489
  try {
207
490
  const doc = await fetcher.fetch(source, ref, cred);
208
491
  if (!doc) {
209
- // 304 / 未变:保持原索引
492
+ // 304 / :保持原索引
210
493
  unchanged += 1;
211
494
  okCount += 1;
212
495
  continue;
213
496
  }
214
- if (this.indexVirtualDoc(source.id, doc)) {
497
+ if (await this.indexVirtualDoc(source.id, doc, source.authRef)) {
215
498
  okCount += 1;
216
499
  }
217
500
  }
@@ -226,22 +509,25 @@ export class KnowledgeIngest extends EventEmitter {
226
509
  });
227
510
  }
228
511
  }
229
- // 差量 prune:discover 成功后,源上已消失的文档才删
230
- const keep = new Set(enriched.map((r) => r.path));
231
- const pruned = this.index.pruneMissing(source.id, keep);
232
- if (pruned > 0) {
233
- this.emitProgress({
234
- type: 'source',
235
- sourceId: source.id,
236
- status: 'pruned',
237
- detail: `removed=${pruned}`,
238
- });
512
+ // prune:仅?discover **完整** 时才;截?walk 失败?keep 不完整,prune ?
513
+ // 后不?prune:半?keep 删已入库文档
514
+ if (!this.isAborted(source.id) && discovered.complete) {
515
+ const keep = new Set(enriched.map((r) => r.path));
516
+ const pruned = await this.index.pruneMissingAsync(source.id, keep, { allowEmptyKeep: true });
517
+ if (pruned > 0) {
518
+ this.emitProgress({
519
+ type: 'source',
520
+ sourceId: source.id,
521
+ status: 'pruned',
522
+ detail: `removed=${pruned}`,
523
+ });
524
+ }
239
525
  }
240
- if (this.embedding) {
241
- this.enqueue(source.id, 'embed_source', null, 3);
526
+ if (this.embedding && !this.isAborted(source.id)) {
527
+ await this.enqueue(source.id, 'embed_source', null, 3);
242
528
  this.kick();
243
529
  }
244
- this.refreshCoverage(source.id);
530
+ this.refreshCoverage(source.id, { force: true });
245
531
  this.emitProgress({
246
532
  type: 'source',
247
533
  sourceId: source.id,
@@ -250,10 +536,10 @@ export class KnowledgeIngest extends EventEmitter {
250
536
  });
251
537
  }
252
538
  /**
253
- * 解析源凭证;配置了 authRef 时 **fail-closed**(解析失败则本轮不抓,不降级匿名)。
539
+ * 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
254
540
  *
255
- * @returns 凭证或 null(仅当源未配置 authRef)
256
- * @throws authRef 配置了但无法解析时抛错,由调用方中止本轮
541
+ * @returns ?null(仅当源?authRef?
542
+ * @throws authRef 配置了但无法解析时抛错,由调用方
257
543
  */
258
544
  async resolveCred(source) {
259
545
  if (!source.authRef)
@@ -268,16 +554,16 @@ export class KnowledgeIngest extends EventEmitter {
268
554
  return resolved;
269
555
  }
270
556
  /**
271
- * 将 VirtualDocument 写入索引(外源 / 规范化后文本)
557
+ * ?VirtualDocument 写入索引?/ 规范化后文本?
272
558
  *
273
- * @returns 是否成功写入
559
+ * @returns 成功写入
274
560
  */
275
- indexVirtualDoc(sourceId, doc) {
561
+ async indexVirtualDoc(sourceId, doc, authRef) {
276
562
  if (!doc.content?.trim()) {
277
563
  this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
278
564
  return false;
279
565
  }
280
- // 逻辑键可能无扩展名(connector id / URL path):按 MIME → 扩展名 → 内容形态回退
566
+ // 逻辑能无扩展名(connector id / URL path):按 MIME / 内容启发式
281
567
  let adapter = this.adapters.match(doc.path, doc.contentType);
282
568
  if (!adapter) {
283
569
  if (looksLikeHtml(doc.content, doc.contentType))
@@ -292,8 +578,20 @@ export class KnowledgeIngest extends EventEmitter {
292
578
  return true;
293
579
  }
294
580
  try {
295
- const chunks = adapter.chunk(doc.content, doc.path);
296
- this.index.upsertFile({
581
+ const chunks = await this.chunkTextInWorker(doc.content, doc.path, undefined, adapter.id);
582
+ let identityKey;
583
+ if (doc.externalUrl) {
584
+ identityKey = urlIdentityKey(doc.externalUrl, authRef);
585
+ }
586
+ else {
587
+ try {
588
+ identityKey = `path:${normalizePathLexical(doc.path)}`;
589
+ }
590
+ catch {
591
+ identityKey = `path:${doc.path.replace(/\\/g, '/')}`;
592
+ }
593
+ }
594
+ await this.index.upsertFile({
297
595
  sourceId,
298
596
  path: doc.path,
299
597
  contentHash,
@@ -301,6 +599,7 @@ export class KnowledgeIngest extends EventEmitter {
301
599
  mtime: Math.floor(Date.now() / 1000),
302
600
  adapterId: adapter.id,
303
601
  chunks,
602
+ identityKey,
304
603
  externalUrl: doc.externalUrl,
305
604
  etag: doc.etag,
306
605
  lastModified: doc.lastModified,
@@ -313,13 +612,129 @@ export class KnowledgeIngest extends EventEmitter {
313
612
  }
314
613
  }
315
614
  /**
316
- * 单文件急索(P0 快车道;同步完成 parse,不 await 全库)
615
+ * 单文(P0 道;完成 parse,不 await 全库?
317
616
  */
318
617
  async ingestFileNow(sourceId, filePath) {
319
618
  const ok = await this.parseOne(sourceId, filePath);
320
619
  this.refreshCoverage(sourceId);
321
620
  return ok;
322
621
  }
622
+ /**
623
+ * 重做指定文件:强制重?解析 ?分块 ?向量
624
+ *
625
+ * - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
626
+ * - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
627
+ * - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
628
+ * - 等非文件:直接清,不入队
629
+ * - 完成后走既有 embed 链路
630
+ *
631
+ * @param sourceId - ?id
632
+ * @param paths - 文件列表
633
+ * @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
634
+ */
635
+ async reprocessFiles(sourceId, paths) {
636
+ if (!paths.length) {
637
+ return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: 0 };
638
+ }
639
+ const source = this.sources.get(sourceId);
640
+ if (!source) {
641
+ return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: paths.length };
642
+ }
643
+ let resumed = false;
644
+ if (this.isAborted(sourceId)) {
645
+ this.beginAbortEpoch(sourceId);
646
+ resumed = true;
647
+ }
648
+ let queued = 0;
649
+ let alreadyActive = 0;
650
+ let cleanedNonFiles = 0;
651
+ let rejected = 0;
652
+ const isRemote = source.kind === 'url' || source.kind === 'connector';
653
+ for (const p of paths) {
654
+ if (!this.sourceOwnsPath(source, p)) {
655
+ rejected += 1;
656
+ continue;
657
+ }
658
+ if (isRemote) {
659
+ // 外源:path 重做 = 重新 fetch + 索引(不读本地盘?
660
+ this.index.invalidateFileForReparse(sourceId, p);
661
+ if (await this.enqueue(sourceId, 'fetch_doc', p, 1)) {
662
+ queued += 1;
663
+ }
664
+ else if (this.hasActiveParseJob(sourceId, p)) {
665
+ alreadyActive += 1;
666
+ }
667
+ continue;
668
+ }
669
+ // /节点:清即可,没?
670
+ try {
671
+ const st = statSync(p);
672
+ if (!st.isFile()) {
673
+ this.index.removeFile(sourceId, p);
674
+ cleanedNonFiles += 1;
675
+ continue;
676
+ }
677
+ }
678
+ catch {
679
+ // 不存仍按文件流程 parse ?
680
+ }
681
+ // 总是失效 hash:即使已在队列,跑起来也不能 isFresh 跳过
682
+ this.index.invalidateFileForReparse(sourceId, p);
683
+ if (await this.enqueue(sourceId, 'parse_file', p, 1)) {
684
+ queued += 1;
685
+ }
686
+ else if (this.hasActiveParseJob(sourceId, p)) {
687
+ alreadyActive += 1;
688
+ }
689
+ }
690
+ this.kick();
691
+ return { queued, alreadyActive, cleanedNonFiles, resumed, rejected };
692
+ }
693
+ /** 上是否已?queued/running ?parse/fetch 任务 */
694
+ hasActiveParseJob(sourceId, path) {
695
+ return this.jobQueue.hasActivePathJob(sourceId, path);
696
+ }
697
+ /**
698
+ * 级任?文件UI 跟踪重做完成?
699
+ *
700
+ * @param sourceId - ?id
701
+ * @param paths - 列表
702
+ */
703
+ jobStateForPaths(sourceId, paths) {
704
+ return paths.map((p) => {
705
+ const jobs = this.sources.database.raw
706
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs
707
+ WHERE source_id = ? AND path = ? AND status IN ('queued','running')`)
708
+ .get(sourceId, p);
709
+ const f = this.index.getFile(sourceId, p);
710
+ let exists = false;
711
+ try {
712
+ exists = statSync(p).isFile();
713
+ }
714
+ catch {
715
+ exists = false;
716
+ }
717
+ return {
718
+ path: p,
719
+ jobsActive: Number(jobs?.n ?? 0),
720
+ fileStatus: f?.status ?? null,
721
+ chunkCount: f?.chunkCount ?? 0,
722
+ error: f?.error ?? null,
723
+ exists,
724
+ };
725
+ });
726
+ }
727
+ /**
728
+ * 按筛选批量重做(当前列表?status/ext/q 条件?
729
+ *
730
+ * @param sourceId - ?id
731
+ * @param opts - 与文件列表筛选一?
732
+ * @returns 入队条数
733
+ */
734
+ async reprocessByFilter(sourceId, opts) {
735
+ const paths = this.index.listFilePathsFiltered(sourceId, opts);
736
+ return this.reprocessFiles(sourceId, paths);
737
+ }
323
738
  /**
324
739
  * 处理 fs 变更(debounce 后入队)
325
740
  */
@@ -332,30 +747,33 @@ export class KnowledgeIngest extends EventEmitter {
332
747
  clearTimeout(prev);
333
748
  this.debounceTimers.set(key, setTimeout(() => {
334
749
  this.debounceTimers.delete(key);
335
- if (event === 'rename') {
336
- // inotify:create/move 也报 rename — 按存在性分支,勿一律 drop
337
- void stat(filePath)
338
- .then((st) => {
339
- if (st.isFile()) {
340
- this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
341
- }
342
- else {
343
- this.enqueue(sourceId, 'drop_file', filePath, 1, filePath);
344
- }
750
+ // 统一?stat:目?change 时不?parse ?no_adapter
751
+ void stat(filePath)
752
+ .then(async (st) => {
753
+ if (st.isFile()) {
754
+ await this.enqueue(sourceId, 'parse_file', filePath, 1);
755
+ }
756
+ else if (event === 'rename' || !st.isDirectory()) {
757
+ await this.enqueue(sourceId, 'drop_file', filePath, 1);
758
+ }
759
+ else {
760
+ this.index.removeFile(sourceId, filePath);
761
+ }
762
+ this.requestGapScan(sourceId);
763
+ this.kick();
764
+ })
765
+ .catch(() => {
766
+ void this.enqueue(sourceId, 'drop_file', filePath, 1)
767
+ .then(() => {
768
+ this.requestGapScan(sourceId);
345
769
  this.kick();
346
770
  })
347
- .catch(() => {
348
- this.enqueue(sourceId, 'drop_file', filePath, 1, filePath);
349
- this.kick();
350
- });
351
- return;
352
- }
353
- this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
354
- this.kick();
771
+ .catch(() => undefined);
772
+ });
355
773
  }, this.debounceMs));
356
774
  }
357
775
  /**
358
- * 启动目录 watch(幂等)
776
+ * watch(幂等)
359
777
  */
360
778
  startWatch(sourceId) {
361
779
  const source = this.sources.get(sourceId);
@@ -399,11 +817,11 @@ export class KnowledgeIngest extends EventEmitter {
399
817
  }
400
818
  }
401
819
  /**
402
- * 等待队列排空(测试/管理)
820
+ * 等待队列排空(测?管理?
403
821
  */
404
822
  async idle(timeoutMs = 30_000) {
405
823
  const start = Date.now();
406
- while (this.running > 0 || this.hasQueuedJobs()) {
824
+ while (this.runningParse > 0 || this.runningEmbed > 0 || this.hasQueuedJobs()) {
407
825
  if (Date.now() - start > timeoutMs) {
408
826
  throw new Error('knowledge ingest idle timeout');
409
827
  }
@@ -414,11 +832,436 @@ export class KnowledgeIngest extends EventEmitter {
414
832
  dispose() {
415
833
  this.disposed = true;
416
834
  this.stopPolling();
835
+ this.stopReconciler();
836
+ if (this.gapScanTimer) {
837
+ clearTimeout(this.gapScanTimer);
838
+ this.gapScanTimer = null;
839
+ }
840
+ this.gapScanUrgent.clear();
841
+ void this.abortJobs();
417
842
  for (const id of this.watchers.keys())
418
843
  this.stopWatch(id);
419
844
  }
420
845
  /**
421
- * 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
846
+ * 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
847
+ *
848
+ * @param opts.sourceId - 仅中源;省略则全部源
849
+ * @returns 取消
850
+ */
851
+ abortJobs(opts) {
852
+ const sid = opts?.sourceId?.trim();
853
+ const ids = sid
854
+ ? [sid]
855
+ : [
856
+ ...this.jobControl.listAbortedSourceIds(),
857
+ ...this.sources.list().map((s) => s.id),
858
+ ];
859
+ const unique = [...new Set(ids)];
860
+ let cancelledQueued = 0;
861
+ let abortedRunning = 0;
862
+ for (const id of unique) {
863
+ // 1) DB 落中跨重+ 打断程信?
864
+ const newly = this.jobControl.markAborted(id);
865
+ if (newly)
866
+ abortedRunning += 1;
867
+ // 2) 清空 queued ?cancelled(共?File ?parse 不杀:仍领则保留?
868
+ const res = this.sources.database.raw
869
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
870
+ WHERE source_id = ? AND status = 'queued'
871
+ AND (
872
+ file_id IS NULL
873
+ OR kind IN ('walk_source', 'embed_source')
874
+ OR NOT EXISTS (
875
+ SELECT 1 FROM knowledge_memberships m
876
+ WHERE m.file_id = knowledge_jobs.file_id AND m.source_id != ?
877
+ )
878
+ )`)
879
+ .run(Date.now(), id, id);
880
+ cancelledQueued += Number(res.changes ?? 0);
881
+ }
882
+ // 3) 看门狗不要再?aborted running 收回 queued(限定本次涉及的源)
883
+ if (sid) {
884
+ this.sources.database.raw
885
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
886
+ WHERE source_id = ? AND status = 'running' AND last_error = 'aborted'`)
887
+ .run(Date.now(), sid);
888
+ }
889
+ else {
890
+ this.sources.database.raw
891
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
892
+ WHERE status = 'running' AND last_error = 'aborted'`)
893
+ .run(Date.now());
894
+ }
895
+ const runningRow = this.sources.database.raw
896
+ .prepare(sid
897
+ ? `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE source_id = ? AND status = 'running'`
898
+ : `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'running'`)
899
+ .get(...(sid ? [sid] : []));
900
+ const runningJobs = Number(runningRow?.n ?? 0);
901
+ this.emitProgress({
902
+ type: 'source',
903
+ sourceId: sid ?? '*',
904
+ status: 'aborted',
905
+ detail: `cancelledQueued=${cancelledQueued} runningTail=${runningJobs}`,
906
+ });
907
+ return { cancelledQueued, abortedRunning, runningJobs };
908
+ }
909
+ /**
910
+ * /继续索引(相轻)
911
+ *
912
+ * - 清除该源?
913
+ * - ?cancelled 任务放回 queued
914
+ * - 若仍缺向量则排队 embed
915
+ * - 不重?walk 全量;需要全 reindex
916
+ *
917
+ * @param opts.sourceId - 仅恢源;省略则全部源
918
+ * @returns
919
+ */
920
+ resumeJobs(opts) {
921
+ const sid = opts?.sourceId?.trim();
922
+ const ids = sid
923
+ ? [sid]
924
+ : [
925
+ ...new Set([
926
+ ...this.jobControl.listAbortedSourceIds(),
927
+ ...this.sources.list().map((s) => s.id),
928
+ ]),
929
+ ];
930
+ let restoredCancelled = 0;
931
+ let embedQueued = 0;
932
+ for (const id of ids) {
933
+ // 1) 清除态(DB + 新纪元信号)
934
+ this.beginAbortEpoch(id);
935
+ // 2) cancelled ?queued(用户中止时的活?
936
+ const res = this.sources.database.raw
937
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', last_error = NULL, updated_at = ?
938
+ WHERE source_id = ? AND status = 'cancelled' AND (last_error IS NULL OR last_error = 'aborted')`)
939
+ .run(Date.now(), id);
940
+ restoredCancelled += Number(res.changes ?? 0);
941
+ // 3) 仍缺向量则确?embed 在队
942
+ if (this.embedding) {
943
+ const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
944
+ const active = this.countActiveJobs(id);
945
+ if (missing && active.embed === 0) {
946
+ this.ensureEmbedJob(id);
947
+ embedQueued += 1;
948
+ }
949
+ }
950
+ }
951
+ this.kick();
952
+ this.emitProgress({
953
+ type: 'source',
954
+ sourceId: sid ?? '*',
955
+ status: 'resumed',
956
+ detail: `restoredCancelled=${restoredCancelled} embedQueued=${embedQueued}`,
957
+ });
958
+ return { restoredCancelled, embedQueued };
959
+ }
960
+ /** 的未(resume / supersede 后用;清 DB 态) */
961
+ beginAbortEpoch(sourceId) {
962
+ this.jobControl.beginEpoch(sourceId);
963
+ }
964
+ /**
965
+ * 重建前作废本源未完成任务(supersede?
966
+ *
967
+ * - queued ?cancelled(superseded_by_reindex?
968
+ * - running 打断(worker/出)
969
+ * - 随后 beginAbortEpoch,本建可继续?
970
+ */
971
+ supersedeSourceWork(sourceId) {
972
+ this.sources.database.raw
973
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
974
+ WHERE source_id = ? AND status = 'queued'`)
975
+ .run(Date.now(), sourceId);
976
+ // 打断在跑 parse/embed;再新纪元(清中,避?walk ?abort
977
+ this.jobControl.markAborted(sourceId);
978
+ this.beginAbortEpoch(sourceId);
979
+ this.emitProgress({
980
+ type: 'source',
981
+ sourceId,
982
+ status: 'superseded',
983
+ detail: 'reindex will resync from disk',
984
+ });
985
+ }
986
+ async ensureParseCoverage(sourceId) {
987
+ const source = this.sources.get(sourceId);
988
+ if (!source || source.status === 'removed' || source.status === 'disabled')
989
+ return 0;
990
+ if (source.kind !== 'directory' && source.kind !== 'workspace')
991
+ return 0;
992
+ if (this.isAborted(sourceId))
993
+ return 0;
994
+ // 仅跳过「全量 walk 在途」:individual parse_file 排队 **不得** 挡住缺口补扫。
995
+ // 否则 watch 漏事件后,当前批次 parse 会一直占着门闩,排空才一次入队数百文件(84 → 726)。
996
+ // 不用 sourceLocks:与 reconcile 同轮 fire-and-forget ingestSource 竞态,会误伤补扫。
997
+ if (this.countQueuedKinds(['walk_source'], sourceId) > 0)
998
+ return 0;
999
+ let found;
1000
+ try {
1001
+ found = await this.localFetcher.discover(source);
1002
+ }
1003
+ catch {
1004
+ return 0;
1005
+ }
1006
+ const byPath = new Map(this.index.listFiles(sourceId).map((f) => [f.path.replace(/\\/g, '/'), f]));
1007
+ let added = 0;
1008
+ let retriedSkipped = 0;
1009
+ let retriedIncomplete = 0;
1010
+ for (const f of found.docs) {
1011
+ const logical = f.path.replace(/\\/g, '/');
1012
+ const existing = byPath.get(logical);
1013
+ // 缺文件必补;indexing=半写入/崩溃残留必补;skipped 且原因可重试时补
1014
+ const need = !existing ||
1015
+ existing.status === 'indexing' ||
1016
+ (existing.status === 'skipped' &&
1017
+ isRetryableSkipReason(existing.error ?? null, existing.size, this.fileLimits));
1018
+ if (!need)
1019
+ continue;
1020
+ if (await this.enqueue(sourceId, 'parse_file', f.path, 2)) {
1021
+ added += 1;
1022
+ if (!existing)
1023
+ continue;
1024
+ if (existing.status === 'indexing')
1025
+ retriedIncomplete += 1;
1026
+ else
1027
+ retriedSkipped += 1;
1028
+ }
1029
+ }
1030
+ if (added > 0) {
1031
+ this.emitProgress({
1032
+ type: 'source',
1033
+ sourceId,
1034
+ status: 'parse_gap_found',
1035
+ detail: `missing_files=${added - retriedSkipped - retriedIncomplete} retry_skipped=${retriedSkipped} retry_indexing=${retriedIncomplete}`,
1036
+ });
1037
+ this.kick();
1038
+ }
1039
+ return added;
1040
+ }
1041
+ /** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
1042
+ requestGapScan(sourceId, delayMs = 3_000) {
1043
+ if (this.disposed || this.isAborted(sourceId))
1044
+ return;
1045
+ this.gapScanUrgent.add(sourceId);
1046
+ if (this.gapScanTimer)
1047
+ return;
1048
+ this.gapScanTimer = setTimeout(() => {
1049
+ this.gapScanTimer = null;
1050
+ const ids = [...this.gapScanUrgent];
1051
+ this.gapScanUrgent.clear();
1052
+ for (const id of ids) {
1053
+ this.lastParseGapScanAt.set(id, Date.now());
1054
+ void this.ensureParseCoverage(id).catch(() => undefined);
1055
+ }
1056
+ }, delayMs);
1057
+ this.gapScanTimer.unref?.();
1058
+ }
1059
+ /**
1060
+ * 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
1061
+ *
1062
+ * 源用?discover walk ?keep 集(;失回并?stat?
1063
+ * embed 前调避免给已移出文件的残 chunks 上向?
1064
+ *
1065
+ * @param sourceId - 知识?id
1066
+ * @returns 删除的文件数
1067
+ */
1068
+ async pruneMissingOnDisk(sourceId) {
1069
+ const source = this.sources.get(sourceId);
1070
+ if (!source || source.kind === 'url' || source.kind === 'connector')
1071
+ return 0;
1072
+ let removed = 0;
1073
+ if (source.kind === 'directory' || source.kind === 'workspace') {
1074
+ try {
1075
+ const found = await this.localFetcher.discover(source);
1076
+ if (!found.complete) {
1077
+ // walk 不完整:?stat止用残缺 keep 清库
1078
+ removed = await this.pruneByStatScan(sourceId);
1079
+ }
1080
+ else {
1081
+ const keep = new Set(found.docs.map((f) => f.path));
1082
+ // walk 完整:即使空也可清空(与逐文?stat 致)
1083
+ removed = await this.index.pruneMissingAsync(sourceId, keep, { allowEmptyKeep: true });
1084
+ }
1085
+ }
1086
+ catch {
1087
+ removed = await this.pruneByStatScan(sourceId);
1088
+ }
1089
+ }
1090
+ else {
1091
+ removed = await this.pruneByStatScan(sourceId);
1092
+ }
1093
+ if (removed > 0) {
1094
+ this.emitProgress({
1095
+ type: 'source',
1096
+ sourceId,
1097
+ status: 'pruned_missing',
1098
+ detail: `removed_files=${removed}`,
1099
+ });
1100
+ this.refreshCoverage(sourceId, { force: true });
1101
+ }
1102
+ return removed;
1103
+ }
1104
+ /** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
1105
+ async pruneByStatScan(sourceId, concurrency = 4) {
1106
+ const files = this.index.listFiles(sourceId);
1107
+ let removed = 0;
1108
+ let i = 0;
1109
+ const worker = async () => {
1110
+ for (;;) {
1111
+ if (this.isAborted(sourceId))
1112
+ return;
1113
+ const idx = i++;
1114
+ if (idx >= files.length)
1115
+ return;
1116
+ const f = files[idx];
1117
+ let exists = true;
1118
+ try {
1119
+ const st = await stat(f.path);
1120
+ exists = st.isFile();
1121
+ }
1122
+ catch {
1123
+ exists = false;
1124
+ }
1125
+ if (!exists) {
1126
+ await this.index.removeFileAsync(sourceId, f.path);
1127
+ removed += 1;
1128
+ }
1129
+ }
1130
+ };
1131
+ await Promise.all(Array.from({ length: Math.min(concurrency, files.length || 1) }, () => worker()));
1132
+ return removed;
1133
+ }
1134
+ abortSignalFor(sourceId) {
1135
+ return this.jobControl.signalFor(sourceId);
1136
+ }
1137
+ isAborted(sourceId) {
1138
+ return this.jobControl.isAborted(sourceId);
1139
+ }
1140
+ /**
1141
+ * /继续 控制数(互斥按钮
1142
+ *
1143
+ * @param sourceId - ?id
1144
+ * @returns 队列计数 + 已中?+ 仍缺向量
1145
+ */
1146
+ jobControlState(sourceId) {
1147
+ return readJobControlState(this.sources.database, this.index, sourceId, {
1148
+ embeddingEnabled: Boolean(this.embedding),
1149
+ });
1150
+ }
1151
+ /**
1152
+ * 任务对账(看门狗?不依赖单?ensureEmbedJob
1153
+ *
1154
+ * 解决「续去重吞掉 / 进程 / partial」等静默停摆?
1155
+ * 1. 回收超时 running(崩?孤儿?
1156
+ * 2. 有缺向量且无 active embed ?排队
1157
+ * 3. 无队缺口 ??coverage/status
1158
+ */
1159
+ startReconciler(intervalMs = 15_000) {
1160
+ if (this.reconcileTimer || this.disposed)
1161
+ return;
1162
+ this.reconcileTimer = setInterval(() => {
1163
+ void this.reconcileJobs().catch((err) => {
1164
+ this.emitProgress({
1165
+ type: 'error',
1166
+ sourceId: '*',
1167
+ status: 'reconcile_failed',
1168
+ detail: err instanceof Error ? err.message : String(err),
1169
+ });
1170
+ });
1171
+ }, intervalMs);
1172
+ this.reconcileTimer.unref?.();
1173
+ }
1174
+ stopReconciler() {
1175
+ if (this.reconcileTimer) {
1176
+ clearInterval(this.reconcileTimer);
1177
+ this.reconcileTimer = null;
1178
+ }
1179
+ }
1180
+ /** 对账入口(也工调 */
1181
+ async reconcileJobs() {
1182
+ if (this.disposed)
1183
+ return;
1184
+ // stale ?**大于** 时,否则合法的长 parse 收回后双?
1185
+ this.reclaimStaleRunningJobs(this.fileLimits.maxParseTimeoutMs + 60_000);
1186
+ // path: 与 win:/unix: 身份分裂的零认领残留
1187
+ this.index.purgeOrphanPathIdentityFiles();
1188
+ // 等历入项(轻量;非每
1189
+ this.cleanupNonFileIndexRows();
1190
+ // ?job 定期清理,防表无?
1191
+ this.cleanupTerminalJobs();
1192
+ // 存量 FTS 后台补齐(分出事件循 rebuild?
1193
+ void this.index.ensureFtsBackfill();
1194
+ // 存量向量回填:小批量 + 让出事件?2000 条同步写(会 abort?
1195
+ await this.index.backfillVecFromEmbeddings(25);
1196
+ for (const s of this.sources.list()) {
1197
+ if (s.status === 'removed' || s.status === 'disabled')
1198
+ continue;
1199
+ if (this.isAborted(s.id))
1200
+ continue;
1201
+ const active = this.countActiveJobs(s.id);
1202
+ // discovering 卡死(注册 kick 失败 / file 源 walk 中断):无任务则 re-kick,不 supersede
1203
+ if (active.total === 0 &&
1204
+ (s.status === 'pending' || s.status === 'discovering') &&
1205
+ (s.kind === 'file' || s.kind === 'directory' || s.kind === 'workspace')) {
1206
+ await this.ingestSource(s.id, { fromQueue: true }).catch(() => undefined);
1207
+ }
1208
+ // 稳定态收尾(auto-describe 等)关键词部署同触发
1209
+ // 先看 active,再 gap ;避免刚入队?gap job 挡住 settled
1210
+ if (active.total === 0) {
1211
+ this.refreshCoverage(s.id, { force: true });
1212
+ try {
1213
+ this.onSourceSettled?.(s.id);
1214
+ }
1215
+ catch {
1216
+ /* ignore settle hook errors */
1217
+ }
1218
+ }
1219
+ else {
1220
+ this.flushCoverageDirty(s.id);
1221
+ }
1222
+ // watch 漏事件 / skip 重试时补齐 parse(与配置 embedding 无关)
1223
+ // urgent(watch 突发)不走 60s 节流;否则批量拷贝后要等一整轮才补
1224
+ const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
1225
+ const gapDue = this.gapScanUrgent.has(s.id) || Date.now() - lastGap > 60_000;
1226
+ if (gapDue) {
1227
+ this.lastParseGapScanAt.set(s.id, Date.now());
1228
+ this.gapScanUrgent.delete(s.id);
1229
+ await this.ensureParseCoverage(s.id);
1230
+ // drop_file 在途时勿再 prune:双路径会同时删同一棵子树,放大写压力
1231
+ const dropsInFlight = this.countQueuedKinds(['drop_file'], s.id) > 0 || this.hasRunningKind(s.id, 'drop_file');
1232
+ if (!dropsInFlight) {
1233
+ await this.pruneMissingOnDisk(s.id);
1234
+ }
1235
+ }
1236
+ const missingEmbed = this.embedding
1237
+ ? this.index.hasChunksMissingEmbedding(s.id)
1238
+ : false;
1239
+ if (this.embedding && missingEmbed && active.embed === 0) {
1240
+ this.ensureEmbedJob(s.id);
1241
+ }
1242
+ }
1243
+ this.kick();
1244
+ }
1245
+ /** ?job 清理(done/failed/cancelled 超过保留期) */
1246
+ cleanupTerminalJobs(retainMs = 24 * 60 * 60_000) {
1247
+ this.jobQueue.cleanupTerminal(retainMs);
1248
+ }
1249
+ /** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
1250
+ reclaimStaleRunningJobs(staleMs) {
1251
+ this.jobQueue.reclaimStaleRunning(staleMs);
1252
+ }
1253
+ countActiveJobs(sourceId) {
1254
+ return this.jobQueue.countActiveJobs(sourceId);
1255
+ }
1256
+ hasRunningKind(sourceId, kind) {
1257
+ const row = this.sources.database.raw
1258
+ .prepare(`SELECT 1 AS ok FROM knowledge_jobs
1259
+ WHERE source_id = ? AND kind = ? AND status = 'running' LIMIT 1`)
1260
+ .get(sourceId, kind);
1261
+ return Boolean(row?.ok);
1262
+ }
1263
+ /**
1264
+ * poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
422
1265
  */
423
1266
  startPolling() {
424
1267
  if (this.pollTimer || this.disposed)
@@ -433,7 +1276,7 @@ export class KnowledgeIngest extends EventEmitter {
433
1276
  });
434
1277
  });
435
1278
  }, this.pollTickMs);
436
- // 不阻止进程退出
1279
+ // 不阻止进?
437
1280
  this.pollTimer.unref?.();
438
1281
  }
439
1282
  stopPolling() {
@@ -443,9 +1286,9 @@ export class KnowledgeIngest extends EventEmitter {
443
1286
  }
444
1287
  }
445
1288
  /**
446
- * 扫描到期 poll 源并入队刷新(内容未变由 content-hash / 304 短路)
1289
+ * 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
447
1290
  *
448
- * @returns 本轮实际触发的 sourceId 列表
1291
+ * @returns 实际触发?sourceId 列表
449
1292
  */
450
1293
  async pollDueSources() {
451
1294
  if (this.disposed)
@@ -475,8 +1318,8 @@ export class KnowledgeIngest extends EventEmitter {
475
1318
  status: 'poll_due',
476
1319
  detail: source.sync.intervalMs?.toString(),
477
1320
  });
478
- // 异步入队,不阻塞 tick
479
- void this.ingestSource(source.id).catch((err) => {
1321
+ // 入队,不 tick量:?active 则跳过, supersede 打断
1322
+ void this.ingestSource(source.id, { incremental: true }).catch((err) => {
480
1323
  this.emitProgress({
481
1324
  type: 'error',
482
1325
  sourceId: source.id,
@@ -487,79 +1330,126 @@ export class KnowledgeIngest extends EventEmitter {
487
1330
  }
488
1331
  return triggered;
489
1332
  }
490
- // ── 内部 ──
1333
+ // 内部
491
1334
  hasQueuedJobs() {
492
- const row = this.sources.database.raw
493
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
494
- .get();
495
- return (row?.n ?? 0) > 0;
496
- }
497
- enqueue(sourceId, kind, path, priority, _detail) {
498
- // 同 source+kind 去重(queued + running,避免并发重复 embed)
499
- const activeStatuses = ['queued', 'running'];
500
- if (path != null) {
501
- const placeholders = activeStatuses.map(() => '?').join(',');
502
- const dup = this.sources.database.raw
503
- .prepare(`SELECT id FROM knowledge_jobs
504
- WHERE source_id = ? AND kind = ? AND path = ? AND status IN (${placeholders})`)
505
- .get(sourceId, kind, path, ...activeStatuses);
506
- if (dup?.id)
507
- return;
508
- }
509
- else {
510
- const placeholders = activeStatuses.map(() => '?').join(',');
511
- const dup = this.sources.database.raw
1335
+ return this.jobQueue.hasQueuedJobs();
1336
+ }
1337
+ /**
1338
+ * 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
1339
+ * 需要计数的路径请 `await enqueueStrict`。
1340
+ */
1341
+ enqueue(sourceId, kind, path, priority, fileIdHint) {
1342
+ return this.enqueueStrict(sourceId, kind, path, priority, fileIdHint).catch(() => false);
1343
+ }
1344
+ /** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
1345
+ *
1346
+ * 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
1347
+ */
1348
+ async enqueueStrict(sourceId, kind, path, priority, fileIdHint) {
1349
+ if (this.disposed)
1350
+ return false;
1351
+ const fileId = fileIdHint ?? (await this.resolveJobFileId(sourceId, kind, path));
1352
+ if (fileId && (kind === 'parse_file' || kind === 'fetch_doc')) {
1353
+ // 共享 File 已有在途 parse:补认领后跳过,避免双写
1354
+ const busy = this.sources.database.raw
512
1355
  .prepare(`SELECT id FROM knowledge_jobs
513
- WHERE source_id = ? AND kind = ? AND path IS NULL AND status IN (${placeholders})`)
514
- .get(sourceId, kind, ...activeStatuses);
515
- if (dup?.id)
516
- return;
517
- }
518
- const now = Date.now();
519
- const depth = this.sources.database.raw
520
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
521
- .get();
522
- if ((depth?.n ?? 0) >= this.maxQueueDepth) {
523
- this.emitProgress({
524
- type: 'error',
525
- sourceId,
526
- status: 'queue_backpressure',
527
- detail: `queued=${depth.n} (delay only, jobs kept)`,
528
- });
1356
+ WHERE file_id = ? AND kind IN ('parse_file','fetch_doc') AND status IN ('queued','running')
1357
+ LIMIT 1`)
1358
+ .get(fileId);
1359
+ if (busy?.id && path) {
1360
+ this.index.attachMembership(sourceId, fileId, path);
1361
+ return false;
1362
+ }
529
1363
  }
530
- if (this.diskWatermarkAlert) {
1364
+ const ok = this.jobQueue.enqueue(sourceId, kind, path, priority, fileId);
1365
+ if (ok && this.diskWatermarkAlert) {
531
1366
  void this.warnDiskWatermark(sourceId);
532
1367
  }
1368
+ return ok;
1369
+ }
1370
+ /** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
1371
+ bindJobFileId(job) {
1372
+ if (job.file_id || !job.path)
1373
+ return;
1374
+ const rec = this.index.getFile(job.source_id, job.path);
1375
+ if (!rec?.id)
1376
+ return;
533
1377
  this.sources.database.raw
534
- .prepare(`INSERT INTO knowledge_jobs (id, source_id, kind, path, priority, status, attempts, created_at, updated_at)
535
- VALUES (?, ?, ?, ?, ?, 'queued', 0, ?, ?)`)
536
- .run(`kj_${randomUUID().slice(0, 12)}`, sourceId, kind, path, priority, now, now);
1378
+ .prepare(`UPDATE knowledge_jobs SET file_id = ? WHERE id = ? AND file_id IS NULL`)
1379
+ .run(rec.id, job.id);
1380
+ job.file_id = rec.id;
1381
+ }
1382
+ /** job 绑定 file_id:membership → identity;本地文件可 identify */
1383
+ async resolveJobFileId(sourceId, kind, path) {
1384
+ if (!path)
1385
+ return null;
1386
+ if (kind !== 'parse_file' && kind !== 'fetch_doc' && kind !== 'drop_file')
1387
+ return null;
1388
+ const logical = path.replace(/\\/g, '/');
1389
+ const existing = this.index.getFile(sourceId, logical);
1390
+ if (existing?.id)
1391
+ return existing.id;
1392
+ try {
1393
+ const ident = await identifyLocalFile(path);
1394
+ return this.index.findFileIdByIdentity(ident.key);
1395
+ }
1396
+ catch {
1397
+ return null;
1398
+ }
537
1399
  }
538
1400
  kick() {
539
- if (this.disposed || this.draining)
1401
+ if (this.disposed)
1402
+ return;
1403
+ if (this.draining) {
1404
+ // 等待 drain 会再 kick;标记一次避免丢信号
1405
+ this.kickAgain = true;
540
1406
  return;
1407
+ }
541
1408
  this.draining = true;
542
1409
  queueMicrotask(() => {
543
1410
  this.draining = false;
544
1411
  this.drain();
1412
+ if (this.kickAgain) {
1413
+ this.kickAgain = false;
1414
+ this.kick();
1415
+ }
545
1416
  });
546
1417
  }
547
1418
  drain() {
548
- while (this.running < this.parseConcurrency) {
549
- const job = this.claimJob();
550
- if (!job)
1419
+ if (this.disposed)
1420
+ return;
1421
+ for (;;) {
1422
+ if (this.disposed)
551
1423
  return;
552
- const isEmbed = job.kind === 'embed_source';
553
- if (isEmbed && this.runningEmbed >= this.embedConcurrency) {
554
- this.sources.database.raw
555
- .prepare(`UPDATE knowledge_jobs SET status = 'queued' WHERE id = ? AND status = 'running'`)
556
- .run(job.id);
557
- // embed 满:本轮不再认领,避免空转把同一 job 抢来抢去
1424
+ const allowParse = this.runningParse < this.parseConcurrency;
1425
+ const allowEmbed = this.runningEmbed < this.embedConcurrency;
1426
+ if (!allowParse && !allowEmbed)
1427
+ return;
1428
+ const kinds = [];
1429
+ if (allowParse)
1430
+ kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
1431
+ // parse 优先(priority 2 < 3),但 **不得** 等全部 parse 结束才 embed:
1432
+ // 大库 parse 可能跑数小时,否则向量永远 0。embed 与 parse 并行,仅受 embed 槽限制。
1433
+ if (allowEmbed)
1434
+ kinds.push('embed_source');
1435
+ if (kinds.length === 0) {
1436
+ return;
1437
+ }
1438
+ const job = this.claimJob(kinds, (j) => this.acceptParseSlot(j));
1439
+ if (!job) {
1440
+ // 队列清空:合并刷 parse/drop 标脏?coverage,避免每文件?COUNT
1441
+ this.flushCoverageDirty();
558
1442
  return;
559
1443
  }
560
- this.running += 1;
1444
+ const isEmbed = job.kind === 'embed_source';
1445
+ const isDocParse = this.isDocParseJob(job);
561
1446
  if (isEmbed)
562
1447
  this.runningEmbed += 1;
1448
+ else {
1449
+ this.runningParse += 1;
1450
+ if (isDocParse)
1451
+ this.runningDocParse += 1;
1452
+ }
563
1453
  void this.runJob(job)
564
1454
  .catch((err) => {
565
1455
  this.emitProgress({
@@ -571,58 +1461,116 @@ export class KnowledgeIngest extends EventEmitter {
571
1461
  });
572
1462
  })
573
1463
  .finally(() => {
574
- this.running -= 1;
575
1464
  if (isEmbed)
576
1465
  this.runningEmbed -= 1;
1466
+ else {
1467
+ this.runningParse -= 1;
1468
+ if (isDocParse)
1469
+ this.runningDocParse -= 1;
1470
+ }
577
1471
  this.kick();
578
1472
  });
579
1473
  }
580
1474
  }
581
- claimJob() {
582
- const row = this.sources.database.raw
583
- .prepare(`SELECT * FROM knowledge_jobs
584
- WHERE status = 'queued'
585
- ORDER BY priority ASC, created_at ASC
586
- LIMIT 1`)
587
- .get();
588
- if (!row)
589
- return null;
590
- const res = this.sources.database.raw
591
- .prepare(`UPDATE knowledge_jobs SET status = 'running', updated_at = ?
592
- WHERE id = ? AND status = 'queued'`)
593
- .run(Date.now(), row.id);
594
- if (Number(res.changes ?? 0) === 0)
595
- return null;
596
- return row;
1475
+ /** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
1476
+ isDocParseJob(job) {
1477
+ return job.kind === 'parse_file' && !!job.path && isDocumentPath(job.path);
1478
+ }
1479
+ acceptParseSlot(job) {
1480
+ if (this.isDocParseJob(job)) {
1481
+ return this.runningDocParse < this.documentParseConcurrency;
1482
+ }
1483
+ return true;
1484
+ }
1485
+ countQueuedKinds(kinds, sourceId) {
1486
+ if (this.disposed)
1487
+ return 0;
1488
+ return this.jobQueue.countQueuedKinds(kinds, sourceId);
1489
+ }
1490
+ claimJob(kinds, accept) {
1491
+ return this.jobQueue.claimJob(kinds, accept);
597
1492
  }
598
1493
  async runJob(job) {
599
1494
  const now = Date.now();
1495
+ if (this.isAborted(job.source_id)) {
1496
+ this.sources.database.raw
1497
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
1498
+ WHERE id = ? AND status = 'running'`)
1499
+ .run(Date.now(), job.id);
1500
+ return;
1501
+ }
600
1502
  try {
1503
+ // 归属闸门:任何会碰本地盘 / 外源逻辑 path 任务都必须属于本?
1504
+ if (job.path && (job.kind === 'parse_file' || job.kind === 'fetch_doc')) {
1505
+ const owner = this.sources.get(job.source_id);
1506
+ if (owner && !this.sourceOwnsPath(owner, job.path)) {
1507
+ throw new Error(`path not owned by source: ${job.path}`);
1508
+ }
1509
+ }
601
1510
  if (job.kind === 'parse_file' && job.path) {
602
- await this.parseOne(job.source_id, job.path);
603
- this.refreshCoverage(job.source_id);
1511
+ // 单文件解析硬超时:大 xlsx/pdf 抽取会堵死事件循超时随体?
1512
+ let parseTimeoutMs = this.parseTimeoutMs;
1513
+ try {
1514
+ const st = await stat(job.path);
1515
+ if (st.isFile())
1516
+ parseTimeoutMs = parseTimeoutForSize(st.size, this.fileLimits);
1517
+ }
1518
+ catch {
1519
+ // 已删时交?parseOne ?
1520
+ }
1521
+ this.heartbeatJob(job.id);
1522
+ await this.withTimeout((signal) => this.parseOne(job.source_id, job.path, signal), parseTimeoutMs, `parse_timeout:${job.path}`, this.abortSignalFor(job.source_id));
1523
+ this.bindJobFileId(job);
1524
+ if (this.isAborted(job.source_id)) {
1525
+ throw new Error('aborted');
1526
+ }
1527
+ this.markCoverageDirty(job.source_id);
604
1528
  if (this.embedding) {
605
- this.enqueue(job.source_id, 'embed_source', null, 3);
1529
+ await this.enqueue(job.source_id, 'embed_source', null, 3);
606
1530
  this.kick();
607
1531
  }
608
1532
  }
609
1533
  else if (job.kind === 'fetch_doc' && job.path) {
610
- await this.fetchDocJob(job.source_id, job.path);
611
- this.refreshCoverage(job.source_id);
1534
+ this.heartbeatJob(job.id);
1535
+ await this.withTimeout((signal) => this.fetchDocJob(job.source_id, job.path, signal), this.parseTimeoutMs, `fetch_timeout:${job.path}`, this.abortSignalFor(job.source_id));
1536
+ this.bindJobFileId(job);
1537
+ if (this.isAborted(job.source_id)) {
1538
+ throw new Error('aborted');
1539
+ }
1540
+ this.markCoverageDirty(job.source_id);
612
1541
  if (this.embedding) {
613
- this.enqueue(job.source_id, 'embed_source', null, 3);
1542
+ await this.enqueue(job.source_id, 'embed_source', null, 3);
614
1543
  this.kick();
615
1544
  }
616
1545
  }
617
1546
  else if (job.kind === 'drop_file' && job.path) {
618
- this.index.removeFile(job.source_id, job.path);
619
- this.refreshCoverage(job.source_id);
1547
+ // 移出/删除:连同子树异步清(分批让出,禁止同步连环 DELETE 堵死 Engine)
1548
+ await this.index.removePathTreeAsync(job.source_id, job.path);
1549
+ this.markCoverageDirty(job.source_id);
620
1550
  }
621
1551
  else if (job.kind === 'walk_source') {
622
- await this.ingestSource(job.source_id);
1552
+ await this.ingestSource(job.source_id, { fromQueue: true });
1553
+ if (this.isAborted(job.source_id)) {
1554
+ throw new Error('aborted');
1555
+ }
623
1556
  }
624
1557
  else if (job.kind === 'embed_source') {
625
- await this.embedPending(job.source_id);
1558
+ if (!this.embedding) {
1559
+ const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
1560
+ if (stillMissing) {
1561
+ // 空跑?done:迁移后缺向量却显示 queue=0
1562
+ throw new Error('embedding_provider_missing');
1563
+ }
1564
+ return;
1565
+ }
1566
+ await this.embedPending(job.source_id, 50, job.id);
1567
+ if (this.isAborted(job.source_id)) {
1568
+ throw new Error('aborted');
1569
+ }
1570
+ }
1571
+ // 统一收尾闸门:任?kind 后都不得?done
1572
+ if (this.isAborted(job.source_id)) {
1573
+ throw new Error('aborted');
626
1574
  }
627
1575
  this.sources.database.raw
628
1576
  .prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
@@ -633,24 +1581,66 @@ export class KnowledgeIngest extends EventEmitter {
633
1581
  path: job.path ?? undefined,
634
1582
  status: 'done',
635
1583
  });
1584
+ // ?done 后再:否?enqueue 去重会把「自?active 而丢掉下?
1585
+ if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
1586
+ this.ensureEmbedJob(job.source_id);
1587
+ this.refreshCoverage(job.source_id, { force: true });
1588
+ }
636
1589
  }
637
1590
  catch (err) {
638
1591
  const message = err instanceof Error ? err.message : String(err);
1592
+ const aborted = message === 'aborted' || this.isAborted(job.source_id);
1593
+ // 归属拒绝不是「文件解析失不得写入 knowledge_files
1594
+ const ownership = message.includes('not owned by source');
1595
+ if (job.kind === 'parse_file' && job.path && !aborted && !ownership) {
1596
+ this.index.markFileError(job.source_id, job.path, message);
1597
+ }
639
1598
  this.sources.database.raw
640
1599
  .prepare(`UPDATE knowledge_jobs
641
- SET status = 'failed', attempts = attempts + 1, last_error = ?, updated_at = ?
1600
+ SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
642
1601
  WHERE id = ?`)
643
- .run(message, now, job.id);
1602
+ .run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, now, job.id);
1603
+ if (job.kind === 'embed_source' && !aborted) {
1604
+ this.ensureEmbedJob(job.source_id);
1605
+ }
1606
+ throw err;
1607
+ }
1608
+ }
1609
+ /**
1610
+ * 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
1611
+ * 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
1612
+ */
1613
+ async withTimeout(work, ms, label, parent) {
1614
+ const controller = new AbortController();
1615
+ const onParentAbort = () => controller.abort();
1616
+ parent?.addEventListener('abort', onParentAbort, { once: true });
1617
+ const timer = setTimeout(() => controller.abort(), ms);
1618
+ try {
1619
+ return await work(controller.signal);
1620
+ }
1621
+ catch (err) {
1622
+ if (controller.signal.aborted) {
1623
+ const msg = err instanceof Error ? err.message : String(err);
1624
+ if (msg === 'aborted' || msg.includes('abort'))
1625
+ throw new Error(label);
1626
+ throw new Error(label);
1627
+ }
644
1628
  throw err;
645
1629
  }
1630
+ finally {
1631
+ clearTimeout(timer);
1632
+ parent?.removeEventListener('abort', onParentAbort);
1633
+ }
646
1634
  }
647
1635
  /**
648
- * 外源单文档 job:按逻辑键 fetch + 索引
1636
+ * 外源单文?job:按逻辑?fetch + 索引
649
1637
  */
650
- async fetchDocJob(sourceId, path) {
1638
+ async fetchDocJob(sourceId, path, signal) {
651
1639
  const source = this.sources.get(sourceId);
652
1640
  if (!source || source.status === 'removed')
653
1641
  return;
1642
+ if (signal?.aborted)
1643
+ throw new Error('aborted');
654
1644
  const cred = await this.resolveCred(source);
655
1645
  const existing = this.index.getFile(sourceId, path);
656
1646
  const ref = {
@@ -666,21 +1656,21 @@ export class KnowledgeIngest extends EventEmitter {
666
1656
  : this.localFetcher;
667
1657
  const doc = await fetcher.fetch(source, ref, cred);
668
1658
  if (doc)
669
- this.indexVirtualDoc(sourceId, doc);
1659
+ await this.indexVirtualDoc(sourceId, doc, source.authRef);
670
1660
  }
671
- async parseOne(sourceId, filePath) {
1661
+ async parseOne(sourceId, filePath, signal) {
1662
+ const source = this.sources.get(sourceId);
1663
+ // 防御深度:即使队污染,也?root 外的文件
1664
+ if (source && !this.sourceOwnsPath(source, filePath)) {
1665
+ throw new Error(`path not owned by source: ${filePath}`);
1666
+ }
1667
+ if (signal?.aborted)
1668
+ throw new Error('aborted');
672
1669
  if (this.shouldSkipFile(filePath)) {
673
1670
  this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
674
1671
  return false;
675
1672
  }
676
- const adapter = this.adapters.match(filePath);
677
- if (!adapter) {
678
- if (this.documentPort && isDocumentPath(filePath)) {
679
- return this.parseDocumentFile(sourceId, filePath);
680
- }
681
- this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
682
- return false;
683
- }
1673
+ // 先确认是件:/节点不得?knowledge_files(历史上?no_adapter?
684
1674
  let st;
685
1675
  try {
686
1676
  st = await stat(filePath);
@@ -689,47 +1679,83 @@ export class KnowledgeIngest extends EventEmitter {
689
1679
  this.index.removeFile(sourceId, filePath);
690
1680
  return false;
691
1681
  }
692
- if (!st.isFile())
1682
+ if (!st.isFile()) {
1683
+ this.index.removeFile(sourceId, filePath);
693
1684
  return false;
694
- if (st.size > this.maxFileBytes) {
695
- this.index.markFileSkipped(sourceId, filePath, 'oversize');
1685
+ }
1686
+ const adapter = this.adapters.match(filePath);
1687
+ if (!adapter) {
1688
+ // worker DocumentPort;注?port 仅作 worker 不可用时的回
1689
+ if (isDocumentPath(filePath)) {
1690
+ return this.parseDocumentFile(sourceId, filePath, signal);
1691
+ }
1692
+ this.index.markFileSkipped(sourceId, filePath, 'no_adapter', st.size, await this.identityKeyFor(filePath));
696
1693
  return false;
697
1694
  }
698
- const content = await readFile(filePath, 'utf8');
699
- const contentHash = hashContent(content);
700
- if (this.index.isFresh(sourceId, filePath, contentHash)) {
701
- return true;
1695
+ // Axure 导出 data.js/document.js:内容嗅探后 skip,避免误伤普通 data.js
1696
+ if (/\.(js|cjs|mjs)$/i.test(filePath)) {
1697
+ const head = await this.readFileHead(filePath, 256);
1698
+ if (head && looksLikeAxureExportJs(head)) {
1699
+ this.index.markFileSkipped(sourceId, filePath, 'axure_export_sidecar', st.size, await this.identityKeyFor(filePath));
1700
+ return false;
1701
+ }
1702
+ }
1703
+ const kind = classifyFileKind(filePath);
1704
+ const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
1705
+ if (sizeDecision.action === 'skip') {
1706
+ this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size, await this.identityKeyFor(filePath));
1707
+ return false;
702
1708
  }
1709
+ // 读盘 + hash + 切块 + FTS token 全在 worker(Service 事件循环只做写库)
1710
+ const maxTextChars = this.fileLimits.partial.maxTextChars;
1711
+ if (signal?.aborted)
1712
+ throw new Error('aborted');
703
1713
  try {
704
- const chunks = adapter.chunk(content, filePath);
705
- this.index.upsertFile({
1714
+ const { parseTextInWorker } = await import('./parse-text-in-worker.js');
1715
+ const parsed = await parseTextInWorker(filePath, {
1716
+ sizeDecision: sizeDecision.action === 'partial' ? 'partial' : 'ok',
1717
+ maxTextChars,
1718
+ timeoutMs: parseTimeoutForSize(st.size, this.fileLimits),
1719
+ signal,
1720
+ });
1721
+ if (this.index.isFresh(sourceId, filePath, parsed.contentHash)) {
1722
+ return true;
1723
+ }
1724
+ if (signal?.aborted)
1725
+ throw new Error('aborted');
1726
+ const ident = await identifyLocalFile(filePath);
1727
+ await this.index.upsertFile({
706
1728
  sourceId,
707
1729
  path: filePath,
708
- contentHash,
1730
+ contentHash: parsed.contentHash,
709
1731
  size: st.size,
710
1732
  mtime: Math.floor(st.mtimeMs),
711
- adapterId: adapter.id,
712
- chunks,
713
- });
1733
+ adapterId: parsed.adapterId,
1734
+ chunks: parsed.chunks,
1735
+ identityKey: ident.key,
1736
+ }, { signal });
714
1737
  return true;
715
1738
  }
716
1739
  catch (err) {
717
- this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
1740
+ const msg = err instanceof Error ? err.message : String(err);
1741
+ if (msg === 'aborted' || msg.startsWith('text_parse_timeout') || signal?.aborted) {
1742
+ throw err;
1743
+ }
1744
+ this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
718
1745
  return false;
719
1746
  }
720
1747
  }
721
1748
  /**
722
- * PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
723
- * 新鲜度以原件字节 hash 为准。
1749
+ * PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
1750
+ * 新鲜度以原件字节 hash 为准?
724
1751
  *
725
- * @param sourceId - 知识源 id
726
- * @param filePath - 本地文件路径
727
- * @returns 是否成功入索引
1752
+ * @param sourceId - 知识?id
1753
+ * @param filePath - 文件
1754
+ * @returns 成功入索?
728
1755
  */
729
- async parseDocumentFile(sourceId, filePath) {
730
- const port = this.documentPort;
731
- if (!port)
732
- return false;
1756
+ async parseDocumentFile(sourceId, filePath, signal) {
1757
+ if (signal?.aborted)
1758
+ throw new Error('aborted');
733
1759
  let st;
734
1760
  try {
735
1761
  st = await stat(filePath);
@@ -740,96 +1766,167 @@ export class KnowledgeIngest extends EventEmitter {
740
1766
  }
741
1767
  if (!st.isFile())
742
1768
  return false;
743
- if (st.size > this.maxFileBytes) {
744
- this.index.markFileSkipped(sourceId, filePath, 'oversize');
1769
+ const kind = classifyFileKind(filePath);
1770
+ const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
1771
+ if (sizeDecision.action === 'skip') {
1772
+ this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size);
745
1773
  return false;
746
1774
  }
747
- const raw = await readFile(filePath);
748
- const contentHash = hashContent(raw);
749
- if (this.index.isFresh(sourceId, filePath, contentHash)) {
750
- return true;
751
- }
752
1775
  try {
753
- const result = await port.extract({ path: filePath, name: filePath });
754
- const markdown = result.markdown?.trim();
755
- if (!markdown) {
756
- this.index.markFileSkipped(sourceId, filePath, 'empty_content');
1776
+ // worker 抽取 + 流式 hash + 切块 + ftsToks:xlsx/pdf 不堵 Service 事件循环
1777
+ const extracted = await this.extractDocument(filePath, sourceId, st.size, signal);
1778
+ const contentHash = extracted.contentHash;
1779
+ if (this.index.isFresh(sourceId, filePath, contentHash)) {
1780
+ return true;
1781
+ }
1782
+ const markdown = extracted.result.markdown?.trim();
1783
+ if (!markdown || extracted.chunks.length === 0) {
1784
+ this.index.markFileSkipped(sourceId, filePath, 'empty_content', st.size);
757
1785
  return false;
758
1786
  }
759
- const chunks = markdownAdapter.chunk(markdown, filePath);
760
- this.index.upsertFile({
1787
+ if (signal?.aborted)
1788
+ throw new Error('aborted');
1789
+ const ident = await identifyLocalFile(filePath);
1790
+ await this.index.upsertFile({
761
1791
  sourceId,
762
1792
  path: filePath,
763
1793
  contentHash,
764
1794
  size: st.size,
765
1795
  mtime: Math.floor(st.mtimeMs),
766
- adapterId: `document:${result.backend}`,
767
- chunks,
768
- });
1796
+ adapterId: `document:${extracted.result.backend}`,
1797
+ chunks: extracted.chunks,
1798
+ identityKey: ident.key,
1799
+ }, { signal });
769
1800
  return true;
770
1801
  }
771
1802
  catch (err) {
1803
+ const msg = err instanceof Error ? err.message : String(err);
1804
+ // /超时不是「这件解析失必须上抛?job ?failed止静?done
1805
+ if (msg === 'aborted' || msg.startsWith('parse_timeout') || signal?.aborted) {
1806
+ throw err;
1807
+ }
772
1808
  if (isDocumentExtractError(err)) {
773
- // 结构化跳过(密码/老格式/超大等),不挡整库
774
- this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase());
1809
+ // 结构化跳过(密码/老格?超大等),不挡整?
1810
+ const identKey = await this.identityKeyFor(filePath);
1811
+ this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase(), st.size, identKey);
775
1812
  return false;
776
1813
  }
777
- this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
1814
+ this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
778
1815
  return false;
779
1816
  }
780
1817
  }
781
- /**
782
- * Phase B:嵌入待处理 chunk(批 + 限速;失败不丢,下次再补)
783
- */
784
- async embedPending(sourceId) {
785
- if (!this.embedding)
786
- return;
787
- const pending = this.index.listChunksMissingEmbedding(sourceId, this.embedBatch);
788
- if (pending.length === 0)
789
- return;
790
- if (this.embedMinIntervalMs > 0) {
791
- const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
792
- if (wait > 0)
793
- await new Promise((r) => setTimeout(r, wait));
794
- }
1818
+ /** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
1819
+ async extractDocument(filePath, sourceId, fileSize = 0, extraSignal) {
1820
+ const sourceSignal = this.abortSignalFor(sourceId);
1821
+ const controller = new AbortController();
1822
+ const abort = () => controller.abort();
1823
+ sourceSignal.addEventListener('abort', abort, { once: true });
1824
+ extraSignal?.addEventListener('abort', abort, { once: true });
1825
+ const signal = controller.signal;
1826
+ const timeoutMs = parseTimeoutForSize(fileSize, this.fileLimits);
1827
+ const extractOpts = {
1828
+ timeoutMs,
1829
+ // knowledge hardMax:软超限仍可 partial 抽取
1830
+ maxFileBytes: this.fileLimits.hardMaxFileBytes,
1831
+ maxSheets: this.fileLimits.partial.maxSheets,
1832
+ maxRowsPerSheet: this.fileLimits.partial.maxRowsPerSheet,
1833
+ maxPages: this.fileLimits.partial.maxPdfPages,
1834
+ maxTextChars: this.fileLimits.partial.maxTextChars,
1835
+ };
795
1836
  try {
796
- const vectors = await this.embedding.embedBatch(pending.map((p) => p.text));
797
- for (let i = 0; i < pending.length; i++) {
798
- const vec = vectors[i];
799
- if (vec?.length)
800
- this.index.setChunkEmbedding(pending[i].id, vec);
1837
+ // 注入的 documentPort 是调用方选定的抽取实现(测试 stub / 自定义后端)
1838
+ if (this.documentPort) {
1839
+ const result = await this.documentPort.extract({ path: filePath, name: filePath }, { ...extractOpts, signal });
1840
+ const markdown = result.markdown?.trim() ?? '';
1841
+ const chunks = markdown
1842
+ ? await this.chunkTextInWorker(markdown, filePath, signal)
1843
+ : [];
1844
+ const contentHash = await hashFileStreaming(filePath, signal);
1845
+ return { result, chunks, contentHash };
801
1846
  }
802
- this.lastEmbedAt = Date.now();
803
- this.emitProgress({
804
- type: 'embed',
805
- sourceId,
806
- status: 'batch_done',
807
- detail: `embedded=${pending.length}`,
1847
+ const { extractDocumentInWorker } = await import('./extract-document.js');
1848
+ return await extractDocumentInWorker(filePath, {
1849
+ ...extractOpts,
1850
+ documentConfig: this.documentConfig,
1851
+ signal,
808
1852
  });
809
1853
  }
810
1854
  catch (err) {
811
- this.emitProgress({
812
- type: 'error',
813
- sourceId,
814
- status: 'embed_failed',
815
- detail: err instanceof Error ? err.message : String(err),
816
- });
817
- return;
1855
+ const msg = err instanceof Error ? err.message : String(err);
1856
+ if (msg === 'extract_aborted' || signal.aborted) {
1857
+ throw new Error('aborted');
1858
+ }
1859
+ throw err;
818
1860
  }
819
- // 仍有 pending 则继续排队
820
- if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
821
- this.enqueue(sourceId, 'embed_source', null, 3);
822
- this.kick();
1861
+ finally {
1862
+ sourceSignal.removeEventListener('abort', abort);
1863
+ extraSignal?.removeEventListener('abort', abort);
823
1864
  }
824
- this.refreshCoverage(sourceId);
825
1865
  }
826
- refreshCoverage(sourceId) {
1866
+ /** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
1867
+ async chunkTextInWorker(text, path, signal, adapterId) {
1868
+ const { chunkTextWithFtsToks } = await import('./parse-text-in-worker.js');
1869
+ return chunkTextWithFtsToks(text, path, signal, adapterId);
1870
+ }
1871
+ /**
1872
+ * Phase B:嵌入待处理 chunk
1873
+ *
1874
+ * ?job 内按 embedConcurrency ****多批调用 API?
1875
+ * 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
1876
+ */
1877
+ async embedPending(sourceId, maxRounds = 20, jobId) {
1878
+ await this.embedRunner.embedPending(sourceId, maxRounds, jobId);
1879
+ }
1880
+ recordEmbedSecretHit(input) {
1881
+ this.sources.database.raw
1882
+ .prepare(`INSERT INTO knowledge_embed_secret_log (id, source_id, chunk_id, action, hits_json, created_at)
1883
+ VALUES (?, ?, ?, ?, ?, ?)`)
1884
+ .run(`kes_${randomUUID().slice(0, 12)}`, input.sourceId, input.chunkId, input.action, JSON.stringify(input.hits), Date.now());
1885
+ }
1886
+ /** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
1887
+ ensureEmbedJob(sourceId) {
1888
+ if (!this.embedding)
1889
+ return;
1890
+ if (this.isAborted(sourceId))
1891
+ return;
1892
+ if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
1893
+ return;
1894
+ void this.enqueue(sourceId, 'embed_source', null, 3)
1895
+ .then(() => this.kick())
1896
+ .catch(() => undefined);
1897
+ }
1898
+ heartbeatJob(jobId) {
1899
+ this.jobQueue.heartbeat(jobId);
1900
+ }
1901
+ /** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
1902
+ coverageDirty = new Set();
1903
+ markCoverageDirty(sourceId) {
1904
+ this.coverageDirty.add(sourceId);
1905
+ }
1906
+ flushCoverageDirty(sourceId) {
1907
+ const ids = sourceId ? [sourceId] : [...this.coverageDirty];
1908
+ if (sourceId)
1909
+ this.coverageDirty.delete(sourceId);
1910
+ else
1911
+ this.coverageDirty.clear();
1912
+ for (const id of ids)
1913
+ this.refreshCoverage(id, { force: true });
1914
+ }
1915
+ lastCoverageRefreshAt = new Map();
1916
+ refreshCoverage(sourceId, opts) {
1917
+ const now = Date.now();
1918
+ const last = this.lastCoverageRefreshAt.get(sourceId) ?? 0;
1919
+ // 大库 sourceStats/embeddingCoverage 是多表 COUNT;解析/嵌入高频调用时节流,
1920
+ // 避免 Engine 线程被统计查询堵住导致 HTTP list 超时。收尾路径 force。
1921
+ if (!opts?.force && now - last < 2_000)
1922
+ return;
1923
+ this.lastCoverageRefreshAt.set(sourceId, now);
827
1924
  const stats = this.index.sourceStats(sourceId);
828
1925
  const source = this.sources.get(sourceId);
829
1926
  if (!source)
830
1927
  return;
831
1928
  const discovered = this.discoveredFiles.get(sourceId) ?? 0;
832
- // Phase A:已落库文件(indexed+skipped+error)/ walk 发现数
1929
+ // Phase A:已落库文件(indexed+skipped+error? walk 发现?
833
1930
  const phaseA = discovered > 0
834
1931
  ? Math.min(1, stats.files / discovered)
835
1932
  : stats.files > 0
@@ -865,10 +1962,29 @@ export class KnowledgeIngest extends EventEmitter {
865
1962
  });
866
1963
  }
867
1964
  emitProgress(evt) {
868
- this.emit('progress', evt);
869
- this.emit('knowledge.index.progress', evt);
1965
+ let out = evt;
1966
+ if (evt.sourceId && evt.registeredBy === undefined) {
1967
+ try {
1968
+ const row = this.sources.database.raw
1969
+ .prepare('SELECT tenant_id, registered_by, visibility FROM knowledge_sources WHERE id = ?')
1970
+ .get(evt.sourceId);
1971
+ if (row) {
1972
+ out = {
1973
+ ...evt,
1974
+ tenantId: row.tenant_id,
1975
+ registeredBy: row.registered_by,
1976
+ visibility: row.visibility,
1977
+ };
1978
+ }
1979
+ }
1980
+ catch {
1981
+ /* 源已删时保持原事件;SSE 侧按无归属放行到 fallback */
1982
+ }
1983
+ }
1984
+ this.emit('progress', out);
1985
+ this.emit('knowledge.index.progress', out);
870
1986
  }
871
- /** 磁盘水位告警(仅告警,不丢任务) */
1987
+ /** 磁盘水位(仅,不务) */
872
1988
  async warnDiskWatermark(sourceId) {
873
1989
  try {
874
1990
  const { statfs } = await import('node:fs/promises');
@@ -881,7 +1997,7 @@ export class KnowledgeIngest extends EventEmitter {
881
1997
  type: 'error',
882
1998
  sourceId,
883
1999
  status: 'disk_watermark',
884
- detail: `free≈${Math.round(freeMb)}MB (jobs kept)`,
2000
+ detail: `free?{Math.round(freeMb)}MB (jobs kept)`,
885
2001
  });
886
2002
  }
887
2003
  }