@octopi-agent/engine 0.58.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +38 -1
- package/dist/config-schema/substrate.d.ts +37 -0
- package/dist/config-schema/substrate.js +47 -0
- package/dist/config.d.ts +30 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/index.d.ts +4 -2
- package/dist/harness/index.js +2 -1
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +80 -17
- package/dist/harness/knowledge/source-store.js +160 -89
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -1,30 +1,71 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
|
|
5
5
|
*/
|
|
6
|
-
import { randomUUID } from 'node:crypto';
|
|
7
|
-
import {
|
|
6
|
+
import { randomUUID, createHash } from 'node:crypto';
|
|
7
|
+
import { open, stat } from 'node:fs/promises';
|
|
8
|
+
import { createReadStream, statSync, realpathSync } from 'node:fs';
|
|
8
9
|
import { watch } from 'node:fs';
|
|
9
|
-
import { join, sep } from 'node:path';
|
|
10
|
+
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
10
11
|
import { EventEmitter } from 'node:events';
|
|
11
|
-
import { FormatAdapterRegistry, htmlAdapter, markdownAdapter, textAdapter } from './adapters.js';
|
|
12
|
+
import { FormatAdapterRegistry, htmlAdapter, looksLikeAxureExportJs, markdownAdapter, textAdapter, } from './adapters.js';
|
|
12
13
|
import { looksLikeHtml } from './html.js';
|
|
13
14
|
import { hashContent, KnowledgeIndexStore } from './index-store.js';
|
|
14
|
-
import {
|
|
15
|
+
import { identifyLocalFile, urlIdentityKey, normalizePathLexical, } from './file-identity.js';
|
|
16
|
+
import { KnowledgeJobControl } from './job-control.js';
|
|
17
|
+
import { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
|
|
15
18
|
import { ConnectorRegistry } from './connectors.js';
|
|
16
19
|
import { ConnectorFetcher } from './connector-fetcher.js';
|
|
17
|
-
import { isDocumentPath } from '../
|
|
18
|
-
import { isDocumentExtractError } from '../
|
|
20
|
+
import { isDocumentPath } from '../capabilities/document/format.js';
|
|
21
|
+
import { isDocumentExtractError } from '../capabilities/document/errors.js';
|
|
22
|
+
import { classifyFileKind, decideBySize, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
|
|
23
|
+
import { EmbedRunner } from './embed-runner.js';
|
|
24
|
+
/**
|
|
25
|
+
* 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
|
|
26
|
+
*
|
|
27
|
+
* @param filePath - 本地文件
|
|
28
|
+
* @param signal - 中止信号
|
|
29
|
+
* @returns hex digest
|
|
30
|
+
*/
|
|
31
|
+
export async function hashFileStreaming(filePath, signal) {
|
|
32
|
+
const hash = createHash('sha256');
|
|
33
|
+
await new Promise((resolve, reject) => {
|
|
34
|
+
if (signal?.aborted) {
|
|
35
|
+
reject(new Error('aborted'));
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const stream = createReadStream(filePath);
|
|
39
|
+
const onAbort = () => {
|
|
40
|
+
stream.destroy();
|
|
41
|
+
reject(new Error('aborted'));
|
|
42
|
+
};
|
|
43
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
44
|
+
stream.on('data', (chunk) => hash.update(chunk));
|
|
45
|
+
stream.once('end', () => {
|
|
46
|
+
signal?.removeEventListener('abort', onAbort);
|
|
47
|
+
resolve();
|
|
48
|
+
});
|
|
49
|
+
stream.once('error', (err) => {
|
|
50
|
+
signal?.removeEventListener('abort', onAbort);
|
|
51
|
+
reject(err);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
return hash.digest('hex');
|
|
55
|
+
}
|
|
56
|
+
import { JobQueue } from './job-queue.js';
|
|
57
|
+
import { readJobControlState } from './job-control-state.js';
|
|
19
58
|
export class KnowledgeIngest extends EventEmitter {
|
|
20
59
|
sources;
|
|
21
60
|
index;
|
|
22
61
|
adapters;
|
|
23
62
|
parseConcurrency;
|
|
63
|
+
documentParseConcurrency;
|
|
24
64
|
maxQueueDepth;
|
|
25
65
|
debounceMs;
|
|
26
66
|
maxFileBytes;
|
|
27
|
-
|
|
67
|
+
fileLimits;
|
|
68
|
+
/** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
|
|
28
69
|
parseTimeoutMs;
|
|
29
70
|
embedding;
|
|
30
71
|
embedBatch;
|
|
@@ -39,22 +80,29 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
39
80
|
pollMinIntervalMs;
|
|
40
81
|
maxPollPerTick;
|
|
41
82
|
documentPort;
|
|
42
|
-
|
|
83
|
+
documentConfig;
|
|
84
|
+
embedSecretPolicy;
|
|
85
|
+
embedRunner;
|
|
43
86
|
pollTimer = null;
|
|
44
87
|
reconcileTimer = null;
|
|
45
|
-
/**
|
|
46
|
-
|
|
47
|
-
|
|
88
|
+
/** 源级:DB 权威 + ?AbortSignal(跨重启?*/
|
|
89
|
+
jobControl;
|
|
90
|
+
jobQueue;
|
|
91
|
+
/** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
|
|
48
92
|
onSourceSettled;
|
|
49
|
-
/**
|
|
93
|
+
/** parse 缺口时间 */
|
|
50
94
|
lastParseGapScanAt = new Map();
|
|
95
|
+
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
96
|
+
gapScanUrgent = new Set();
|
|
97
|
+
gapScanTimer = null;
|
|
51
98
|
runningParse = 0;
|
|
99
|
+
runningDocParse = 0;
|
|
52
100
|
runningEmbed = 0;
|
|
53
101
|
draining = false;
|
|
54
102
|
kickAgain = false;
|
|
55
|
-
/** source
|
|
103
|
+
/** source 级锁:同 source 不并?walk/parse 冲突?*/
|
|
56
104
|
sourceLocks = new Set();
|
|
57
|
-
/** walk 发现的文件数(Phase A coverage
|
|
105
|
+
/** walk 发现的文件数(Phase A coverage 分母?*/
|
|
58
106
|
discoveredFiles = new Map();
|
|
59
107
|
watchers = new Map();
|
|
60
108
|
debounceTimers = new Map();
|
|
@@ -62,22 +110,68 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
62
110
|
constructor(options) {
|
|
63
111
|
super();
|
|
64
112
|
this.sources = options.sourceStore;
|
|
113
|
+
this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
|
|
114
|
+
this.jobControl = new KnowledgeJobControl(options.sourceStore.database);
|
|
115
|
+
this.jobQueue = new JobQueue({
|
|
116
|
+
db: options.sourceStore.database,
|
|
117
|
+
maxQueueDepth: this.maxQueueDepth,
|
|
118
|
+
isAborted: (id) => this.isAborted(id),
|
|
119
|
+
onBackpressure: (sourceId, queued) => {
|
|
120
|
+
this.emitProgress({
|
|
121
|
+
type: 'error',
|
|
122
|
+
sourceId,
|
|
123
|
+
status: 'queue_backpressure',
|
|
124
|
+
detail: `queued=${queued} (delay only, jobs kept)`,
|
|
125
|
+
});
|
|
126
|
+
},
|
|
127
|
+
});
|
|
65
128
|
this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
|
|
66
129
|
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
67
130
|
this.parseConcurrency = options.parseConcurrency ?? 8;
|
|
68
|
-
this.
|
|
69
|
-
// watch
|
|
131
|
+
this.documentParseConcurrency = options.documentParseConcurrency ?? 2;
|
|
132
|
+
// watch 批量落盘时缩默窗,新增文件更?
|
|
70
133
|
this.debounceMs = options.debounceMs ?? 800;
|
|
71
|
-
this.
|
|
72
|
-
|
|
134
|
+
this.fileLimits = resolveKnowledgeFileLimits({
|
|
135
|
+
...options.fileLimits,
|
|
136
|
+
maxFileBytes: options.fileLimits?.maxFileBytes ?? options.maxFileBytes,
|
|
137
|
+
parseTimeoutMs: options.fileLimits?.parseTimeoutMs ?? options.parseTimeoutMs,
|
|
138
|
+
});
|
|
139
|
+
this.maxFileBytes = this.fileLimits.maxFileBytes;
|
|
140
|
+
this.parseTimeoutMs = this.fileLimits.parseTimeoutMs;
|
|
73
141
|
this.embedding = options.embeddingProvider ?? null;
|
|
74
142
|
this.embedBatch = options.embedBatch ?? 32;
|
|
75
143
|
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
76
|
-
// embed
|
|
144
|
+
// embed ?parse 分槽;缺?4 行打 embedding API
|
|
77
145
|
this.embedConcurrency = options.embedConcurrency ?? 4;
|
|
78
146
|
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
79
147
|
this.credentials = options.credentials ?? null;
|
|
80
148
|
this.documentPort = options.documentPort ?? null;
|
|
149
|
+
this.documentConfig = options.documentConfig ?? null;
|
|
150
|
+
this.embedSecretPolicy = options.embedSecretPolicy ?? 'redact';
|
|
151
|
+
this.embedRunner = new EmbedRunner({
|
|
152
|
+
index: this.index,
|
|
153
|
+
embedding: this.embedding,
|
|
154
|
+
embedBatch: this.embedBatch,
|
|
155
|
+
embedConcurrency: this.embedConcurrency,
|
|
156
|
+
embedMinIntervalMs: this.embedMinIntervalMs,
|
|
157
|
+
embedSecretPolicy: this.embedSecretPolicy,
|
|
158
|
+
isAborted: (id) => this.isAborted(id),
|
|
159
|
+
heartbeatJob: (id) => this.heartbeatJob(id),
|
|
160
|
+
refreshCoverage: (id) => this.refreshCoverage(id),
|
|
161
|
+
emitProgress: (evt) => this.emitProgress(evt),
|
|
162
|
+
recordSecretHit: (input) => this.recordEmbedSecretHit(input),
|
|
163
|
+
});
|
|
164
|
+
// embedding 外发(合规可查;非权量)
|
|
165
|
+
this.sources.database.raw.exec(`
|
|
166
|
+
CREATE TABLE IF NOT EXISTS knowledge_embed_secret_log (
|
|
167
|
+
id TEXT PRIMARY KEY,
|
|
168
|
+
source_id TEXT NOT NULL,
|
|
169
|
+
chunk_id TEXT NOT NULL,
|
|
170
|
+
action TEXT NOT NULL,
|
|
171
|
+
hits_json TEXT NOT NULL,
|
|
172
|
+
created_at INTEGER NOT NULL
|
|
173
|
+
);
|
|
174
|
+
`);
|
|
81
175
|
this.localFetcher =
|
|
82
176
|
options.fetchers?.local ?? new LocalFsFetcher((p) => this.shouldSkipFile(p));
|
|
83
177
|
this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
|
|
@@ -87,38 +181,157 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
87
181
|
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
88
182
|
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
89
183
|
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
90
|
-
//
|
|
184
|
+
// 崩溃/重启遗留?running 会堵?enqueue 去重并永久显?indexing
|
|
91
185
|
this.reclaimOrphanRunningJobs();
|
|
186
|
+
// 历史:目录等非文件曾?no_adapter,启动即?
|
|
187
|
+
this.cleanupNonFileIndexRows();
|
|
188
|
+
}
|
|
189
|
+
lastCleanupAt = 0;
|
|
190
|
+
/**
|
|
191
|
+
* 清掉「非?knowledge_files (目录曾 no_adapter?
|
|
192
|
+
*
|
|
193
|
+
* ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
|
|
194
|
+
* 60s 节流,避免每?reconcile ?stat?
|
|
195
|
+
*
|
|
196
|
+
* @param sourceId - 限定源;省略则全部源
|
|
197
|
+
* @param opts.force - 忽略节流
|
|
198
|
+
* @returns 删除行数
|
|
199
|
+
*/
|
|
200
|
+
cleanupNonFileIndexRows(sourceId, opts) {
|
|
201
|
+
const now = Date.now();
|
|
202
|
+
if (!opts?.force && sourceId == null && now - this.lastCleanupAt < 60_000) {
|
|
203
|
+
return 0;
|
|
204
|
+
}
|
|
205
|
+
this.lastCleanupAt = now;
|
|
206
|
+
const ids = sourceId
|
|
207
|
+
? [sourceId]
|
|
208
|
+
: this.sources.list().map((s) => s.id);
|
|
209
|
+
let removed = 0;
|
|
210
|
+
for (const id of ids) {
|
|
211
|
+
const rows = this.sources.database.raw
|
|
212
|
+
.prepare(`SELECT m.logical_path AS path
|
|
213
|
+
FROM knowledge_memberships m
|
|
214
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
215
|
+
WHERE m.source_id = ? AND f.status IN ('skipped', 'error') AND f.chunk_count = 0`)
|
|
216
|
+
.all(id);
|
|
217
|
+
for (const r of rows) {
|
|
218
|
+
try {
|
|
219
|
+
const st = statSync(r.path);
|
|
220
|
+
if (!st.isFile()) {
|
|
221
|
+
this.index.removeFile(id, r.path);
|
|
222
|
+
removed += 1;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
// 不存留给 pruneMissing齐),避删有效但暂时不可见的文件
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return removed;
|
|
92
231
|
}
|
|
93
232
|
/**
|
|
94
|
-
* 无跨进程 Lease
|
|
95
|
-
*
|
|
233
|
+
* 无跨进程 Lease:本进程时把历史 running 收回 queued?
|
|
234
|
+
* 仅构造时调用行中同进程的 running 法的?
|
|
96
235
|
*/
|
|
97
236
|
reclaimOrphanRunningJobs() {
|
|
98
|
-
this.
|
|
99
|
-
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ? WHERE status = 'running'`)
|
|
100
|
-
.run(Date.now());
|
|
237
|
+
this.jobQueue.reclaimOrphanRunning();
|
|
101
238
|
}
|
|
102
239
|
get indexStore() {
|
|
103
240
|
return this.index;
|
|
104
241
|
}
|
|
105
242
|
/**
|
|
106
|
-
*
|
|
243
|
+
* 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
|
|
107
244
|
*
|
|
108
|
-
* @param p -
|
|
109
|
-
* @returns
|
|
245
|
+
* @param p - 文件
|
|
246
|
+
* @returns 跳过
|
|
110
247
|
*/
|
|
111
248
|
shouldSkipFile(p) {
|
|
112
|
-
if (
|
|
249
|
+
if (isDocumentPath(p))
|
|
113
250
|
return false;
|
|
114
251
|
return this.adapters.shouldSkipPath(p);
|
|
115
252
|
}
|
|
253
|
+
/** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
|
|
254
|
+
async readFileHead(filePath, bytes = 256) {
|
|
255
|
+
try {
|
|
256
|
+
const fh = await open(filePath, 'r');
|
|
257
|
+
try {
|
|
258
|
+
const buf = Buffer.alloc(bytes);
|
|
259
|
+
const { bytesRead } = await fh.read(buf, 0, bytes, 0);
|
|
260
|
+
return buf.subarray(0, bytesRead).toString('utf8');
|
|
261
|
+
}
|
|
262
|
+
finally {
|
|
263
|
+
await fh.close();
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
catch {
|
|
267
|
+
return null;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
|
|
271
|
+
async identityKeyFor(filePath) {
|
|
272
|
+
try {
|
|
273
|
+
return (await identifyLocalFile(filePath)).key;
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
return undefined;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
116
279
|
/**
|
|
117
|
-
*
|
|
280
|
+
* 归属:`parse_file` / 读文件只读源 root 内的?
|
|
281
|
+
* ?**realpath** 比较,防符号链接逃出 root?
|
|
282
|
+
*
|
|
283
|
+
* @param source - 知识?
|
|
284
|
+
* @param path - 请求(本地绝对路径,
|
|
285
|
+
*/
|
|
286
|
+
sourceOwnsPath(source, path) {
|
|
287
|
+
if (source.kind === 'url' || source.kind === 'connector') {
|
|
288
|
+
return Boolean(this.index.getFile(source.id, path));
|
|
289
|
+
}
|
|
290
|
+
const root = resolve(source.location);
|
|
291
|
+
const target = resolve(path);
|
|
292
|
+
const eq = (a, b) => process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b;
|
|
293
|
+
// 词法(路径不存在时也拒绝明显越界?
|
|
294
|
+
if (!eq(root, target)) {
|
|
295
|
+
const relLex = relative(root, target);
|
|
296
|
+
if (!relLex || relLex.startsWith('..') || isAbsolute(relLex))
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
// realpath ?symlink stat 失败回词法结果)
|
|
300
|
+
try {
|
|
301
|
+
const rootReal = realpathSync(root);
|
|
302
|
+
const targetReal = realpathSync(target);
|
|
303
|
+
if (eq(rootReal, targetReal))
|
|
304
|
+
return true;
|
|
305
|
+
const rel = relative(rootReal, targetReal);
|
|
306
|
+
return Boolean(rel) && !rel.startsWith('..') && !isAbsolute(rel);
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
return true; // 词法;文件尚在时交给后续 stat
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* 全量/增量索引某源(Phase A?
|
|
314
|
+
*
|
|
315
|
+
* @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
|
|
316
|
+
* @param opts.incremental - 增量(poll):?active 任务则跳过本?
|
|
118
317
|
*/
|
|
119
318
|
async ingestSource(sourceId, opts) {
|
|
120
|
-
//
|
|
121
|
-
|
|
319
|
+
// fromQueue(walk_source):不 supersede、不跳过 active;reindex 才作废队列
|
|
320
|
+
if (opts?.fromQueue) {
|
|
321
|
+
// fall through to walk
|
|
322
|
+
}
|
|
323
|
+
else if (opts?.full !== false && !opts?.incremental) {
|
|
324
|
+
this.supersedeSourceWork(sourceId);
|
|
325
|
+
}
|
|
326
|
+
else if (this.countActiveJobs(sourceId).total > 0) {
|
|
327
|
+
this.emitProgress({
|
|
328
|
+
type: 'source',
|
|
329
|
+
sourceId,
|
|
330
|
+
status: 'poll_skipped',
|
|
331
|
+
detail: 'active jobs in flight',
|
|
332
|
+
});
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
122
335
|
const source = this.sources.get(sourceId);
|
|
123
336
|
if (!source || source.status === 'removed') {
|
|
124
337
|
throw new Error(`knowledge source not found: ${sourceId}`);
|
|
@@ -140,24 +353,34 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
140
353
|
return;
|
|
141
354
|
}
|
|
142
355
|
if (this.sourceLocks.has(sourceId)) {
|
|
143
|
-
//
|
|
144
|
-
this.enqueue(sourceId, 'walk_source', null, 1
|
|
356
|
+
// 已有同源任务在跑:入?walk 即可(去重)
|
|
357
|
+
await this.enqueue(sourceId, 'walk_source', null, 1);
|
|
145
358
|
return;
|
|
146
359
|
}
|
|
147
360
|
this.sourceLocks.add(sourceId);
|
|
148
361
|
try {
|
|
149
|
-
|
|
362
|
+
// 保留旧 coverage:重建期间索引仍在库中,写 0 会让 UI 误判为清空
|
|
363
|
+
this.sources.update(sourceId, { status: 'discovering' });
|
|
150
364
|
this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
|
|
151
|
-
// full 不再 clearSource
|
|
365
|
+
// full 不再 clearSource:靠 keepPaths prune,discover 失败不丢?
|
|
152
366
|
if (source.kind === 'url' || source.kind === 'connector') {
|
|
153
367
|
await this.ingestRemoteSource(source);
|
|
154
368
|
return;
|
|
155
369
|
}
|
|
156
|
-
|
|
370
|
+
// watch 策略索引同时挂上 fs 监听(幂等;否则后续变更静默丢失?
|
|
371
|
+
this.startWatch(sourceId);
|
|
372
|
+
const localDiscovery = await this.localFetcher.discover(source);
|
|
373
|
+
const files = localDiscovery.docs;
|
|
157
374
|
this.discoveredFiles.set(sourceId, files.length);
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
|
|
375
|
+
// 后不 prune、不入队(半?keep 删)
|
|
376
|
+
if (this.isAborted(sourceId))
|
|
377
|
+
return;
|
|
378
|
+
// prune:仅 walk 完整时执行;出错/?keep 不完整会
|
|
379
|
+
if (localDiscovery.complete) {
|
|
380
|
+
const localKeep = new Set(files.map((f) => f.path));
|
|
381
|
+
// walk 成功且目录为空时允许清空 Membership(§6.2)
|
|
382
|
+
await this.index.pruneMissingAsync(sourceId, localKeep, { allowEmptyKeep: true });
|
|
383
|
+
}
|
|
161
384
|
this.sources.update(sourceId, { status: 'partial', coverage: 0 });
|
|
162
385
|
this.emitProgress({
|
|
163
386
|
type: 'source',
|
|
@@ -166,10 +389,14 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
166
389
|
detail: `${files.length} files`,
|
|
167
390
|
});
|
|
168
391
|
for (const file of files) {
|
|
169
|
-
this.
|
|
392
|
+
if (this.isAborted(sourceId))
|
|
393
|
+
return;
|
|
394
|
+
if (this.isUnchangedIndexed(sourceId, file))
|
|
395
|
+
continue;
|
|
396
|
+
await this.enqueue(sourceId, 'parse_file', file.path, 2);
|
|
170
397
|
}
|
|
171
|
-
if (this.embedding) {
|
|
172
|
-
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
398
|
+
if (this.embedding && !this.isAborted(sourceId)) {
|
|
399
|
+
await this.enqueue(sourceId, 'embed_source', null, 3);
|
|
173
400
|
}
|
|
174
401
|
this.kick();
|
|
175
402
|
}
|
|
@@ -177,6 +404,25 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
177
404
|
this.sourceLocks.delete(sourceId);
|
|
178
405
|
}
|
|
179
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
|
|
409
|
+
* 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
|
|
410
|
+
* indexing/error/半成品必须重做,避免把未完成当成新鲜。
|
|
411
|
+
*/
|
|
412
|
+
isUnchangedIndexed(sourceId, ref) {
|
|
413
|
+
if (ref.size == null || ref.mtime == null)
|
|
414
|
+
return false;
|
|
415
|
+
const existing = this.index.getFile(sourceId, ref.path);
|
|
416
|
+
if (!existing)
|
|
417
|
+
return false;
|
|
418
|
+
if (existing.status !== 'indexed')
|
|
419
|
+
return false;
|
|
420
|
+
if (existing.chunkCount <= 0)
|
|
421
|
+
return false;
|
|
422
|
+
if (!existing.contentHash)
|
|
423
|
+
return false;
|
|
424
|
+
return existing.size === ref.size && existing.mtime === ref.mtime;
|
|
425
|
+
}
|
|
180
426
|
/**
|
|
181
427
|
* URL / connector 源:discover + fetch + 索引
|
|
182
428
|
*/
|
|
@@ -196,9 +442,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
196
442
|
return;
|
|
197
443
|
}
|
|
198
444
|
const fetcher = source.kind === 'connector' ? this.connectorFetcher : this.urlFetcher;
|
|
199
|
-
let
|
|
445
|
+
let discovered;
|
|
200
446
|
try {
|
|
201
|
-
|
|
447
|
+
discovered = await fetcher.discover(source, cred);
|
|
202
448
|
}
|
|
203
449
|
catch (err) {
|
|
204
450
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -209,7 +455,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
209
455
|
this.emitProgress({ type: 'error', sourceId: source.id, status: 'discover_failed', detail: msg });
|
|
210
456
|
return;
|
|
211
457
|
}
|
|
212
|
-
|
|
458
|
+
const refs = discovered.docs;
|
|
459
|
+
// 条件 GET:带上已?etag/Last-Modified?04 则整文档跳过
|
|
213
460
|
const enriched = refs.map((ref) => {
|
|
214
461
|
const existing = this.index.getFile(source.id, ref.path);
|
|
215
462
|
return {
|
|
@@ -229,15 +476,25 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
229
476
|
let okCount = 0;
|
|
230
477
|
let unchanged = 0;
|
|
231
478
|
for (const ref of enriched) {
|
|
479
|
+
// 后立刻停:不再写索引
|
|
480
|
+
if (this.isAborted(source.id)) {
|
|
481
|
+
this.emitProgress({
|
|
482
|
+
type: 'source',
|
|
483
|
+
sourceId: source.id,
|
|
484
|
+
status: 'aborted',
|
|
485
|
+
detail: `stopped after ${okCount + unchanged}/${enriched.length}`,
|
|
486
|
+
});
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
232
489
|
try {
|
|
233
490
|
const doc = await fetcher.fetch(source, ref, cred);
|
|
234
491
|
if (!doc) {
|
|
235
|
-
// 304 /
|
|
492
|
+
// 304 / :保持原索引
|
|
236
493
|
unchanged += 1;
|
|
237
494
|
okCount += 1;
|
|
238
495
|
continue;
|
|
239
496
|
}
|
|
240
|
-
if (this.indexVirtualDoc(source.id, doc)) {
|
|
497
|
+
if (await this.indexVirtualDoc(source.id, doc, source.authRef)) {
|
|
241
498
|
okCount += 1;
|
|
242
499
|
}
|
|
243
500
|
}
|
|
@@ -252,22 +509,25 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
252
509
|
});
|
|
253
510
|
}
|
|
254
511
|
}
|
|
255
|
-
//
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
this.
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
512
|
+
// prune:仅?discover **完整** 时才;截?walk 失败?keep 不完整,prune ?
|
|
513
|
+
// 后不?prune:半?keep 删已入库文档
|
|
514
|
+
if (!this.isAborted(source.id) && discovered.complete) {
|
|
515
|
+
const keep = new Set(enriched.map((r) => r.path));
|
|
516
|
+
const pruned = await this.index.pruneMissingAsync(source.id, keep, { allowEmptyKeep: true });
|
|
517
|
+
if (pruned > 0) {
|
|
518
|
+
this.emitProgress({
|
|
519
|
+
type: 'source',
|
|
520
|
+
sourceId: source.id,
|
|
521
|
+
status: 'pruned',
|
|
522
|
+
detail: `removed=${pruned}`,
|
|
523
|
+
});
|
|
524
|
+
}
|
|
265
525
|
}
|
|
266
|
-
if (this.embedding) {
|
|
267
|
-
this.enqueue(source.id, 'embed_source', null, 3);
|
|
526
|
+
if (this.embedding && !this.isAborted(source.id)) {
|
|
527
|
+
await this.enqueue(source.id, 'embed_source', null, 3);
|
|
268
528
|
this.kick();
|
|
269
529
|
}
|
|
270
|
-
this.refreshCoverage(source.id);
|
|
530
|
+
this.refreshCoverage(source.id, { force: true });
|
|
271
531
|
this.emitProgress({
|
|
272
532
|
type: 'source',
|
|
273
533
|
sourceId: source.id,
|
|
@@ -276,10 +536,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
276
536
|
});
|
|
277
537
|
}
|
|
278
538
|
/**
|
|
279
|
-
*
|
|
539
|
+
* 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
|
|
280
540
|
*
|
|
281
|
-
* @returns
|
|
282
|
-
* @throws authRef
|
|
541
|
+
* @returns ?null(仅当源?authRef?
|
|
542
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方
|
|
283
543
|
*/
|
|
284
544
|
async resolveCred(source) {
|
|
285
545
|
if (!source.authRef)
|
|
@@ -294,16 +554,16 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
294
554
|
return resolved;
|
|
295
555
|
}
|
|
296
556
|
/**
|
|
297
|
-
*
|
|
557
|
+
* ?VirtualDocument 写入索引?/ 规范化后文本?
|
|
298
558
|
*
|
|
299
|
-
* @returns
|
|
559
|
+
* @returns 成功写入
|
|
300
560
|
*/
|
|
301
|
-
indexVirtualDoc(sourceId, doc) {
|
|
561
|
+
async indexVirtualDoc(sourceId, doc, authRef) {
|
|
302
562
|
if (!doc.content?.trim()) {
|
|
303
563
|
this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
|
|
304
564
|
return false;
|
|
305
565
|
}
|
|
306
|
-
//
|
|
566
|
+
// 逻辑能无扩展名(connector id / URL path):按 MIME / 内容启发式
|
|
307
567
|
let adapter = this.adapters.match(doc.path, doc.contentType);
|
|
308
568
|
if (!adapter) {
|
|
309
569
|
if (looksLikeHtml(doc.content, doc.contentType))
|
|
@@ -318,8 +578,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
318
578
|
return true;
|
|
319
579
|
}
|
|
320
580
|
try {
|
|
321
|
-
const chunks =
|
|
322
|
-
|
|
581
|
+
const chunks = await this.chunkTextInWorker(doc.content, doc.path, undefined, adapter.id);
|
|
582
|
+
let identityKey;
|
|
583
|
+
if (doc.externalUrl) {
|
|
584
|
+
identityKey = urlIdentityKey(doc.externalUrl, authRef);
|
|
585
|
+
}
|
|
586
|
+
else {
|
|
587
|
+
try {
|
|
588
|
+
identityKey = `path:${normalizePathLexical(doc.path)}`;
|
|
589
|
+
}
|
|
590
|
+
catch {
|
|
591
|
+
identityKey = `path:${doc.path.replace(/\\/g, '/')}`;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
await this.index.upsertFile({
|
|
323
595
|
sourceId,
|
|
324
596
|
path: doc.path,
|
|
325
597
|
contentHash,
|
|
@@ -327,6 +599,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
327
599
|
mtime: Math.floor(Date.now() / 1000),
|
|
328
600
|
adapterId: adapter.id,
|
|
329
601
|
chunks,
|
|
602
|
+
identityKey,
|
|
330
603
|
externalUrl: doc.externalUrl,
|
|
331
604
|
etag: doc.etag,
|
|
332
605
|
lastModified: doc.lastModified,
|
|
@@ -339,13 +612,129 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
339
612
|
}
|
|
340
613
|
}
|
|
341
614
|
/**
|
|
342
|
-
*
|
|
615
|
+
* 单文(P0 道;完成 parse,不 await 全库?
|
|
343
616
|
*/
|
|
344
617
|
async ingestFileNow(sourceId, filePath) {
|
|
345
618
|
const ok = await this.parseOne(sourceId, filePath);
|
|
346
619
|
this.refreshCoverage(sourceId);
|
|
347
620
|
return ok;
|
|
348
621
|
}
|
|
622
|
+
/**
|
|
623
|
+
* 重做指定文件:强制重?解析 ?分块 ?向量
|
|
624
|
+
*
|
|
625
|
+
* - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
|
|
626
|
+
* - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
|
|
627
|
+
* - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
|
|
628
|
+
* - 等非文件:直接清,不入队
|
|
629
|
+
* - 完成后走既有 embed 链路
|
|
630
|
+
*
|
|
631
|
+
* @param sourceId - ?id
|
|
632
|
+
* @param paths - 文件列表
|
|
633
|
+
* @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
|
|
634
|
+
*/
|
|
635
|
+
async reprocessFiles(sourceId, paths) {
|
|
636
|
+
if (!paths.length) {
|
|
637
|
+
return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: 0 };
|
|
638
|
+
}
|
|
639
|
+
const source = this.sources.get(sourceId);
|
|
640
|
+
if (!source) {
|
|
641
|
+
return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: paths.length };
|
|
642
|
+
}
|
|
643
|
+
let resumed = false;
|
|
644
|
+
if (this.isAborted(sourceId)) {
|
|
645
|
+
this.beginAbortEpoch(sourceId);
|
|
646
|
+
resumed = true;
|
|
647
|
+
}
|
|
648
|
+
let queued = 0;
|
|
649
|
+
let alreadyActive = 0;
|
|
650
|
+
let cleanedNonFiles = 0;
|
|
651
|
+
let rejected = 0;
|
|
652
|
+
const isRemote = source.kind === 'url' || source.kind === 'connector';
|
|
653
|
+
for (const p of paths) {
|
|
654
|
+
if (!this.sourceOwnsPath(source, p)) {
|
|
655
|
+
rejected += 1;
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
if (isRemote) {
|
|
659
|
+
// 外源:path 重做 = 重新 fetch + 索引(不读本地盘?
|
|
660
|
+
this.index.invalidateFileForReparse(sourceId, p);
|
|
661
|
+
if (await this.enqueue(sourceId, 'fetch_doc', p, 1)) {
|
|
662
|
+
queued += 1;
|
|
663
|
+
}
|
|
664
|
+
else if (this.hasActiveParseJob(sourceId, p)) {
|
|
665
|
+
alreadyActive += 1;
|
|
666
|
+
}
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
// /节点:清即可,没?
|
|
670
|
+
try {
|
|
671
|
+
const st = statSync(p);
|
|
672
|
+
if (!st.isFile()) {
|
|
673
|
+
this.index.removeFile(sourceId, p);
|
|
674
|
+
cleanedNonFiles += 1;
|
|
675
|
+
continue;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
catch {
|
|
679
|
+
// 不存仍按文件流程 parse ?
|
|
680
|
+
}
|
|
681
|
+
// 总是失效 hash:即使已在队列,跑起来也不能 isFresh 跳过
|
|
682
|
+
this.index.invalidateFileForReparse(sourceId, p);
|
|
683
|
+
if (await this.enqueue(sourceId, 'parse_file', p, 1)) {
|
|
684
|
+
queued += 1;
|
|
685
|
+
}
|
|
686
|
+
else if (this.hasActiveParseJob(sourceId, p)) {
|
|
687
|
+
alreadyActive += 1;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
this.kick();
|
|
691
|
+
return { queued, alreadyActive, cleanedNonFiles, resumed, rejected };
|
|
692
|
+
}
|
|
693
|
+
/** 上是否已?queued/running ?parse/fetch 任务 */
|
|
694
|
+
hasActiveParseJob(sourceId, path) {
|
|
695
|
+
return this.jobQueue.hasActivePathJob(sourceId, path);
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* 级任?文件UI 跟踪重做完成?
|
|
699
|
+
*
|
|
700
|
+
* @param sourceId - ?id
|
|
701
|
+
* @param paths - 列表
|
|
702
|
+
*/
|
|
703
|
+
jobStateForPaths(sourceId, paths) {
|
|
704
|
+
return paths.map((p) => {
|
|
705
|
+
const jobs = this.sources.database.raw
|
|
706
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs
|
|
707
|
+
WHERE source_id = ? AND path = ? AND status IN ('queued','running')`)
|
|
708
|
+
.get(sourceId, p);
|
|
709
|
+
const f = this.index.getFile(sourceId, p);
|
|
710
|
+
let exists = false;
|
|
711
|
+
try {
|
|
712
|
+
exists = statSync(p).isFile();
|
|
713
|
+
}
|
|
714
|
+
catch {
|
|
715
|
+
exists = false;
|
|
716
|
+
}
|
|
717
|
+
return {
|
|
718
|
+
path: p,
|
|
719
|
+
jobsActive: Number(jobs?.n ?? 0),
|
|
720
|
+
fileStatus: f?.status ?? null,
|
|
721
|
+
chunkCount: f?.chunkCount ?? 0,
|
|
722
|
+
error: f?.error ?? null,
|
|
723
|
+
exists,
|
|
724
|
+
};
|
|
725
|
+
});
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* 按筛选批量重做(当前列表?status/ext/q 条件?
|
|
729
|
+
*
|
|
730
|
+
* @param sourceId - ?id
|
|
731
|
+
* @param opts - 与文件列表筛选一?
|
|
732
|
+
* @returns 入队条数
|
|
733
|
+
*/
|
|
734
|
+
async reprocessByFilter(sourceId, opts) {
|
|
735
|
+
const paths = this.index.listFilePathsFiltered(sourceId, opts);
|
|
736
|
+
return this.reprocessFiles(sourceId, paths);
|
|
737
|
+
}
|
|
349
738
|
/**
|
|
350
739
|
* 处理 fs 变更(debounce 后入队)
|
|
351
740
|
*/
|
|
@@ -358,30 +747,33 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
358
747
|
clearTimeout(prev);
|
|
359
748
|
this.debounceTimers.set(key, setTimeout(() => {
|
|
360
749
|
this.debounceTimers.delete(key);
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
750
|
+
// 统一?stat:目?change 时不?parse ?no_adapter
|
|
751
|
+
void stat(filePath)
|
|
752
|
+
.then(async (st) => {
|
|
753
|
+
if (st.isFile()) {
|
|
754
|
+
await this.enqueue(sourceId, 'parse_file', filePath, 1);
|
|
755
|
+
}
|
|
756
|
+
else if (event === 'rename' || !st.isDirectory()) {
|
|
757
|
+
await this.enqueue(sourceId, 'drop_file', filePath, 1);
|
|
758
|
+
}
|
|
759
|
+
else {
|
|
760
|
+
this.index.removeFile(sourceId, filePath);
|
|
761
|
+
}
|
|
762
|
+
this.requestGapScan(sourceId);
|
|
763
|
+
this.kick();
|
|
764
|
+
})
|
|
765
|
+
.catch(() => {
|
|
766
|
+
void this.enqueue(sourceId, 'drop_file', filePath, 1)
|
|
767
|
+
.then(() => {
|
|
768
|
+
this.requestGapScan(sourceId);
|
|
371
769
|
this.kick();
|
|
372
770
|
})
|
|
373
|
-
.catch(() =>
|
|
374
|
-
|
|
375
|
-
this.kick();
|
|
376
|
-
});
|
|
377
|
-
return;
|
|
378
|
-
}
|
|
379
|
-
this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
|
|
380
|
-
this.kick();
|
|
771
|
+
.catch(() => undefined);
|
|
772
|
+
});
|
|
381
773
|
}, this.debounceMs));
|
|
382
774
|
}
|
|
383
775
|
/**
|
|
384
|
-
*
|
|
776
|
+
* watch(幂等)
|
|
385
777
|
*/
|
|
386
778
|
startWatch(sourceId) {
|
|
387
779
|
const source = this.sources.get(sourceId);
|
|
@@ -425,7 +817,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
425
817
|
}
|
|
426
818
|
}
|
|
427
819
|
/**
|
|
428
|
-
*
|
|
820
|
+
* 等待队列排空(测?管理?
|
|
429
821
|
*/
|
|
430
822
|
async idle(timeoutMs = 30_000) {
|
|
431
823
|
const start = Date.now();
|
|
@@ -441,81 +833,112 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
441
833
|
this.disposed = true;
|
|
442
834
|
this.stopPolling();
|
|
443
835
|
this.stopReconciler();
|
|
836
|
+
if (this.gapScanTimer) {
|
|
837
|
+
clearTimeout(this.gapScanTimer);
|
|
838
|
+
this.gapScanTimer = null;
|
|
839
|
+
}
|
|
840
|
+
this.gapScanUrgent.clear();
|
|
444
841
|
void this.abortJobs();
|
|
445
842
|
for (const id of this.watchers.keys())
|
|
446
843
|
this.stopWatch(id);
|
|
447
844
|
}
|
|
448
845
|
/**
|
|
449
|
-
*
|
|
846
|
+
* 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
|
|
450
847
|
*
|
|
451
|
-
* @param opts.sourceId -
|
|
452
|
-
* @returns
|
|
848
|
+
* @param opts.sourceId - 仅中源;省略则全部源
|
|
849
|
+
* @returns 取消
|
|
453
850
|
*/
|
|
454
851
|
abortJobs(opts) {
|
|
455
852
|
const sid = opts?.sourceId?.trim();
|
|
456
|
-
const ids = sid
|
|
853
|
+
const ids = sid
|
|
854
|
+
? [sid]
|
|
855
|
+
: [
|
|
856
|
+
...this.jobControl.listAbortedSourceIds(),
|
|
857
|
+
...this.sources.list().map((s) => s.id),
|
|
858
|
+
];
|
|
457
859
|
const unique = [...new Set(ids)];
|
|
458
860
|
let cancelledQueued = 0;
|
|
459
861
|
let abortedRunning = 0;
|
|
460
|
-
// 1) 清空 queued → cancelled
|
|
461
862
|
for (const id of unique) {
|
|
863
|
+
// 1) DB 落中跨重+ 打断程信?
|
|
864
|
+
const newly = this.jobControl.markAborted(id);
|
|
865
|
+
if (newly)
|
|
866
|
+
abortedRunning += 1;
|
|
867
|
+
// 2) 清空 queued ?cancelled(共?File ?parse 不杀:仍领则保留?
|
|
462
868
|
const res = this.sources.database.raw
|
|
463
869
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
464
|
-
WHERE source_id = ? AND status = 'queued'
|
|
465
|
-
|
|
870
|
+
WHERE source_id = ? AND status = 'queued'
|
|
871
|
+
AND (
|
|
872
|
+
file_id IS NULL
|
|
873
|
+
OR kind IN ('walk_source', 'embed_source')
|
|
874
|
+
OR NOT EXISTS (
|
|
875
|
+
SELECT 1 FROM knowledge_memberships m
|
|
876
|
+
WHERE m.file_id = knowledge_jobs.file_id AND m.source_id != ?
|
|
877
|
+
)
|
|
878
|
+
)`)
|
|
879
|
+
.run(Date.now(), id, id);
|
|
466
880
|
cancelledQueued += Number(res.changes ?? 0);
|
|
467
|
-
// 2) 打断在跑(worker terminate / embed 循环退出);保持 aborted 直到下次 reindex
|
|
468
|
-
let ctrl = this.abortBySource.get(id);
|
|
469
|
-
if (!ctrl) {
|
|
470
|
-
ctrl = new AbortController();
|
|
471
|
-
this.abortBySource.set(id, ctrl);
|
|
472
|
-
}
|
|
473
|
-
if (!ctrl.signal.aborted) {
|
|
474
|
-
ctrl.abort();
|
|
475
|
-
abortedRunning += 1;
|
|
476
|
-
}
|
|
477
881
|
}
|
|
478
|
-
// 3)
|
|
479
|
-
|
|
480
|
-
.
|
|
481
|
-
|
|
482
|
-
|
|
882
|
+
// 3) 看门狗不要再?aborted running 收回 queued(限定本次涉及的源)
|
|
883
|
+
if (sid) {
|
|
884
|
+
this.sources.database.raw
|
|
885
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
886
|
+
WHERE source_id = ? AND status = 'running' AND last_error = 'aborted'`)
|
|
887
|
+
.run(Date.now(), sid);
|
|
888
|
+
}
|
|
889
|
+
else {
|
|
890
|
+
this.sources.database.raw
|
|
891
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
892
|
+
WHERE status = 'running' AND last_error = 'aborted'`)
|
|
893
|
+
.run(Date.now());
|
|
894
|
+
}
|
|
895
|
+
const runningRow = this.sources.database.raw
|
|
896
|
+
.prepare(sid
|
|
897
|
+
? `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE source_id = ? AND status = 'running'`
|
|
898
|
+
: `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'running'`)
|
|
899
|
+
.get(...(sid ? [sid] : []));
|
|
900
|
+
const runningJobs = Number(runningRow?.n ?? 0);
|
|
483
901
|
this.emitProgress({
|
|
484
902
|
type: 'source',
|
|
485
903
|
sourceId: sid ?? '*',
|
|
486
904
|
status: 'aborted',
|
|
487
|
-
detail: `cancelledQueued=${cancelledQueued}`,
|
|
905
|
+
detail: `cancelledQueued=${cancelledQueued} runningTail=${runningJobs}`,
|
|
488
906
|
});
|
|
489
|
-
return { cancelledQueued, abortedRunning };
|
|
907
|
+
return { cancelledQueued, abortedRunning, runningJobs };
|
|
490
908
|
}
|
|
491
909
|
/**
|
|
492
|
-
*
|
|
910
|
+
* /继续索引(相轻)
|
|
493
911
|
*
|
|
494
|
-
* -
|
|
495
|
-
* -
|
|
912
|
+
* - 清除该源?
|
|
913
|
+
* - ?cancelled 任务放回 queued
|
|
496
914
|
* - 若仍缺向量则排队 embed
|
|
497
|
-
* -
|
|
915
|
+
* - 不重?walk 全量;需要全 reindex
|
|
498
916
|
*
|
|
499
|
-
* @param opts.sourceId -
|
|
500
|
-
* @returns
|
|
917
|
+
* @param opts.sourceId - 仅恢源;省略则全部源
|
|
918
|
+
* @returns
|
|
501
919
|
*/
|
|
502
920
|
resumeJobs(opts) {
|
|
503
921
|
const sid = opts?.sourceId?.trim();
|
|
504
922
|
const ids = sid
|
|
505
923
|
? [sid]
|
|
506
|
-
: [
|
|
924
|
+
: [
|
|
925
|
+
...new Set([
|
|
926
|
+
...this.jobControl.listAbortedSourceIds(),
|
|
927
|
+
...this.sources.list().map((s) => s.id),
|
|
928
|
+
]),
|
|
929
|
+
];
|
|
507
930
|
let restoredCancelled = 0;
|
|
508
931
|
let embedQueued = 0;
|
|
509
932
|
for (const id of ids) {
|
|
510
|
-
// 1)
|
|
933
|
+
// 1) 清除态(DB + 新纪元信号)
|
|
511
934
|
this.beginAbortEpoch(id);
|
|
512
|
-
// 2) cancelled
|
|
935
|
+
// 2) cancelled ?queued(用户中止时的活?
|
|
513
936
|
const res = this.sources.database.raw
|
|
514
|
-
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
|
|
515
|
-
WHERE source_id = ? AND status = 'cancelled'`)
|
|
937
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', last_error = NULL, updated_at = ?
|
|
938
|
+
WHERE source_id = ? AND status = 'cancelled' AND (last_error IS NULL OR last_error = 'aborted')`)
|
|
516
939
|
.run(Date.now(), id);
|
|
517
940
|
restoredCancelled += Number(res.changes ?? 0);
|
|
518
|
-
// 3)
|
|
941
|
+
// 3) 仍缺向量则确?embed 在队
|
|
519
942
|
if (this.embedding) {
|
|
520
943
|
const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
|
|
521
944
|
const active = this.countActiveJobs(id);
|
|
@@ -534,31 +957,24 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
534
957
|
});
|
|
535
958
|
return { restoredCancelled, embedQueued };
|
|
536
959
|
}
|
|
537
|
-
/**
|
|
960
|
+
/** 的未(resume / supersede 后用;清 DB 态) */
|
|
538
961
|
beginAbortEpoch(sourceId) {
|
|
539
|
-
this.
|
|
962
|
+
this.jobControl.beginEpoch(sourceId);
|
|
540
963
|
}
|
|
541
964
|
/**
|
|
542
|
-
* 重建前作废本源未完成任务(supersede
|
|
965
|
+
* 重建前作废本源未完成任务(supersede?
|
|
543
966
|
*
|
|
544
|
-
* - queued
|
|
545
|
-
* - running 打断(worker
|
|
546
|
-
* - 随后 beginAbortEpoch
|
|
967
|
+
* - queued ?cancelled(superseded_by_reindex?
|
|
968
|
+
* - running 打断(worker/出)
|
|
969
|
+
* - 随后 beginAbortEpoch,本建可继续?
|
|
547
970
|
*/
|
|
548
971
|
supersedeSourceWork(sourceId) {
|
|
549
972
|
this.sources.database.raw
|
|
550
973
|
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
|
|
551
974
|
WHERE source_id = ? AND status = 'queued'`)
|
|
552
975
|
.run(Date.now(), sourceId);
|
|
553
|
-
// 打断在跑 parse/embed
|
|
554
|
-
|
|
555
|
-
if (!ctrl) {
|
|
556
|
-
ctrl = new AbortController();
|
|
557
|
-
this.abortBySource.set(sourceId, ctrl);
|
|
558
|
-
}
|
|
559
|
-
if (!ctrl.signal.aborted) {
|
|
560
|
-
ctrl.abort();
|
|
561
|
-
}
|
|
976
|
+
// 打断在跑 parse/embed;再新纪元(清中,避?walk ?abort
|
|
977
|
+
this.jobControl.markAborted(sourceId);
|
|
562
978
|
this.beginAbortEpoch(sourceId);
|
|
563
979
|
this.emitProgress({
|
|
564
980
|
type: 'source',
|
|
@@ -575,8 +991,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
575
991
|
return 0;
|
|
576
992
|
if (this.isAborted(sourceId))
|
|
577
993
|
return 0;
|
|
578
|
-
//
|
|
579
|
-
|
|
994
|
+
// 仅跳过「全量 walk 在途」:individual parse_file 排队 **不得** 挡住缺口补扫。
|
|
995
|
+
// 否则 watch 漏事件后,当前批次 parse 会一直占着门闩,排空才一次入队数百文件(84 → 726)。
|
|
996
|
+
// 不用 sourceLocks:与 reconcile 同轮 fire-and-forget ingestSource 竞态,会误伤补扫。
|
|
997
|
+
if (this.countQueuedKinds(['walk_source'], sourceId) > 0)
|
|
580
998
|
return 0;
|
|
581
999
|
let found;
|
|
582
1000
|
try {
|
|
@@ -585,12 +1003,28 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
585
1003
|
catch {
|
|
586
1004
|
return 0;
|
|
587
1005
|
}
|
|
588
|
-
const
|
|
1006
|
+
const byPath = new Map(this.index.listFiles(sourceId).map((f) => [f.path.replace(/\\/g, '/'), f]));
|
|
589
1007
|
let added = 0;
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
1008
|
+
let retriedSkipped = 0;
|
|
1009
|
+
let retriedIncomplete = 0;
|
|
1010
|
+
for (const f of found.docs) {
|
|
1011
|
+
const logical = f.path.replace(/\\/g, '/');
|
|
1012
|
+
const existing = byPath.get(logical);
|
|
1013
|
+
// 缺文件必补;indexing=半写入/崩溃残留必补;skipped 且原因可重试时补
|
|
1014
|
+
const need = !existing ||
|
|
1015
|
+
existing.status === 'indexing' ||
|
|
1016
|
+
(existing.status === 'skipped' &&
|
|
1017
|
+
isRetryableSkipReason(existing.error ?? null, existing.size, this.fileLimits));
|
|
1018
|
+
if (!need)
|
|
1019
|
+
continue;
|
|
1020
|
+
if (await this.enqueue(sourceId, 'parse_file', f.path, 2)) {
|
|
593
1021
|
added += 1;
|
|
1022
|
+
if (!existing)
|
|
1023
|
+
continue;
|
|
1024
|
+
if (existing.status === 'indexing')
|
|
1025
|
+
retriedIncomplete += 1;
|
|
1026
|
+
else
|
|
1027
|
+
retriedSkipped += 1;
|
|
594
1028
|
}
|
|
595
1029
|
}
|
|
596
1030
|
if (added > 0) {
|
|
@@ -598,19 +1032,37 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
598
1032
|
type: 'source',
|
|
599
1033
|
sourceId,
|
|
600
1034
|
status: 'parse_gap_found',
|
|
601
|
-
detail: `missing_files=${added}`,
|
|
1035
|
+
detail: `missing_files=${added - retriedSkipped - retriedIncomplete} retry_skipped=${retriedSkipped} retry_indexing=${retriedIncomplete}`,
|
|
602
1036
|
});
|
|
603
1037
|
this.kick();
|
|
604
1038
|
}
|
|
605
1039
|
return added;
|
|
606
1040
|
}
|
|
1041
|
+
/** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
|
|
1042
|
+
requestGapScan(sourceId, delayMs = 3_000) {
|
|
1043
|
+
if (this.disposed || this.isAborted(sourceId))
|
|
1044
|
+
return;
|
|
1045
|
+
this.gapScanUrgent.add(sourceId);
|
|
1046
|
+
if (this.gapScanTimer)
|
|
1047
|
+
return;
|
|
1048
|
+
this.gapScanTimer = setTimeout(() => {
|
|
1049
|
+
this.gapScanTimer = null;
|
|
1050
|
+
const ids = [...this.gapScanUrgent];
|
|
1051
|
+
this.gapScanUrgent.clear();
|
|
1052
|
+
for (const id of ids) {
|
|
1053
|
+
this.lastParseGapScanAt.set(id, Date.now());
|
|
1054
|
+
void this.ensureParseCoverage(id).catch(() => undefined);
|
|
1055
|
+
}
|
|
1056
|
+
}, delayMs);
|
|
1057
|
+
this.gapScanTimer.unref?.();
|
|
1058
|
+
}
|
|
607
1059
|
/**
|
|
608
|
-
*
|
|
1060
|
+
* 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
|
|
609
1061
|
*
|
|
610
|
-
*
|
|
611
|
-
* embed
|
|
1062
|
+
* 源用?discover walk ?keep 集(;失回并?stat?
|
|
1063
|
+
* embed 前调避免给已移出文件的残 chunks 上向?
|
|
612
1064
|
*
|
|
613
|
-
* @param sourceId -
|
|
1065
|
+
* @param sourceId - 知识?id
|
|
614
1066
|
* @returns 删除的文件数
|
|
615
1067
|
*/
|
|
616
1068
|
async pruneMissingOnDisk(sourceId) {
|
|
@@ -621,9 +1073,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
621
1073
|
if (source.kind === 'directory' || source.kind === 'workspace') {
|
|
622
1074
|
try {
|
|
623
1075
|
const found = await this.localFetcher.discover(source);
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
1076
|
+
if (!found.complete) {
|
|
1077
|
+
// walk 不完整:?stat止用残缺 keep 清库
|
|
1078
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
1079
|
+
}
|
|
1080
|
+
else {
|
|
1081
|
+
const keep = new Set(found.docs.map((f) => f.path));
|
|
1082
|
+
// walk 完整:即使空也可清空(与逐文?stat 致)
|
|
1083
|
+
removed = await this.index.pruneMissingAsync(sourceId, keep, { allowEmptyKeep: true });
|
|
1084
|
+
}
|
|
627
1085
|
}
|
|
628
1086
|
catch {
|
|
629
1087
|
removed = await this.pruneByStatScan(sourceId);
|
|
@@ -639,12 +1097,12 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
639
1097
|
status: 'pruned_missing',
|
|
640
1098
|
detail: `removed_files=${removed}`,
|
|
641
1099
|
});
|
|
642
|
-
this.refreshCoverage(sourceId);
|
|
1100
|
+
this.refreshCoverage(sourceId, { force: true });
|
|
643
1101
|
}
|
|
644
1102
|
return removed;
|
|
645
1103
|
}
|
|
646
|
-
/**
|
|
647
|
-
async pruneByStatScan(sourceId, concurrency =
|
|
1104
|
+
/** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
1105
|
+
async pruneByStatScan(sourceId, concurrency = 4) {
|
|
648
1106
|
const files = this.index.listFiles(sourceId);
|
|
649
1107
|
let removed = 0;
|
|
650
1108
|
let i = 0;
|
|
@@ -665,7 +1123,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
665
1123
|
exists = false;
|
|
666
1124
|
}
|
|
667
1125
|
if (!exists) {
|
|
668
|
-
this.index.
|
|
1126
|
+
await this.index.removeFileAsync(sourceId, f.path);
|
|
669
1127
|
removed += 1;
|
|
670
1128
|
}
|
|
671
1129
|
}
|
|
@@ -674,23 +1132,29 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
674
1132
|
return removed;
|
|
675
1133
|
}
|
|
676
1134
|
abortSignalFor(sourceId) {
|
|
677
|
-
|
|
678
|
-
if (!ac) {
|
|
679
|
-
ac = new AbortController();
|
|
680
|
-
this.abortBySource.set(sourceId, ac);
|
|
681
|
-
}
|
|
682
|
-
return ac.signal;
|
|
1135
|
+
return this.jobControl.signalFor(sourceId);
|
|
683
1136
|
}
|
|
684
1137
|
isAborted(sourceId) {
|
|
685
|
-
return this.
|
|
1138
|
+
return this.jobControl.isAborted(sourceId);
|
|
686
1139
|
}
|
|
687
1140
|
/**
|
|
688
|
-
*
|
|
1141
|
+
* /继续 控制数(互斥按钮
|
|
689
1142
|
*
|
|
690
|
-
*
|
|
691
|
-
*
|
|
692
|
-
|
|
693
|
-
|
|
1143
|
+
* @param sourceId - ?id
|
|
1144
|
+
* @returns 队列计数 + 已中?+ 仍缺向量
|
|
1145
|
+
*/
|
|
1146
|
+
jobControlState(sourceId) {
|
|
1147
|
+
return readJobControlState(this.sources.database, this.index, sourceId, {
|
|
1148
|
+
embeddingEnabled: Boolean(this.embedding),
|
|
1149
|
+
});
|
|
1150
|
+
}
|
|
1151
|
+
/**
|
|
1152
|
+
* 任务对账(看门狗?不依赖单?ensureEmbedJob
|
|
1153
|
+
*
|
|
1154
|
+
* 解决「续去重吞掉 / 进程 / partial」等静默停摆?
|
|
1155
|
+
* 1. 回收超时 running(崩?孤儿?
|
|
1156
|
+
* 2. 有缺向量且无 active embed ?排队
|
|
1157
|
+
* 3. 无队缺口 ??coverage/status
|
|
694
1158
|
*/
|
|
695
1159
|
startReconciler(intervalMs = 15_000) {
|
|
696
1160
|
if (this.reconcileTimer || this.disposed)
|
|
@@ -713,38 +1177,38 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
713
1177
|
this.reconcileTimer = null;
|
|
714
1178
|
}
|
|
715
1179
|
}
|
|
716
|
-
/**
|
|
1180
|
+
/** 对账入口(也工调 */
|
|
717
1181
|
async reconcileJobs() {
|
|
718
1182
|
if (this.disposed)
|
|
719
1183
|
return;
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
1184
|
+
// stale ?**大于** 时,否则合法的长 parse 收回后双?
|
|
1185
|
+
this.reclaimStaleRunningJobs(this.fileLimits.maxParseTimeoutMs + 60_000);
|
|
1186
|
+
// path: 与 win:/unix: 身份分裂的零认领残留
|
|
1187
|
+
this.index.purgeOrphanPathIdentityFiles();
|
|
1188
|
+
// 等历入项(轻量;非每
|
|
1189
|
+
this.cleanupNonFileIndexRows();
|
|
1190
|
+
// ?job 定期清理,防表无?
|
|
1191
|
+
this.cleanupTerminalJobs();
|
|
1192
|
+
// 存量 FTS 后台补齐(分出事件循 rebuild?
|
|
1193
|
+
void this.index.ensureFtsBackfill();
|
|
1194
|
+
// 存量向量回填:小批量 + 让出事件?2000 条同步写(会 abort?
|
|
1195
|
+
await this.index.backfillVecFromEmbeddings(25);
|
|
729
1196
|
for (const s of this.sources.list()) {
|
|
730
1197
|
if (s.status === 'removed' || s.status === 'disabled')
|
|
731
1198
|
continue;
|
|
732
1199
|
if (this.isAborted(s.id))
|
|
733
1200
|
continue;
|
|
734
|
-
// watch 漏事件时补齐 parse:磁盘有、索引无 → 入队(解析优先于 embed)
|
|
735
|
-
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
736
|
-
if (Date.now() - lastGap > 60_000) {
|
|
737
|
-
this.lastParseGapScanAt.set(s.id, Date.now());
|
|
738
|
-
await this.ensureParseCoverage(s.id);
|
|
739
|
-
}
|
|
740
|
-
const missing = this.index.listChunksMissingEmbedding(s.id, 1).length > 0;
|
|
741
1201
|
const active = this.countActiveJobs(s.id);
|
|
742
|
-
|
|
743
|
-
|
|
1202
|
+
// discovering 卡死(注册 kick 失败 / file 源 walk 中断):无任务则 re-kick,不 supersede
|
|
1203
|
+
if (active.total === 0 &&
|
|
1204
|
+
(s.status === 'pending' || s.status === 'discovering') &&
|
|
1205
|
+
(s.kind === 'file' || s.kind === 'directory' || s.kind === 'workspace')) {
|
|
1206
|
+
await this.ingestSource(s.id, { fromQueue: true }).catch(() => undefined);
|
|
744
1207
|
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
1208
|
+
// 稳定态收尾(auto-describe 等)关键词部署同触发
|
|
1209
|
+
// 先看 active,再 gap ;避免刚入队?gap job 挡住 settled
|
|
1210
|
+
if (active.total === 0) {
|
|
1211
|
+
this.refreshCoverage(s.id, { force: true });
|
|
748
1212
|
try {
|
|
749
1213
|
this.onSourceSettled?.(s.id);
|
|
750
1214
|
}
|
|
@@ -752,34 +1216,52 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
752
1216
|
/* ignore settle hook errors */
|
|
753
1217
|
}
|
|
754
1218
|
}
|
|
1219
|
+
else {
|
|
1220
|
+
this.flushCoverageDirty(s.id);
|
|
1221
|
+
}
|
|
1222
|
+
// watch 漏事件 / skip 重试时补齐 parse(与配置 embedding 无关)
|
|
1223
|
+
// urgent(watch 突发)不走 60s 节流;否则批量拷贝后要等一整轮才补
|
|
1224
|
+
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
1225
|
+
const gapDue = this.gapScanUrgent.has(s.id) || Date.now() - lastGap > 60_000;
|
|
1226
|
+
if (gapDue) {
|
|
1227
|
+
this.lastParseGapScanAt.set(s.id, Date.now());
|
|
1228
|
+
this.gapScanUrgent.delete(s.id);
|
|
1229
|
+
await this.ensureParseCoverage(s.id);
|
|
1230
|
+
// drop_file 在途时勿再 prune:双路径会同时删同一棵子树,放大写压力
|
|
1231
|
+
const dropsInFlight = this.countQueuedKinds(['drop_file'], s.id) > 0 || this.hasRunningKind(s.id, 'drop_file');
|
|
1232
|
+
if (!dropsInFlight) {
|
|
1233
|
+
await this.pruneMissingOnDisk(s.id);
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
const missingEmbed = this.embedding
|
|
1237
|
+
? this.index.hasChunksMissingEmbedding(s.id)
|
|
1238
|
+
: false;
|
|
1239
|
+
if (this.embedding && missingEmbed && active.embed === 0) {
|
|
1240
|
+
this.ensureEmbedJob(s.id);
|
|
1241
|
+
}
|
|
755
1242
|
}
|
|
756
1243
|
this.kick();
|
|
757
1244
|
}
|
|
758
|
-
/**
|
|
1245
|
+
/** ?job 清理(done/failed/cancelled 超过保留期) */
|
|
1246
|
+
cleanupTerminalJobs(retainMs = 24 * 60 * 60_000) {
|
|
1247
|
+
this.jobQueue.cleanupTerminal(retainMs);
|
|
1248
|
+
}
|
|
1249
|
+
/** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
|
|
759
1250
|
reclaimStaleRunningJobs(staleMs) {
|
|
760
|
-
|
|
761
|
-
// 先放弃重试过多的
|
|
762
|
-
this.sources.database.raw
|
|
763
|
-
.prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
|
|
764
|
-
WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
|
|
765
|
-
.run(Date.now(), cutoff);
|
|
766
|
-
this.sources.database.raw
|
|
767
|
-
.prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
|
|
768
|
-
WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
|
|
769
|
-
.run(Date.now(), cutoff);
|
|
1251
|
+
this.jobQueue.reclaimStaleRunning(staleMs);
|
|
770
1252
|
}
|
|
771
1253
|
countActiveJobs(sourceId) {
|
|
1254
|
+
return this.jobQueue.countActiveJobs(sourceId);
|
|
1255
|
+
}
|
|
1256
|
+
hasRunningKind(sourceId, kind) {
|
|
772
1257
|
const row = this.sources.database.raw
|
|
773
|
-
.prepare(`SELECT
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
WHERE source_id = ? AND status IN ('queued', 'running')`)
|
|
778
|
-
.get(sourceId);
|
|
779
|
-
return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
|
|
1258
|
+
.prepare(`SELECT 1 AS ok FROM knowledge_jobs
|
|
1259
|
+
WHERE source_id = ? AND kind = ? AND status = 'running' LIMIT 1`)
|
|
1260
|
+
.get(sourceId, kind);
|
|
1261
|
+
return Boolean(row?.ok);
|
|
780
1262
|
}
|
|
781
1263
|
/**
|
|
782
|
-
*
|
|
1264
|
+
* poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
|
|
783
1265
|
*/
|
|
784
1266
|
startPolling() {
|
|
785
1267
|
if (this.pollTimer || this.disposed)
|
|
@@ -794,7 +1276,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
794
1276
|
});
|
|
795
1277
|
});
|
|
796
1278
|
}, this.pollTickMs);
|
|
797
|
-
//
|
|
1279
|
+
// 不阻止进?
|
|
798
1280
|
this.pollTimer.unref?.();
|
|
799
1281
|
}
|
|
800
1282
|
stopPolling() {
|
|
@@ -804,9 +1286,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
804
1286
|
}
|
|
805
1287
|
}
|
|
806
1288
|
/**
|
|
807
|
-
*
|
|
1289
|
+
* 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
|
|
808
1290
|
*
|
|
809
|
-
* @returns
|
|
1291
|
+
* @returns 实际触发?sourceId 列表
|
|
810
1292
|
*/
|
|
811
1293
|
async pollDueSources() {
|
|
812
1294
|
if (this.disposed)
|
|
@@ -836,8 +1318,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
836
1318
|
status: 'poll_due',
|
|
837
1319
|
detail: source.sync.intervalMs?.toString(),
|
|
838
1320
|
});
|
|
839
|
-
//
|
|
840
|
-
void this.ingestSource(source.id).catch((err) => {
|
|
1321
|
+
// 入队,不 tick量:?active 则跳过, supersede 打断
|
|
1322
|
+
void this.ingestSource(source.id, { incremental: true }).catch((err) => {
|
|
841
1323
|
this.emitProgress({
|
|
842
1324
|
type: 'error',
|
|
843
1325
|
sourceId: source.id,
|
|
@@ -848,61 +1330,78 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
848
1330
|
}
|
|
849
1331
|
return triggered;
|
|
850
1332
|
}
|
|
851
|
-
//
|
|
1333
|
+
// 内部
|
|
852
1334
|
hasQueuedJobs() {
|
|
853
|
-
|
|
854
|
-
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
|
|
855
|
-
.get();
|
|
856
|
-
return (row?.n ?? 0) > 0;
|
|
1335
|
+
return this.jobQueue.hasQueuedJobs();
|
|
857
1336
|
}
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
1337
|
+
/**
|
|
1338
|
+
* 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
|
|
1339
|
+
* 需要计数的路径请 `await enqueueStrict`。
|
|
1340
|
+
*/
|
|
1341
|
+
enqueue(sourceId, kind, path, priority, fileIdHint) {
|
|
1342
|
+
return this.enqueueStrict(sourceId, kind, path, priority, fileIdHint).catch(() => false);
|
|
1343
|
+
}
|
|
1344
|
+
/** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
|
|
1345
|
+
*
|
|
1346
|
+
* 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
|
|
1347
|
+
*/
|
|
1348
|
+
async enqueueStrict(sourceId, kind, path, priority, fileIdHint) {
|
|
1349
|
+
if (this.disposed)
|
|
1350
|
+
return false;
|
|
1351
|
+
const fileId = fileIdHint ?? (await this.resolveJobFileId(sourceId, kind, path));
|
|
1352
|
+
if (fileId && (kind === 'parse_file' || kind === 'fetch_doc')) {
|
|
1353
|
+
// 共享 File 已有在途 parse:补认领后跳过,避免双写
|
|
1354
|
+
const busy = this.sources.database.raw
|
|
875
1355
|
.prepare(`SELECT id FROM knowledge_jobs
|
|
876
|
-
WHERE
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
|
|
884
|
-
.get();
|
|
885
|
-
if ((depth?.n ?? 0) >= this.maxQueueDepth) {
|
|
886
|
-
this.emitProgress({
|
|
887
|
-
type: 'error',
|
|
888
|
-
sourceId,
|
|
889
|
-
status: 'queue_backpressure',
|
|
890
|
-
detail: `queued=${depth.n} (delay only, jobs kept)`,
|
|
891
|
-
});
|
|
1356
|
+
WHERE file_id = ? AND kind IN ('parse_file','fetch_doc') AND status IN ('queued','running')
|
|
1357
|
+
LIMIT 1`)
|
|
1358
|
+
.get(fileId);
|
|
1359
|
+
if (busy?.id && path) {
|
|
1360
|
+
this.index.attachMembership(sourceId, fileId, path);
|
|
1361
|
+
return false;
|
|
1362
|
+
}
|
|
892
1363
|
}
|
|
893
|
-
|
|
1364
|
+
const ok = this.jobQueue.enqueue(sourceId, kind, path, priority, fileId);
|
|
1365
|
+
if (ok && this.diskWatermarkAlert) {
|
|
894
1366
|
void this.warnDiskWatermark(sourceId);
|
|
895
1367
|
}
|
|
1368
|
+
return ok;
|
|
1369
|
+
}
|
|
1370
|
+
/** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
|
|
1371
|
+
bindJobFileId(job) {
|
|
1372
|
+
if (job.file_id || !job.path)
|
|
1373
|
+
return;
|
|
1374
|
+
const rec = this.index.getFile(job.source_id, job.path);
|
|
1375
|
+
if (!rec?.id)
|
|
1376
|
+
return;
|
|
896
1377
|
this.sources.database.raw
|
|
897
|
-
.prepare(`
|
|
898
|
-
|
|
899
|
-
|
|
1378
|
+
.prepare(`UPDATE knowledge_jobs SET file_id = ? WHERE id = ? AND file_id IS NULL`)
|
|
1379
|
+
.run(rec.id, job.id);
|
|
1380
|
+
job.file_id = rec.id;
|
|
1381
|
+
}
|
|
1382
|
+
/** job 绑定 file_id:membership → identity;本地文件可 identify */
|
|
1383
|
+
async resolveJobFileId(sourceId, kind, path) {
|
|
1384
|
+
if (!path)
|
|
1385
|
+
return null;
|
|
1386
|
+
if (kind !== 'parse_file' && kind !== 'fetch_doc' && kind !== 'drop_file')
|
|
1387
|
+
return null;
|
|
1388
|
+
const logical = path.replace(/\\/g, '/');
|
|
1389
|
+
const existing = this.index.getFile(sourceId, logical);
|
|
1390
|
+
if (existing?.id)
|
|
1391
|
+
return existing.id;
|
|
1392
|
+
try {
|
|
1393
|
+
const ident = await identifyLocalFile(path);
|
|
1394
|
+
return this.index.findFileIdByIdentity(ident.key);
|
|
1395
|
+
}
|
|
1396
|
+
catch {
|
|
1397
|
+
return null;
|
|
1398
|
+
}
|
|
900
1399
|
}
|
|
901
1400
|
kick() {
|
|
902
1401
|
if (this.disposed)
|
|
903
1402
|
return;
|
|
904
1403
|
if (this.draining) {
|
|
905
|
-
//
|
|
1404
|
+
// 等待 drain 会再 kick;标记一次避免丢信号
|
|
906
1405
|
this.kickAgain = true;
|
|
907
1406
|
return;
|
|
908
1407
|
}
|
|
@@ -917,39 +1416,40 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
917
1416
|
});
|
|
918
1417
|
}
|
|
919
1418
|
drain() {
|
|
1419
|
+
if (this.disposed)
|
|
1420
|
+
return;
|
|
920
1421
|
for (;;) {
|
|
1422
|
+
if (this.disposed)
|
|
1423
|
+
return;
|
|
921
1424
|
const allowParse = this.runningParse < this.parseConcurrency;
|
|
922
1425
|
const allowEmbed = this.runningEmbed < this.embedConcurrency;
|
|
923
1426
|
if (!allowParse && !allowEmbed)
|
|
924
1427
|
return;
|
|
925
|
-
const parsePending = this.countQueuedKinds([
|
|
926
|
-
'parse_file',
|
|
927
|
-
'walk_source',
|
|
928
|
-
'drop_file',
|
|
929
|
-
'fetch_doc',
|
|
930
|
-
]);
|
|
931
1428
|
const kinds = [];
|
|
932
1429
|
if (allowParse)
|
|
933
1430
|
kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
|
|
934
|
-
//
|
|
935
|
-
|
|
1431
|
+
// parse 优先(priority 2 < 3),但 **不得** 等全部 parse 结束才 embed:
|
|
1432
|
+
// 大库 parse 可能跑数小时,否则向量永远 0。embed 与 parse 并行,仅受 embed 槽限制。
|
|
1433
|
+
if (allowEmbed)
|
|
936
1434
|
kinds.push('embed_source');
|
|
937
1435
|
if (kinds.length === 0) {
|
|
938
|
-
if (!allowParse && allowEmbed && parsePending === 0)
|
|
939
|
-
return;
|
|
940
|
-
// parse 槽满但仍有 parse 积压 → 等 parse,不抢 embed
|
|
941
|
-
if (parsePending > 0 && !allowParse)
|
|
942
|
-
return;
|
|
943
1436
|
return;
|
|
944
1437
|
}
|
|
945
|
-
const job = this.claimJob(kinds);
|
|
946
|
-
if (!job)
|
|
1438
|
+
const job = this.claimJob(kinds, (j) => this.acceptParseSlot(j));
|
|
1439
|
+
if (!job) {
|
|
1440
|
+
// 队列清空:合并刷 parse/drop 标脏?coverage,避免每文件?COUNT
|
|
1441
|
+
this.flushCoverageDirty();
|
|
947
1442
|
return;
|
|
1443
|
+
}
|
|
948
1444
|
const isEmbed = job.kind === 'embed_source';
|
|
1445
|
+
const isDocParse = this.isDocParseJob(job);
|
|
949
1446
|
if (isEmbed)
|
|
950
1447
|
this.runningEmbed += 1;
|
|
951
|
-
else
|
|
1448
|
+
else {
|
|
952
1449
|
this.runningParse += 1;
|
|
1450
|
+
if (isDocParse)
|
|
1451
|
+
this.runningDocParse += 1;
|
|
1452
|
+
}
|
|
953
1453
|
void this.runJob(job)
|
|
954
1454
|
.catch((err) => {
|
|
955
1455
|
this.emitProgress({
|
|
@@ -963,40 +1463,32 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
963
1463
|
.finally(() => {
|
|
964
1464
|
if (isEmbed)
|
|
965
1465
|
this.runningEmbed -= 1;
|
|
966
|
-
else
|
|
1466
|
+
else {
|
|
967
1467
|
this.runningParse -= 1;
|
|
1468
|
+
if (isDocParse)
|
|
1469
|
+
this.runningDocParse -= 1;
|
|
1470
|
+
}
|
|
968
1471
|
this.kick();
|
|
969
1472
|
});
|
|
970
1473
|
}
|
|
971
1474
|
}
|
|
972
|
-
|
|
973
|
-
|
|
1475
|
+
/** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
|
|
1476
|
+
isDocParseJob(job) {
|
|
1477
|
+
return job.kind === 'parse_file' && !!job.path && isDocumentPath(job.path);
|
|
1478
|
+
}
|
|
1479
|
+
acceptParseSlot(job) {
|
|
1480
|
+
if (this.isDocParseJob(job)) {
|
|
1481
|
+
return this.runningDocParse < this.documentParseConcurrency;
|
|
1482
|
+
}
|
|
1483
|
+
return true;
|
|
1484
|
+
}
|
|
1485
|
+
countQueuedKinds(kinds, sourceId) {
|
|
1486
|
+
if (this.disposed)
|
|
974
1487
|
return 0;
|
|
975
|
-
|
|
976
|
-
const row = this.sources.database.raw
|
|
977
|
-
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`)
|
|
978
|
-
.get(...kinds);
|
|
979
|
-
return row?.n ?? 0;
|
|
1488
|
+
return this.jobQueue.countQueuedKinds(kinds, sourceId);
|
|
980
1489
|
}
|
|
981
|
-
claimJob(kinds) {
|
|
982
|
-
|
|
983
|
-
return null;
|
|
984
|
-
const placeholders = kinds.map(() => '?').join(',');
|
|
985
|
-
const row = this.sources.database.raw
|
|
986
|
-
.prepare(`SELECT * FROM knowledge_jobs
|
|
987
|
-
WHERE status = 'queued' AND kind IN (${placeholders})
|
|
988
|
-
ORDER BY priority ASC, created_at ASC
|
|
989
|
-
LIMIT 1`)
|
|
990
|
-
.get(...kinds);
|
|
991
|
-
if (!row)
|
|
992
|
-
return null;
|
|
993
|
-
const res = this.sources.database.raw
|
|
994
|
-
.prepare(`UPDATE knowledge_jobs SET status = 'running', updated_at = ?
|
|
995
|
-
WHERE id = ? AND status = 'queued'`)
|
|
996
|
-
.run(Date.now(), row.id);
|
|
997
|
-
if (Number(res.changes ?? 0) === 0)
|
|
998
|
-
return null;
|
|
999
|
-
return row;
|
|
1490
|
+
claimJob(kinds, accept) {
|
|
1491
|
+
return this.jobQueue.claimJob(kinds, accept);
|
|
1000
1492
|
}
|
|
1001
1493
|
async runJob(job) {
|
|
1002
1494
|
const now = Date.now();
|
|
@@ -1008,44 +1500,77 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1008
1500
|
return;
|
|
1009
1501
|
}
|
|
1010
1502
|
try {
|
|
1503
|
+
// 归属闸门:任何会碰本地盘 / 外源逻辑 path 任务都必须属于本?
|
|
1504
|
+
if (job.path && (job.kind === 'parse_file' || job.kind === 'fetch_doc')) {
|
|
1505
|
+
const owner = this.sources.get(job.source_id);
|
|
1506
|
+
if (owner && !this.sourceOwnsPath(owner, job.path)) {
|
|
1507
|
+
throw new Error(`path not owned by source: ${job.path}`);
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1011
1510
|
if (job.kind === 'parse_file' && job.path) {
|
|
1012
|
-
// 单文件解析硬超时:大 xlsx/pdf
|
|
1013
|
-
|
|
1511
|
+
// 单文件解析硬超时:大 xlsx/pdf 抽取会堵死事件循超时随体?
|
|
1512
|
+
let parseTimeoutMs = this.parseTimeoutMs;
|
|
1513
|
+
try {
|
|
1514
|
+
const st = await stat(job.path);
|
|
1515
|
+
if (st.isFile())
|
|
1516
|
+
parseTimeoutMs = parseTimeoutForSize(st.size, this.fileLimits);
|
|
1517
|
+
}
|
|
1518
|
+
catch {
|
|
1519
|
+
// 已删时交?parseOne ?
|
|
1520
|
+
}
|
|
1521
|
+
this.heartbeatJob(job.id);
|
|
1522
|
+
await this.withTimeout((signal) => this.parseOne(job.source_id, job.path, signal), parseTimeoutMs, `parse_timeout:${job.path}`, this.abortSignalFor(job.source_id));
|
|
1523
|
+
this.bindJobFileId(job);
|
|
1014
1524
|
if (this.isAborted(job.source_id)) {
|
|
1015
1525
|
throw new Error('aborted');
|
|
1016
1526
|
}
|
|
1017
|
-
this.
|
|
1527
|
+
this.markCoverageDirty(job.source_id);
|
|
1018
1528
|
if (this.embedding) {
|
|
1019
|
-
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
1529
|
+
await this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
1020
1530
|
this.kick();
|
|
1021
1531
|
}
|
|
1022
1532
|
}
|
|
1023
1533
|
else if (job.kind === 'fetch_doc' && job.path) {
|
|
1024
|
-
|
|
1025
|
-
this.
|
|
1534
|
+
this.heartbeatJob(job.id);
|
|
1535
|
+
await this.withTimeout((signal) => this.fetchDocJob(job.source_id, job.path, signal), this.parseTimeoutMs, `fetch_timeout:${job.path}`, this.abortSignalFor(job.source_id));
|
|
1536
|
+
this.bindJobFileId(job);
|
|
1537
|
+
if (this.isAborted(job.source_id)) {
|
|
1538
|
+
throw new Error('aborted');
|
|
1539
|
+
}
|
|
1540
|
+
this.markCoverageDirty(job.source_id);
|
|
1026
1541
|
if (this.embedding) {
|
|
1027
|
-
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
1542
|
+
await this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
1028
1543
|
this.kick();
|
|
1029
1544
|
}
|
|
1030
1545
|
}
|
|
1031
1546
|
else if (job.kind === 'drop_file' && job.path) {
|
|
1032
|
-
//
|
|
1033
|
-
this.index.
|
|
1034
|
-
this.
|
|
1547
|
+
// 移出/删除:连同子树异步清(分批让出,禁止同步连环 DELETE 堵死 Engine)
|
|
1548
|
+
await this.index.removePathTreeAsync(job.source_id, job.path);
|
|
1549
|
+
this.markCoverageDirty(job.source_id);
|
|
1035
1550
|
}
|
|
1036
1551
|
else if (job.kind === 'walk_source') {
|
|
1037
|
-
await this.ingestSource(job.source_id);
|
|
1552
|
+
await this.ingestSource(job.source_id, { fromQueue: true });
|
|
1553
|
+
if (this.isAborted(job.source_id)) {
|
|
1554
|
+
throw new Error('aborted');
|
|
1555
|
+
}
|
|
1038
1556
|
}
|
|
1039
1557
|
else if (job.kind === 'embed_source') {
|
|
1040
1558
|
if (!this.embedding) {
|
|
1041
1559
|
const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
|
|
1042
1560
|
if (stillMissing) {
|
|
1043
|
-
//
|
|
1561
|
+
// 空跑?done:迁移后缺向量却显示 queue=0
|
|
1044
1562
|
throw new Error('embedding_provider_missing');
|
|
1045
1563
|
}
|
|
1046
1564
|
return;
|
|
1047
1565
|
}
|
|
1048
1566
|
await this.embedPending(job.source_id, 50, job.id);
|
|
1567
|
+
if (this.isAborted(job.source_id)) {
|
|
1568
|
+
throw new Error('aborted');
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
// 统一收尾闸门:任?kind 后都不得?done
|
|
1572
|
+
if (this.isAborted(job.source_id)) {
|
|
1573
|
+
throw new Error('aborted');
|
|
1049
1574
|
}
|
|
1050
1575
|
this.sources.database.raw
|
|
1051
1576
|
.prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
|
|
@@ -1056,16 +1581,18 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1056
1581
|
path: job.path ?? undefined,
|
|
1057
1582
|
status: 'done',
|
|
1058
1583
|
});
|
|
1059
|
-
//
|
|
1584
|
+
// ?done 后再:否?enqueue 去重会把「自?active 而丢掉下?
|
|
1060
1585
|
if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
|
|
1061
1586
|
this.ensureEmbedJob(job.source_id);
|
|
1062
|
-
this.refreshCoverage(job.source_id);
|
|
1587
|
+
this.refreshCoverage(job.source_id, { force: true });
|
|
1063
1588
|
}
|
|
1064
1589
|
}
|
|
1065
1590
|
catch (err) {
|
|
1066
1591
|
const message = err instanceof Error ? err.message : String(err);
|
|
1067
1592
|
const aborted = message === 'aborted' || this.isAborted(job.source_id);
|
|
1068
|
-
|
|
1593
|
+
// 归属拒绝不是「文件解析失不得写入 knowledge_files
|
|
1594
|
+
const ownership = message.includes('not owned by source');
|
|
1595
|
+
if (job.kind === 'parse_file' && job.path && !aborted && !ownership) {
|
|
1069
1596
|
this.index.markFileError(job.source_id, job.path, message);
|
|
1070
1597
|
}
|
|
1071
1598
|
this.sources.database.raw
|
|
@@ -1079,28 +1606,41 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1079
1606
|
throw err;
|
|
1080
1607
|
}
|
|
1081
1608
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1609
|
+
/**
|
|
1610
|
+
* 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
|
|
1611
|
+
* 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
|
|
1612
|
+
*/
|
|
1613
|
+
async withTimeout(work, ms, label, parent) {
|
|
1614
|
+
const controller = new AbortController();
|
|
1615
|
+
const onParentAbort = () => controller.abort();
|
|
1616
|
+
parent?.addEventListener('abort', onParentAbort, { once: true });
|
|
1617
|
+
const timer = setTimeout(() => controller.abort(), ms);
|
|
1084
1618
|
try {
|
|
1085
|
-
return await
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1619
|
+
return await work(controller.signal);
|
|
1620
|
+
}
|
|
1621
|
+
catch (err) {
|
|
1622
|
+
if (controller.signal.aborted) {
|
|
1623
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1624
|
+
if (msg === 'aborted' || msg.includes('abort'))
|
|
1625
|
+
throw new Error(label);
|
|
1626
|
+
throw new Error(label);
|
|
1627
|
+
}
|
|
1628
|
+
throw err;
|
|
1091
1629
|
}
|
|
1092
1630
|
finally {
|
|
1093
|
-
|
|
1094
|
-
|
|
1631
|
+
clearTimeout(timer);
|
|
1632
|
+
parent?.removeEventListener('abort', onParentAbort);
|
|
1095
1633
|
}
|
|
1096
1634
|
}
|
|
1097
1635
|
/**
|
|
1098
|
-
*
|
|
1636
|
+
* 外源单文?job:按逻辑?fetch + 索引
|
|
1099
1637
|
*/
|
|
1100
|
-
async fetchDocJob(sourceId, path) {
|
|
1638
|
+
async fetchDocJob(sourceId, path, signal) {
|
|
1101
1639
|
const source = this.sources.get(sourceId);
|
|
1102
1640
|
if (!source || source.status === 'removed')
|
|
1103
1641
|
return;
|
|
1642
|
+
if (signal?.aborted)
|
|
1643
|
+
throw new Error('aborted');
|
|
1104
1644
|
const cred = await this.resolveCred(source);
|
|
1105
1645
|
const existing = this.index.getFile(sourceId, path);
|
|
1106
1646
|
const ref = {
|
|
@@ -1116,21 +1656,21 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1116
1656
|
: this.localFetcher;
|
|
1117
1657
|
const doc = await fetcher.fetch(source, ref, cred);
|
|
1118
1658
|
if (doc)
|
|
1119
|
-
this.indexVirtualDoc(sourceId, doc);
|
|
1659
|
+
await this.indexVirtualDoc(sourceId, doc, source.authRef);
|
|
1120
1660
|
}
|
|
1121
|
-
async parseOne(sourceId, filePath) {
|
|
1661
|
+
async parseOne(sourceId, filePath, signal) {
|
|
1662
|
+
const source = this.sources.get(sourceId);
|
|
1663
|
+
// 防御深度:即使队污染,也?root 外的文件
|
|
1664
|
+
if (source && !this.sourceOwnsPath(source, filePath)) {
|
|
1665
|
+
throw new Error(`path not owned by source: ${filePath}`);
|
|
1666
|
+
}
|
|
1667
|
+
if (signal?.aborted)
|
|
1668
|
+
throw new Error('aborted');
|
|
1122
1669
|
if (this.shouldSkipFile(filePath)) {
|
|
1123
1670
|
this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
|
|
1124
1671
|
return false;
|
|
1125
1672
|
}
|
|
1126
|
-
|
|
1127
|
-
if (!adapter) {
|
|
1128
|
-
if (this.documentPort && isDocumentPath(filePath)) {
|
|
1129
|
-
return this.parseDocumentFile(sourceId, filePath);
|
|
1130
|
-
}
|
|
1131
|
-
this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
|
|
1132
|
-
return false;
|
|
1133
|
-
}
|
|
1673
|
+
// 先确认是件:/节点不得?knowledge_files(历史上?no_adapter?
|
|
1134
1674
|
let st;
|
|
1135
1675
|
try {
|
|
1136
1676
|
st = await stat(filePath);
|
|
@@ -1139,47 +1679,83 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1139
1679
|
this.index.removeFile(sourceId, filePath);
|
|
1140
1680
|
return false;
|
|
1141
1681
|
}
|
|
1142
|
-
if (!st.isFile())
|
|
1682
|
+
if (!st.isFile()) {
|
|
1683
|
+
this.index.removeFile(sourceId, filePath);
|
|
1143
1684
|
return false;
|
|
1144
|
-
|
|
1145
|
-
|
|
1685
|
+
}
|
|
1686
|
+
const adapter = this.adapters.match(filePath);
|
|
1687
|
+
if (!adapter) {
|
|
1688
|
+
// worker DocumentPort;注?port 仅作 worker 不可用时的回
|
|
1689
|
+
if (isDocumentPath(filePath)) {
|
|
1690
|
+
return this.parseDocumentFile(sourceId, filePath, signal);
|
|
1691
|
+
}
|
|
1692
|
+
this.index.markFileSkipped(sourceId, filePath, 'no_adapter', st.size, await this.identityKeyFor(filePath));
|
|
1146
1693
|
return false;
|
|
1147
1694
|
}
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1695
|
+
// Axure 导出 data.js/document.js:内容嗅探后 skip,避免误伤普通 data.js
|
|
1696
|
+
if (/\.(js|cjs|mjs)$/i.test(filePath)) {
|
|
1697
|
+
const head = await this.readFileHead(filePath, 256);
|
|
1698
|
+
if (head && looksLikeAxureExportJs(head)) {
|
|
1699
|
+
this.index.markFileSkipped(sourceId, filePath, 'axure_export_sidecar', st.size, await this.identityKeyFor(filePath));
|
|
1700
|
+
return false;
|
|
1701
|
+
}
|
|
1152
1702
|
}
|
|
1703
|
+
const kind = classifyFileKind(filePath);
|
|
1704
|
+
const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
|
|
1705
|
+
if (sizeDecision.action === 'skip') {
|
|
1706
|
+
this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size, await this.identityKeyFor(filePath));
|
|
1707
|
+
return false;
|
|
1708
|
+
}
|
|
1709
|
+
// 读盘 + hash + 切块 + FTS token 全在 worker(Service 事件循环只做写库)
|
|
1710
|
+
const maxTextChars = this.fileLimits.partial.maxTextChars;
|
|
1711
|
+
if (signal?.aborted)
|
|
1712
|
+
throw new Error('aborted');
|
|
1153
1713
|
try {
|
|
1154
|
-
const
|
|
1155
|
-
|
|
1714
|
+
const { parseTextInWorker } = await import('./parse-text-in-worker.js');
|
|
1715
|
+
const parsed = await parseTextInWorker(filePath, {
|
|
1716
|
+
sizeDecision: sizeDecision.action === 'partial' ? 'partial' : 'ok',
|
|
1717
|
+
maxTextChars,
|
|
1718
|
+
timeoutMs: parseTimeoutForSize(st.size, this.fileLimits),
|
|
1719
|
+
signal,
|
|
1720
|
+
});
|
|
1721
|
+
if (this.index.isFresh(sourceId, filePath, parsed.contentHash)) {
|
|
1722
|
+
return true;
|
|
1723
|
+
}
|
|
1724
|
+
if (signal?.aborted)
|
|
1725
|
+
throw new Error('aborted');
|
|
1726
|
+
const ident = await identifyLocalFile(filePath);
|
|
1727
|
+
await this.index.upsertFile({
|
|
1156
1728
|
sourceId,
|
|
1157
1729
|
path: filePath,
|
|
1158
|
-
contentHash,
|
|
1730
|
+
contentHash: parsed.contentHash,
|
|
1159
1731
|
size: st.size,
|
|
1160
1732
|
mtime: Math.floor(st.mtimeMs),
|
|
1161
|
-
adapterId:
|
|
1162
|
-
chunks,
|
|
1163
|
-
|
|
1733
|
+
adapterId: parsed.adapterId,
|
|
1734
|
+
chunks: parsed.chunks,
|
|
1735
|
+
identityKey: ident.key,
|
|
1736
|
+
}, { signal });
|
|
1164
1737
|
return true;
|
|
1165
1738
|
}
|
|
1166
1739
|
catch (err) {
|
|
1167
|
-
|
|
1740
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1741
|
+
if (msg === 'aborted' || msg.startsWith('text_parse_timeout') || signal?.aborted) {
|
|
1742
|
+
throw err;
|
|
1743
|
+
}
|
|
1744
|
+
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
|
|
1168
1745
|
return false;
|
|
1169
1746
|
}
|
|
1170
1747
|
}
|
|
1171
1748
|
/**
|
|
1172
|
-
* PDF/Office:DocumentPort
|
|
1173
|
-
* 新鲜度以原件字节 hash
|
|
1749
|
+
* PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
|
|
1750
|
+
* 新鲜度以原件字节 hash 为准?
|
|
1174
1751
|
*
|
|
1175
|
-
* @param sourceId -
|
|
1176
|
-
* @param filePath -
|
|
1177
|
-
* @returns
|
|
1752
|
+
* @param sourceId - 知识?id
|
|
1753
|
+
* @param filePath - 文件
|
|
1754
|
+
* @returns 成功入索?
|
|
1178
1755
|
*/
|
|
1179
|
-
async parseDocumentFile(sourceId, filePath) {
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
return false;
|
|
1756
|
+
async parseDocumentFile(sourceId, filePath, signal) {
|
|
1757
|
+
if (signal?.aborted)
|
|
1758
|
+
throw new Error('aborted');
|
|
1183
1759
|
let st;
|
|
1184
1760
|
try {
|
|
1185
1761
|
st = await stat(filePath);
|
|
@@ -1190,53 +1766,88 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1190
1766
|
}
|
|
1191
1767
|
if (!st.isFile())
|
|
1192
1768
|
return false;
|
|
1193
|
-
|
|
1194
|
-
|
|
1769
|
+
const kind = classifyFileKind(filePath);
|
|
1770
|
+
const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
|
|
1771
|
+
if (sizeDecision.action === 'skip') {
|
|
1772
|
+
this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size);
|
|
1195
1773
|
return false;
|
|
1196
1774
|
}
|
|
1197
|
-
const raw = await readFile(filePath);
|
|
1198
|
-
const contentHash = hashContent(raw);
|
|
1199
|
-
if (this.index.isFresh(sourceId, filePath, contentHash)) {
|
|
1200
|
-
return true;
|
|
1201
|
-
}
|
|
1202
1775
|
try {
|
|
1203
|
-
// worker
|
|
1204
|
-
const
|
|
1205
|
-
const
|
|
1206
|
-
if (
|
|
1207
|
-
|
|
1776
|
+
// worker 抽取 + 流式 hash + 切块 + ftsToks:xlsx/pdf 不堵 Service 事件循环
|
|
1777
|
+
const extracted = await this.extractDocument(filePath, sourceId, st.size, signal);
|
|
1778
|
+
const contentHash = extracted.contentHash;
|
|
1779
|
+
if (this.index.isFresh(sourceId, filePath, contentHash)) {
|
|
1780
|
+
return true;
|
|
1781
|
+
}
|
|
1782
|
+
const markdown = extracted.result.markdown?.trim();
|
|
1783
|
+
if (!markdown || extracted.chunks.length === 0) {
|
|
1784
|
+
this.index.markFileSkipped(sourceId, filePath, 'empty_content', st.size);
|
|
1208
1785
|
return false;
|
|
1209
1786
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1787
|
+
if (signal?.aborted)
|
|
1788
|
+
throw new Error('aborted');
|
|
1789
|
+
const ident = await identifyLocalFile(filePath);
|
|
1790
|
+
await this.index.upsertFile({
|
|
1212
1791
|
sourceId,
|
|
1213
1792
|
path: filePath,
|
|
1214
1793
|
contentHash,
|
|
1215
1794
|
size: st.size,
|
|
1216
1795
|
mtime: Math.floor(st.mtimeMs),
|
|
1217
|
-
adapterId: `document:${result.backend}`,
|
|
1218
|
-
chunks,
|
|
1219
|
-
|
|
1796
|
+
adapterId: `document:${extracted.result.backend}`,
|
|
1797
|
+
chunks: extracted.chunks,
|
|
1798
|
+
identityKey: ident.key,
|
|
1799
|
+
}, { signal });
|
|
1220
1800
|
return true;
|
|
1221
1801
|
}
|
|
1222
1802
|
catch (err) {
|
|
1803
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1804
|
+
// /超时不是「这件解析失必须上抛?job ?failed止静?done
|
|
1805
|
+
if (msg === 'aborted' || msg.startsWith('parse_timeout') || signal?.aborted) {
|
|
1806
|
+
throw err;
|
|
1807
|
+
}
|
|
1223
1808
|
if (isDocumentExtractError(err)) {
|
|
1224
|
-
//
|
|
1225
|
-
this.
|
|
1809
|
+
// 结构化跳过(密码/老格?超大等),不挡整?
|
|
1810
|
+
const identKey = await this.identityKeyFor(filePath);
|
|
1811
|
+
this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase(), st.size, identKey);
|
|
1226
1812
|
return false;
|
|
1227
1813
|
}
|
|
1228
|
-
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
|
|
1814
|
+
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
|
|
1229
1815
|
return false;
|
|
1230
1816
|
}
|
|
1231
1817
|
}
|
|
1232
|
-
/**
|
|
1233
|
-
async extractDocument(filePath, sourceId) {
|
|
1234
|
-
const
|
|
1818
|
+
/** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
|
|
1819
|
+
async extractDocument(filePath, sourceId, fileSize = 0, extraSignal) {
|
|
1820
|
+
const sourceSignal = this.abortSignalFor(sourceId);
|
|
1821
|
+
const controller = new AbortController();
|
|
1822
|
+
const abort = () => controller.abort();
|
|
1823
|
+
sourceSignal.addEventListener('abort', abort, { once: true });
|
|
1824
|
+
extraSignal?.addEventListener('abort', abort, { once: true });
|
|
1825
|
+
const signal = controller.signal;
|
|
1826
|
+
const timeoutMs = parseTimeoutForSize(fileSize, this.fileLimits);
|
|
1827
|
+
const extractOpts = {
|
|
1828
|
+
timeoutMs,
|
|
1829
|
+
// knowledge hardMax:软超限仍可 partial 抽取
|
|
1830
|
+
maxFileBytes: this.fileLimits.hardMaxFileBytes,
|
|
1831
|
+
maxSheets: this.fileLimits.partial.maxSheets,
|
|
1832
|
+
maxRowsPerSheet: this.fileLimits.partial.maxRowsPerSheet,
|
|
1833
|
+
maxPages: this.fileLimits.partial.maxPdfPages,
|
|
1834
|
+
maxTextChars: this.fileLimits.partial.maxTextChars,
|
|
1835
|
+
};
|
|
1235
1836
|
try {
|
|
1837
|
+
// 注入的 documentPort 是调用方选定的抽取实现(测试 stub / 自定义后端)
|
|
1838
|
+
if (this.documentPort) {
|
|
1839
|
+
const result = await this.documentPort.extract({ path: filePath, name: filePath }, { ...extractOpts, signal });
|
|
1840
|
+
const markdown = result.markdown?.trim() ?? '';
|
|
1841
|
+
const chunks = markdown
|
|
1842
|
+
? await this.chunkTextInWorker(markdown, filePath, signal)
|
|
1843
|
+
: [];
|
|
1844
|
+
const contentHash = await hashFileStreaming(filePath, signal);
|
|
1845
|
+
return { result, chunks, contentHash };
|
|
1846
|
+
}
|
|
1236
1847
|
const { extractDocumentInWorker } = await import('./extract-document.js');
|
|
1237
1848
|
return await extractDocumentInWorker(filePath, {
|
|
1238
|
-
|
|
1239
|
-
|
|
1849
|
+
...extractOpts,
|
|
1850
|
+
documentConfig: this.documentConfig,
|
|
1240
1851
|
signal,
|
|
1241
1852
|
});
|
|
1242
1853
|
}
|
|
@@ -1245,93 +1856,34 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1245
1856
|
if (msg === 'extract_aborted' || signal.aborted) {
|
|
1246
1857
|
throw new Error('aborted');
|
|
1247
1858
|
}
|
|
1248
|
-
if (msg.includes('Cannot find module') || msg.includes('ERR_MODULE')) {
|
|
1249
|
-
return this.documentPort.extract({ path: filePath, name: filePath }, { timeoutMs: this.parseTimeoutMs });
|
|
1250
|
-
}
|
|
1251
1859
|
throw err;
|
|
1252
1860
|
}
|
|
1861
|
+
finally {
|
|
1862
|
+
sourceSignal.removeEventListener('abort', abort);
|
|
1863
|
+
extraSignal?.removeEventListener('abort', abort);
|
|
1864
|
+
}
|
|
1865
|
+
}
|
|
1866
|
+
/** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
|
|
1867
|
+
async chunkTextInWorker(text, path, signal, adapterId) {
|
|
1868
|
+
const { chunkTextWithFtsToks } = await import('./parse-text-in-worker.js');
|
|
1869
|
+
return chunkTextWithFtsToks(text, path, signal, adapterId);
|
|
1253
1870
|
}
|
|
1254
1871
|
/**
|
|
1255
1872
|
* Phase B:嵌入待处理 chunk
|
|
1256
1873
|
*
|
|
1257
|
-
*
|
|
1258
|
-
*
|
|
1874
|
+
* ?job 内按 embedConcurrency ****多批调用 API?
|
|
1875
|
+
* 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
|
|
1259
1876
|
*/
|
|
1260
1877
|
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
1261
|
-
|
|
1262
|
-
if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
|
|
1263
|
-
throw new Error('embedding_provider_missing');
|
|
1264
|
-
}
|
|
1265
|
-
return;
|
|
1266
|
-
}
|
|
1267
|
-
// 磁盘对齐在 reindex/walk 做;embed 任务内不再全量 prune(曾拖慢解析)
|
|
1268
|
-
const lanes = Math.max(1, this.embedConcurrency);
|
|
1269
|
-
const batchSize = Math.max(1, this.embedBatch);
|
|
1270
|
-
for (let round = 0; round < maxRounds; round++) {
|
|
1271
|
-
if (this.isAborted(sourceId))
|
|
1272
|
-
return;
|
|
1273
|
-
const pending = this.index.listChunksMissingEmbedding(sourceId, batchSize * lanes);
|
|
1274
|
-
if (pending.length === 0) {
|
|
1275
|
-
this.refreshCoverage(sourceId);
|
|
1276
|
-
return;
|
|
1277
|
-
}
|
|
1278
|
-
if (jobId)
|
|
1279
|
-
this.heartbeatJob(jobId);
|
|
1280
|
-
// 轮次限速(并行片共享同一闸门)
|
|
1281
|
-
if (this.embedMinIntervalMs > 0) {
|
|
1282
|
-
const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
|
|
1283
|
-
if (wait > 0)
|
|
1284
|
-
await new Promise((r) => setTimeout(r, wait));
|
|
1285
|
-
}
|
|
1286
|
-
const slices = [];
|
|
1287
|
-
for (let i = 0; i < pending.length; i += batchSize) {
|
|
1288
|
-
slices.push(pending.slice(i, i + batchSize));
|
|
1289
|
-
}
|
|
1290
|
-
const settled = await Promise.allSettled(slices.map((slice) => this.embedOneSlice(sourceId, slice)));
|
|
1291
|
-
let ok = 0;
|
|
1292
|
-
let failed = false;
|
|
1293
|
-
for (const s of settled) {
|
|
1294
|
-
if (s.status === 'fulfilled')
|
|
1295
|
-
ok += s.value;
|
|
1296
|
-
else
|
|
1297
|
-
failed = true;
|
|
1298
|
-
}
|
|
1299
|
-
this.lastEmbedAt = Date.now();
|
|
1300
|
-
this.emitProgress({
|
|
1301
|
-
type: 'embed',
|
|
1302
|
-
sourceId,
|
|
1303
|
-
status: 'batch_done',
|
|
1304
|
-
detail: `embedded=${ok}/${pending.length} lanes=${slices.length}`,
|
|
1305
|
-
});
|
|
1306
|
-
if (failed) {
|
|
1307
|
-
// 至少一片失败:交给 done 后的 ensureEmbedJob 续跑,不把整 job 打成 failed
|
|
1308
|
-
return;
|
|
1309
|
-
}
|
|
1310
|
-
}
|
|
1311
|
-
this.refreshCoverage(sourceId);
|
|
1878
|
+
await this.embedRunner.embedPending(sourceId, maxRounds, jobId);
|
|
1312
1879
|
}
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
this.embedding.embedBatch(slice.map((p) => p.text)),
|
|
1319
|
-
new Promise((_, reject) => setTimeout(() => reject(new Error('embed_batch_timeout')), 120_000)),
|
|
1320
|
-
]);
|
|
1321
|
-
let ok = 0;
|
|
1322
|
-
const batch = [];
|
|
1323
|
-
for (let i = 0; i < slice.length; i++) {
|
|
1324
|
-
const vec = vectors[i];
|
|
1325
|
-
if (vec?.length) {
|
|
1326
|
-
batch.push([slice[i].id, vec]);
|
|
1327
|
-
ok += 1;
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
if (batch.length > 0)
|
|
1331
|
-
this.index.setChunkEmbeddings(batch);
|
|
1332
|
-
return ok;
|
|
1880
|
+
recordEmbedSecretHit(input) {
|
|
1881
|
+
this.sources.database.raw
|
|
1882
|
+
.prepare(`INSERT INTO knowledge_embed_secret_log (id, source_id, chunk_id, action, hits_json, created_at)
|
|
1883
|
+
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
1884
|
+
.run(`kes_${randomUUID().slice(0, 12)}`, input.sourceId, input.chunkId, input.action, JSON.stringify(input.hits), Date.now());
|
|
1333
1885
|
}
|
|
1334
|
-
/**
|
|
1886
|
+
/** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
|
|
1335
1887
|
ensureEmbedJob(sourceId) {
|
|
1336
1888
|
if (!this.embedding)
|
|
1337
1889
|
return;
|
|
@@ -1339,21 +1891,42 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1339
1891
|
return;
|
|
1340
1892
|
if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
|
|
1341
1893
|
return;
|
|
1342
|
-
this.enqueue(sourceId, 'embed_source', null, 3)
|
|
1343
|
-
|
|
1894
|
+
void this.enqueue(sourceId, 'embed_source', null, 3)
|
|
1895
|
+
.then(() => this.kick())
|
|
1896
|
+
.catch(() => undefined);
|
|
1344
1897
|
}
|
|
1345
1898
|
heartbeatJob(jobId) {
|
|
1346
|
-
this.
|
|
1347
|
-
|
|
1348
|
-
|
|
1899
|
+
this.jobQueue.heartbeat(jobId);
|
|
1900
|
+
}
|
|
1901
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
|
|
1902
|
+
coverageDirty = new Set();
|
|
1903
|
+
markCoverageDirty(sourceId) {
|
|
1904
|
+
this.coverageDirty.add(sourceId);
|
|
1905
|
+
}
|
|
1906
|
+
flushCoverageDirty(sourceId) {
|
|
1907
|
+
const ids = sourceId ? [sourceId] : [...this.coverageDirty];
|
|
1908
|
+
if (sourceId)
|
|
1909
|
+
this.coverageDirty.delete(sourceId);
|
|
1910
|
+
else
|
|
1911
|
+
this.coverageDirty.clear();
|
|
1912
|
+
for (const id of ids)
|
|
1913
|
+
this.refreshCoverage(id, { force: true });
|
|
1349
1914
|
}
|
|
1350
|
-
|
|
1915
|
+
lastCoverageRefreshAt = new Map();
|
|
1916
|
+
refreshCoverage(sourceId, opts) {
|
|
1917
|
+
const now = Date.now();
|
|
1918
|
+
const last = this.lastCoverageRefreshAt.get(sourceId) ?? 0;
|
|
1919
|
+
// 大库 sourceStats/embeddingCoverage 是多表 COUNT;解析/嵌入高频调用时节流,
|
|
1920
|
+
// 避免 Engine 线程被统计查询堵住导致 HTTP list 超时。收尾路径 force。
|
|
1921
|
+
if (!opts?.force && now - last < 2_000)
|
|
1922
|
+
return;
|
|
1923
|
+
this.lastCoverageRefreshAt.set(sourceId, now);
|
|
1351
1924
|
const stats = this.index.sourceStats(sourceId);
|
|
1352
1925
|
const source = this.sources.get(sourceId);
|
|
1353
1926
|
if (!source)
|
|
1354
1927
|
return;
|
|
1355
1928
|
const discovered = this.discoveredFiles.get(sourceId) ?? 0;
|
|
1356
|
-
// Phase A:已落库文件(indexed+skipped+error
|
|
1929
|
+
// Phase A:已落库文件(indexed+skipped+error? walk 发现?
|
|
1357
1930
|
const phaseA = discovered > 0
|
|
1358
1931
|
? Math.min(1, stats.files / discovered)
|
|
1359
1932
|
: stats.files > 0
|
|
@@ -1389,10 +1962,29 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1389
1962
|
});
|
|
1390
1963
|
}
|
|
1391
1964
|
emitProgress(evt) {
|
|
1392
|
-
|
|
1393
|
-
|
|
1965
|
+
let out = evt;
|
|
1966
|
+
if (evt.sourceId && evt.registeredBy === undefined) {
|
|
1967
|
+
try {
|
|
1968
|
+
const row = this.sources.database.raw
|
|
1969
|
+
.prepare('SELECT tenant_id, registered_by, visibility FROM knowledge_sources WHERE id = ?')
|
|
1970
|
+
.get(evt.sourceId);
|
|
1971
|
+
if (row) {
|
|
1972
|
+
out = {
|
|
1973
|
+
...evt,
|
|
1974
|
+
tenantId: row.tenant_id,
|
|
1975
|
+
registeredBy: row.registered_by,
|
|
1976
|
+
visibility: row.visibility,
|
|
1977
|
+
};
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
catch {
|
|
1981
|
+
/* 源已删时保持原事件;SSE 侧按无归属放行到 fallback */
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
this.emit('progress', out);
|
|
1985
|
+
this.emit('knowledge.index.progress', out);
|
|
1394
1986
|
}
|
|
1395
|
-
/**
|
|
1987
|
+
/** 磁盘水位(仅,不务) */
|
|
1396
1988
|
async warnDiskWatermark(sourceId) {
|
|
1397
1989
|
try {
|
|
1398
1990
|
const { statfs } = await import('node:fs/promises');
|
|
@@ -1405,7 +1997,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
1405
1997
|
type: 'error',
|
|
1406
1998
|
sourceId,
|
|
1407
1999
|
status: 'disk_watermark',
|
|
1408
|
-
detail: `free
|
|
2000
|
+
detail: `free?{Math.round(freeMb)}MB (jobs kept)`,
|
|
1409
2001
|
});
|
|
1410
2002
|
}
|
|
1411
2003
|
}
|