@octopi-agent/engine 0.56.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +43 -1
- package/dist/config-schema/substrate.d.ts +42 -0
- package/dist/config-schema/substrate.js +58 -0
- package/dist/config.d.ts +39 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +5 -3
- package/dist/harness/index.js +3 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +42 -3
- package/dist/harness/knowledge/db.js +466 -83
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +68 -0
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +33 -0
- package/dist/harness/knowledge/extract-document.js +86 -0
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +166 -63
- package/dist/harness/knowledge/index-store.js +1025 -282
- package/dist/harness/knowledge/index.d.ts +47 -5
- package/dist/harness/knowledge/index.js +24 -2
- package/dist/harness/knowledge/ingest.d.ts +322 -49
- package/dist/harness/knowledge/ingest.js +1383 -267
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +30 -5
- package/dist/harness/knowledge/retriever.js +98 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +83 -16
- package/dist/harness/knowledge/source-store.js +205 -78
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
- package/dist/harness/memory/sqlite/keyword-search.js +104 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -1,29 +1,72 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIngest
|
|
2
|
+
* KnowledgeIngest ?解析/切块/索引队列(P2 Phase A?
|
|
3
3
|
*
|
|
4
|
-
* 原则:ingest
|
|
4
|
+
* 原则:ingest 话并发,不作 turn 前置;队列背压不务;source 级锁?
|
|
5
5
|
*/
|
|
6
|
-
import { randomUUID } from 'node:crypto';
|
|
7
|
-
import {
|
|
6
|
+
import { randomUUID, createHash } from 'node:crypto';
|
|
7
|
+
import { open, stat } from 'node:fs/promises';
|
|
8
|
+
import { createReadStream, statSync, realpathSync } from 'node:fs';
|
|
8
9
|
import { watch } from 'node:fs';
|
|
9
|
-
import { join, sep } from 'node:path';
|
|
10
|
+
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
10
11
|
import { EventEmitter } from 'node:events';
|
|
11
|
-
import { FormatAdapterRegistry, htmlAdapter, markdownAdapter, textAdapter } from './adapters.js';
|
|
12
|
+
import { FormatAdapterRegistry, htmlAdapter, looksLikeAxureExportJs, markdownAdapter, textAdapter, } from './adapters.js';
|
|
12
13
|
import { looksLikeHtml } from './html.js';
|
|
13
14
|
import { hashContent, KnowledgeIndexStore } from './index-store.js';
|
|
14
|
-
import {
|
|
15
|
+
import { identifyLocalFile, urlIdentityKey, normalizePathLexical, } from './file-identity.js';
|
|
16
|
+
import { KnowledgeJobControl } from './job-control.js';
|
|
17
|
+
import { LocalFsFetcher, UrlFetcher, } from './fetchers.js';
|
|
15
18
|
import { ConnectorRegistry } from './connectors.js';
|
|
16
19
|
import { ConnectorFetcher } from './connector-fetcher.js';
|
|
17
|
-
import { isDocumentPath } from '../
|
|
18
|
-
import { isDocumentExtractError } from '../
|
|
20
|
+
import { isDocumentPath } from '../capabilities/document/format.js';
|
|
21
|
+
import { isDocumentExtractError } from '../capabilities/document/errors.js';
|
|
22
|
+
import { classifyFileKind, decideBySize, isRetryableSkipReason, parseTimeoutForSize, resolveKnowledgeFileLimits, } from './file-limits.js';
|
|
23
|
+
import { EmbedRunner } from './embed-runner.js';
|
|
24
|
+
/**
|
|
25
|
+
* 流式 SHA-256(可中止):禁止在 Engine 事件循环 `readFile` 整文件再 hash。
|
|
26
|
+
*
|
|
27
|
+
* @param filePath - 本地文件
|
|
28
|
+
* @param signal - 中止信号
|
|
29
|
+
* @returns hex digest
|
|
30
|
+
*/
|
|
31
|
+
export async function hashFileStreaming(filePath, signal) {
|
|
32
|
+
const hash = createHash('sha256');
|
|
33
|
+
await new Promise((resolve, reject) => {
|
|
34
|
+
if (signal?.aborted) {
|
|
35
|
+
reject(new Error('aborted'));
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const stream = createReadStream(filePath);
|
|
39
|
+
const onAbort = () => {
|
|
40
|
+
stream.destroy();
|
|
41
|
+
reject(new Error('aborted'));
|
|
42
|
+
};
|
|
43
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
44
|
+
stream.on('data', (chunk) => hash.update(chunk));
|
|
45
|
+
stream.once('end', () => {
|
|
46
|
+
signal?.removeEventListener('abort', onAbort);
|
|
47
|
+
resolve();
|
|
48
|
+
});
|
|
49
|
+
stream.once('error', (err) => {
|
|
50
|
+
signal?.removeEventListener('abort', onAbort);
|
|
51
|
+
reject(err);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
return hash.digest('hex');
|
|
55
|
+
}
|
|
56
|
+
import { JobQueue } from './job-queue.js';
|
|
57
|
+
import { readJobControlState } from './job-control-state.js';
|
|
19
58
|
export class KnowledgeIngest extends EventEmitter {
|
|
20
59
|
sources;
|
|
21
60
|
index;
|
|
22
61
|
adapters;
|
|
23
62
|
parseConcurrency;
|
|
63
|
+
documentParseConcurrency;
|
|
24
64
|
maxQueueDepth;
|
|
25
65
|
debounceMs;
|
|
26
66
|
maxFileBytes;
|
|
67
|
+
fileLimits;
|
|
68
|
+
/** 单文?parse/fetch 超时(默?45s;大 Office 抽取会堵事件?*/
|
|
69
|
+
parseTimeoutMs;
|
|
27
70
|
embedding;
|
|
28
71
|
embedBatch;
|
|
29
72
|
embedMinIntervalMs;
|
|
@@ -37,14 +80,29 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
37
80
|
pollMinIntervalMs;
|
|
38
81
|
maxPollPerTick;
|
|
39
82
|
documentPort;
|
|
40
|
-
|
|
83
|
+
documentConfig;
|
|
84
|
+
embedSecretPolicy;
|
|
85
|
+
embedRunner;
|
|
41
86
|
pollTimer = null;
|
|
42
|
-
|
|
87
|
+
reconcileTimer = null;
|
|
88
|
+
/** 源级:DB 权威 + ?AbortSignal(跨重启?*/
|
|
89
|
+
jobControl;
|
|
90
|
+
jobQueue;
|
|
91
|
+
/** 源任务稳定(?queued/running)时回调 ?用于 auto-describe 等收?*/
|
|
92
|
+
onSourceSettled;
|
|
93
|
+
/** parse 缺口时间 */
|
|
94
|
+
lastParseGapScanAt = new Map();
|
|
95
|
+
/** watch 突发后要求尽快缺口补扫(覆盖 60s 节流) */
|
|
96
|
+
gapScanUrgent = new Set();
|
|
97
|
+
gapScanTimer = null;
|
|
98
|
+
runningParse = 0;
|
|
99
|
+
runningDocParse = 0;
|
|
43
100
|
runningEmbed = 0;
|
|
44
101
|
draining = false;
|
|
45
|
-
|
|
102
|
+
kickAgain = false;
|
|
103
|
+
/** source 级锁:同 source 不并?walk/parse 冲突?*/
|
|
46
104
|
sourceLocks = new Set();
|
|
47
|
-
/** walk 发现的文件数(Phase A coverage
|
|
105
|
+
/** walk 发现的文件数(Phase A coverage 分母?*/
|
|
48
106
|
discoveredFiles = new Map();
|
|
49
107
|
watchers = new Map();
|
|
50
108
|
debounceTimers = new Map();
|
|
@@ -52,19 +110,68 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
52
110
|
constructor(options) {
|
|
53
111
|
super();
|
|
54
112
|
this.sources = options.sourceStore;
|
|
113
|
+
this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
|
|
114
|
+
this.jobControl = new KnowledgeJobControl(options.sourceStore.database);
|
|
115
|
+
this.jobQueue = new JobQueue({
|
|
116
|
+
db: options.sourceStore.database,
|
|
117
|
+
maxQueueDepth: this.maxQueueDepth,
|
|
118
|
+
isAborted: (id) => this.isAborted(id),
|
|
119
|
+
onBackpressure: (sourceId, queued) => {
|
|
120
|
+
this.emitProgress({
|
|
121
|
+
type: 'error',
|
|
122
|
+
sourceId,
|
|
123
|
+
status: 'queue_backpressure',
|
|
124
|
+
detail: `queued=${queued} (delay only, jobs kept)`,
|
|
125
|
+
});
|
|
126
|
+
},
|
|
127
|
+
});
|
|
55
128
|
this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
|
|
56
129
|
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
57
|
-
this.parseConcurrency = options.parseConcurrency ??
|
|
58
|
-
this.
|
|
59
|
-
|
|
60
|
-
this.
|
|
130
|
+
this.parseConcurrency = options.parseConcurrency ?? 8;
|
|
131
|
+
this.documentParseConcurrency = options.documentParseConcurrency ?? 2;
|
|
132
|
+
// watch 批量落盘时缩默窗,新增文件更?
|
|
133
|
+
this.debounceMs = options.debounceMs ?? 800;
|
|
134
|
+
this.fileLimits = resolveKnowledgeFileLimits({
|
|
135
|
+
...options.fileLimits,
|
|
136
|
+
maxFileBytes: options.fileLimits?.maxFileBytes ?? options.maxFileBytes,
|
|
137
|
+
parseTimeoutMs: options.fileLimits?.parseTimeoutMs ?? options.parseTimeoutMs,
|
|
138
|
+
});
|
|
139
|
+
this.maxFileBytes = this.fileLimits.maxFileBytes;
|
|
140
|
+
this.parseTimeoutMs = this.fileLimits.parseTimeoutMs;
|
|
61
141
|
this.embedding = options.embeddingProvider ?? null;
|
|
62
142
|
this.embedBatch = options.embedBatch ?? 32;
|
|
63
143
|
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
64
|
-
|
|
144
|
+
// embed ?parse 分槽;缺?4 行打 embedding API
|
|
145
|
+
this.embedConcurrency = options.embedConcurrency ?? 4;
|
|
65
146
|
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
66
147
|
this.credentials = options.credentials ?? null;
|
|
67
148
|
this.documentPort = options.documentPort ?? null;
|
|
149
|
+
this.documentConfig = options.documentConfig ?? null;
|
|
150
|
+
this.embedSecretPolicy = options.embedSecretPolicy ?? 'redact';
|
|
151
|
+
this.embedRunner = new EmbedRunner({
|
|
152
|
+
index: this.index,
|
|
153
|
+
embedding: this.embedding,
|
|
154
|
+
embedBatch: this.embedBatch,
|
|
155
|
+
embedConcurrency: this.embedConcurrency,
|
|
156
|
+
embedMinIntervalMs: this.embedMinIntervalMs,
|
|
157
|
+
embedSecretPolicy: this.embedSecretPolicy,
|
|
158
|
+
isAborted: (id) => this.isAborted(id),
|
|
159
|
+
heartbeatJob: (id) => this.heartbeatJob(id),
|
|
160
|
+
refreshCoverage: (id) => this.refreshCoverage(id),
|
|
161
|
+
emitProgress: (evt) => this.emitProgress(evt),
|
|
162
|
+
recordSecretHit: (input) => this.recordEmbedSecretHit(input),
|
|
163
|
+
});
|
|
164
|
+
// embedding 外发(合规可查;非权量)
|
|
165
|
+
this.sources.database.raw.exec(`
|
|
166
|
+
CREATE TABLE IF NOT EXISTS knowledge_embed_secret_log (
|
|
167
|
+
id TEXT PRIMARY KEY,
|
|
168
|
+
source_id TEXT NOT NULL,
|
|
169
|
+
chunk_id TEXT NOT NULL,
|
|
170
|
+
action TEXT NOT NULL,
|
|
171
|
+
hits_json TEXT NOT NULL,
|
|
172
|
+
created_at INTEGER NOT NULL
|
|
173
|
+
);
|
|
174
|
+
`);
|
|
68
175
|
this.localFetcher =
|
|
69
176
|
options.fetchers?.local ?? new LocalFsFetcher((p) => this.shouldSkipFile(p));
|
|
70
177
|
this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
|
|
@@ -74,25 +181,157 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
74
181
|
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
75
182
|
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
76
183
|
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
184
|
+
// 崩溃/重启遗留?running 会堵?enqueue 去重并永久显?indexing
|
|
185
|
+
this.reclaimOrphanRunningJobs();
|
|
186
|
+
// 历史:目录等非文件曾?no_adapter,启动即?
|
|
187
|
+
this.cleanupNonFileIndexRows();
|
|
188
|
+
}
|
|
189
|
+
lastCleanupAt = 0;
|
|
190
|
+
/**
|
|
191
|
+
* 清掉「非?knowledge_files (目录曾 no_adapter?
|
|
192
|
+
*
|
|
193
|
+
* ?skipped/error 且无 chunk :有 chunk 的仍?pruneMissing/磁盘对齐?
|
|
194
|
+
* 60s 节流,避免每?reconcile ?stat?
|
|
195
|
+
*
|
|
196
|
+
* @param sourceId - 限定源;省略则全部源
|
|
197
|
+
* @param opts.force - 忽略节流
|
|
198
|
+
* @returns 删除行数
|
|
199
|
+
*/
|
|
200
|
+
cleanupNonFileIndexRows(sourceId, opts) {
|
|
201
|
+
const now = Date.now();
|
|
202
|
+
if (!opts?.force && sourceId == null && now - this.lastCleanupAt < 60_000) {
|
|
203
|
+
return 0;
|
|
204
|
+
}
|
|
205
|
+
this.lastCleanupAt = now;
|
|
206
|
+
const ids = sourceId
|
|
207
|
+
? [sourceId]
|
|
208
|
+
: this.sources.list().map((s) => s.id);
|
|
209
|
+
let removed = 0;
|
|
210
|
+
for (const id of ids) {
|
|
211
|
+
const rows = this.sources.database.raw
|
|
212
|
+
.prepare(`SELECT m.logical_path AS path
|
|
213
|
+
FROM knowledge_memberships m
|
|
214
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
215
|
+
WHERE m.source_id = ? AND f.status IN ('skipped', 'error') AND f.chunk_count = 0`)
|
|
216
|
+
.all(id);
|
|
217
|
+
for (const r of rows) {
|
|
218
|
+
try {
|
|
219
|
+
const st = statSync(r.path);
|
|
220
|
+
if (!st.isFile()) {
|
|
221
|
+
this.index.removeFile(id, r.path);
|
|
222
|
+
removed += 1;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
catch {
|
|
226
|
+
// 不存留给 pruneMissing齐),避删有效但暂时不可见的文件
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return removed;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* 无跨进程 Lease:本进程时把历史 running 收回 queued?
|
|
234
|
+
* 仅构造时调用行中同进程的 running 法的?
|
|
235
|
+
*/
|
|
236
|
+
reclaimOrphanRunningJobs() {
|
|
237
|
+
this.jobQueue.reclaimOrphanRunning();
|
|
77
238
|
}
|
|
78
239
|
get indexStore() {
|
|
79
240
|
return this.index;
|
|
80
241
|
}
|
|
81
242
|
/**
|
|
82
|
-
*
|
|
243
|
+
* 过滤:文档扩展名交给 worker 抽取,不 BINARY ?
|
|
83
244
|
*
|
|
84
|
-
* @param p -
|
|
85
|
-
* @returns
|
|
245
|
+
* @param p - 文件
|
|
246
|
+
* @returns 跳过
|
|
86
247
|
*/
|
|
87
248
|
shouldSkipFile(p) {
|
|
88
|
-
if (
|
|
249
|
+
if (isDocumentPath(p))
|
|
89
250
|
return false;
|
|
90
251
|
return this.adapters.shouldSkipPath(p);
|
|
91
252
|
}
|
|
253
|
+
/** 读文件头做内容嗅探;失败返回 null(不挡后续 parse) */
|
|
254
|
+
async readFileHead(filePath, bytes = 256) {
|
|
255
|
+
try {
|
|
256
|
+
const fh = await open(filePath, 'r');
|
|
257
|
+
try {
|
|
258
|
+
const buf = Buffer.alloc(bytes);
|
|
259
|
+
const { bytesRead } = await fh.read(buf, 0, bytes, 0);
|
|
260
|
+
return buf.subarray(0, bytesRead).toString('utf8');
|
|
261
|
+
}
|
|
262
|
+
finally {
|
|
263
|
+
await fh.close();
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
catch {
|
|
267
|
+
return null;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/** 尽量拿真实 identity;失败返回 undefined(退回 path: 降级键) */
|
|
271
|
+
async identityKeyFor(filePath) {
|
|
272
|
+
try {
|
|
273
|
+
return (await identifyLocalFile(filePath)).key;
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
return undefined;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* 归属:`parse_file` / 读文件只读源 root 内的?
|
|
281
|
+
* ?**realpath** 比较,防符号链接逃出 root?
|
|
282
|
+
*
|
|
283
|
+
* @param source - 知识?
|
|
284
|
+
* @param path - 请求(本地绝对路径,
|
|
285
|
+
*/
|
|
286
|
+
sourceOwnsPath(source, path) {
|
|
287
|
+
if (source.kind === 'url' || source.kind === 'connector') {
|
|
288
|
+
return Boolean(this.index.getFile(source.id, path));
|
|
289
|
+
}
|
|
290
|
+
const root = resolve(source.location);
|
|
291
|
+
const target = resolve(path);
|
|
292
|
+
const eq = (a, b) => process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b;
|
|
293
|
+
// 词法(路径不存在时也拒绝明显越界?
|
|
294
|
+
if (!eq(root, target)) {
|
|
295
|
+
const relLex = relative(root, target);
|
|
296
|
+
if (!relLex || relLex.startsWith('..') || isAbsolute(relLex))
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
// realpath ?symlink stat 失败回词法结果)
|
|
300
|
+
try {
|
|
301
|
+
const rootReal = realpathSync(root);
|
|
302
|
+
const targetReal = realpathSync(target);
|
|
303
|
+
if (eq(rootReal, targetReal))
|
|
304
|
+
return true;
|
|
305
|
+
const rel = relative(rootReal, targetReal);
|
|
306
|
+
return Boolean(rel) && !rel.startsWith('..') && !isAbsolute(rel);
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
return true; // 词法;文件尚在时交给后续 stat
|
|
310
|
+
}
|
|
311
|
+
}
|
|
92
312
|
/**
|
|
93
|
-
* 全量/增量索引某源(Phase A
|
|
313
|
+
* 全量/增量索引某源(Phase A?
|
|
314
|
+
*
|
|
315
|
+
* @param opts.full - 显式重建(supersede 成任务);poll/增量不打?
|
|
316
|
+
* @param opts.incremental - 增量(poll):?active 任务则跳过本?
|
|
94
317
|
*/
|
|
95
318
|
async ingestSource(sourceId, opts) {
|
|
319
|
+
// fromQueue(walk_source):不 supersede、不跳过 active;reindex 才作废队列
|
|
320
|
+
if (opts?.fromQueue) {
|
|
321
|
+
// fall through to walk
|
|
322
|
+
}
|
|
323
|
+
else if (opts?.full !== false && !opts?.incremental) {
|
|
324
|
+
this.supersedeSourceWork(sourceId);
|
|
325
|
+
}
|
|
326
|
+
else if (this.countActiveJobs(sourceId).total > 0) {
|
|
327
|
+
this.emitProgress({
|
|
328
|
+
type: 'source',
|
|
329
|
+
sourceId,
|
|
330
|
+
status: 'poll_skipped',
|
|
331
|
+
detail: 'active jobs in flight',
|
|
332
|
+
});
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
96
335
|
const source = this.sources.get(sourceId);
|
|
97
336
|
if (!source || source.status === 'removed') {
|
|
98
337
|
throw new Error(`knowledge source not found: ${sourceId}`);
|
|
@@ -114,24 +353,34 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
114
353
|
return;
|
|
115
354
|
}
|
|
116
355
|
if (this.sourceLocks.has(sourceId)) {
|
|
117
|
-
//
|
|
118
|
-
this.enqueue(sourceId, 'walk_source', null, 1
|
|
356
|
+
// 已有同源任务在跑:入?walk 即可(去重)
|
|
357
|
+
await this.enqueue(sourceId, 'walk_source', null, 1);
|
|
119
358
|
return;
|
|
120
359
|
}
|
|
121
360
|
this.sourceLocks.add(sourceId);
|
|
122
361
|
try {
|
|
123
|
-
|
|
362
|
+
// 保留旧 coverage:重建期间索引仍在库中,写 0 会让 UI 误判为清空
|
|
363
|
+
this.sources.update(sourceId, { status: 'discovering' });
|
|
124
364
|
this.emitProgress({ type: 'source', sourceId, status: 'discovering' });
|
|
125
|
-
// full 不再 clearSource
|
|
365
|
+
// full 不再 clearSource:靠 keepPaths prune,discover 失败不丢?
|
|
126
366
|
if (source.kind === 'url' || source.kind === 'connector') {
|
|
127
367
|
await this.ingestRemoteSource(source);
|
|
128
368
|
return;
|
|
129
369
|
}
|
|
130
|
-
|
|
370
|
+
// watch 策略索引同时挂上 fs 监听(幂等;否则后续变更静默丢失?
|
|
371
|
+
this.startWatch(sourceId);
|
|
372
|
+
const localDiscovery = await this.localFetcher.discover(source);
|
|
373
|
+
const files = localDiscovery.docs;
|
|
131
374
|
this.discoveredFiles.set(sourceId, files.length);
|
|
132
|
-
//
|
|
133
|
-
|
|
134
|
-
|
|
375
|
+
// 后不 prune、不入队(半?keep 删)
|
|
376
|
+
if (this.isAborted(sourceId))
|
|
377
|
+
return;
|
|
378
|
+
// prune:仅 walk 完整时执行;出错/?keep 不完整会
|
|
379
|
+
if (localDiscovery.complete) {
|
|
380
|
+
const localKeep = new Set(files.map((f) => f.path));
|
|
381
|
+
// walk 成功且目录为空时允许清空 Membership(§6.2)
|
|
382
|
+
await this.index.pruneMissingAsync(sourceId, localKeep, { allowEmptyKeep: true });
|
|
383
|
+
}
|
|
135
384
|
this.sources.update(sourceId, { status: 'partial', coverage: 0 });
|
|
136
385
|
this.emitProgress({
|
|
137
386
|
type: 'source',
|
|
@@ -140,10 +389,14 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
140
389
|
detail: `${files.length} files`,
|
|
141
390
|
});
|
|
142
391
|
for (const file of files) {
|
|
143
|
-
this.
|
|
392
|
+
if (this.isAborted(sourceId))
|
|
393
|
+
return;
|
|
394
|
+
if (this.isUnchangedIndexed(sourceId, file))
|
|
395
|
+
continue;
|
|
396
|
+
await this.enqueue(sourceId, 'parse_file', file.path, 2);
|
|
144
397
|
}
|
|
145
|
-
if (this.embedding) {
|
|
146
|
-
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
398
|
+
if (this.embedding && !this.isAborted(sourceId)) {
|
|
399
|
+
await this.enqueue(sourceId, 'embed_source', null, 3);
|
|
147
400
|
}
|
|
148
401
|
this.kick();
|
|
149
402
|
}
|
|
@@ -151,6 +404,25 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
151
404
|
this.sourceLocks.delete(sourceId);
|
|
152
405
|
}
|
|
153
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* 入队层廉价变更检测:已完整索引且 size/mtime 未变则跳过 parse_file。
|
|
409
|
+
* 仅对 `status='indexed' && chunk_count>0 && content_hash` 生效——
|
|
410
|
+
* indexing/error/半成品必须重做,避免把未完成当成新鲜。
|
|
411
|
+
*/
|
|
412
|
+
isUnchangedIndexed(sourceId, ref) {
|
|
413
|
+
if (ref.size == null || ref.mtime == null)
|
|
414
|
+
return false;
|
|
415
|
+
const existing = this.index.getFile(sourceId, ref.path);
|
|
416
|
+
if (!existing)
|
|
417
|
+
return false;
|
|
418
|
+
if (existing.status !== 'indexed')
|
|
419
|
+
return false;
|
|
420
|
+
if (existing.chunkCount <= 0)
|
|
421
|
+
return false;
|
|
422
|
+
if (!existing.contentHash)
|
|
423
|
+
return false;
|
|
424
|
+
return existing.size === ref.size && existing.mtime === ref.mtime;
|
|
425
|
+
}
|
|
154
426
|
/**
|
|
155
427
|
* URL / connector 源:discover + fetch + 索引
|
|
156
428
|
*/
|
|
@@ -170,9 +442,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
170
442
|
return;
|
|
171
443
|
}
|
|
172
444
|
const fetcher = source.kind === 'connector' ? this.connectorFetcher : this.urlFetcher;
|
|
173
|
-
let
|
|
445
|
+
let discovered;
|
|
174
446
|
try {
|
|
175
|
-
|
|
447
|
+
discovered = await fetcher.discover(source, cred);
|
|
176
448
|
}
|
|
177
449
|
catch (err) {
|
|
178
450
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -183,7 +455,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
183
455
|
this.emitProgress({ type: 'error', sourceId: source.id, status: 'discover_failed', detail: msg });
|
|
184
456
|
return;
|
|
185
457
|
}
|
|
186
|
-
|
|
458
|
+
const refs = discovered.docs;
|
|
459
|
+
// 条件 GET:带上已?etag/Last-Modified?04 则整文档跳过
|
|
187
460
|
const enriched = refs.map((ref) => {
|
|
188
461
|
const existing = this.index.getFile(source.id, ref.path);
|
|
189
462
|
return {
|
|
@@ -203,15 +476,25 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
203
476
|
let okCount = 0;
|
|
204
477
|
let unchanged = 0;
|
|
205
478
|
for (const ref of enriched) {
|
|
479
|
+
// 后立刻停:不再写索引
|
|
480
|
+
if (this.isAborted(source.id)) {
|
|
481
|
+
this.emitProgress({
|
|
482
|
+
type: 'source',
|
|
483
|
+
sourceId: source.id,
|
|
484
|
+
status: 'aborted',
|
|
485
|
+
detail: `stopped after ${okCount + unchanged}/${enriched.length}`,
|
|
486
|
+
});
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
206
489
|
try {
|
|
207
490
|
const doc = await fetcher.fetch(source, ref, cred);
|
|
208
491
|
if (!doc) {
|
|
209
|
-
// 304 /
|
|
492
|
+
// 304 / :保持原索引
|
|
210
493
|
unchanged += 1;
|
|
211
494
|
okCount += 1;
|
|
212
495
|
continue;
|
|
213
496
|
}
|
|
214
|
-
if (this.indexVirtualDoc(source.id, doc)) {
|
|
497
|
+
if (await this.indexVirtualDoc(source.id, doc, source.authRef)) {
|
|
215
498
|
okCount += 1;
|
|
216
499
|
}
|
|
217
500
|
}
|
|
@@ -226,22 +509,25 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
226
509
|
});
|
|
227
510
|
}
|
|
228
511
|
}
|
|
229
|
-
//
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
this.
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
512
|
+
// prune:仅?discover **完整** 时才;截?walk 失败?keep 不完整,prune ?
|
|
513
|
+
// 后不?prune:半?keep 删已入库文档
|
|
514
|
+
if (!this.isAborted(source.id) && discovered.complete) {
|
|
515
|
+
const keep = new Set(enriched.map((r) => r.path));
|
|
516
|
+
const pruned = await this.index.pruneMissingAsync(source.id, keep, { allowEmptyKeep: true });
|
|
517
|
+
if (pruned > 0) {
|
|
518
|
+
this.emitProgress({
|
|
519
|
+
type: 'source',
|
|
520
|
+
sourceId: source.id,
|
|
521
|
+
status: 'pruned',
|
|
522
|
+
detail: `removed=${pruned}`,
|
|
523
|
+
});
|
|
524
|
+
}
|
|
239
525
|
}
|
|
240
|
-
if (this.embedding) {
|
|
241
|
-
this.enqueue(source.id, 'embed_source', null, 3);
|
|
526
|
+
if (this.embedding && !this.isAborted(source.id)) {
|
|
527
|
+
await this.enqueue(source.id, 'embed_source', null, 3);
|
|
242
528
|
this.kick();
|
|
243
529
|
}
|
|
244
|
-
this.refreshCoverage(source.id);
|
|
530
|
+
this.refreshCoverage(source.id, { force: true });
|
|
245
531
|
this.emitProgress({
|
|
246
532
|
type: 'source',
|
|
247
533
|
sourceId: source.id,
|
|
@@ -250,10 +536,10 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
250
536
|
});
|
|
251
537
|
}
|
|
252
538
|
/**
|
|
253
|
-
*
|
|
539
|
+
* 解析源凭证;配置?authRef ?**fail-closed**(解析失败则不抓,不降级匿名?
|
|
254
540
|
*
|
|
255
|
-
* @returns
|
|
256
|
-
* @throws authRef
|
|
541
|
+
* @returns ?null(仅当源?authRef?
|
|
542
|
+
* @throws authRef 配置了但无法解析时抛错,由调用方
|
|
257
543
|
*/
|
|
258
544
|
async resolveCred(source) {
|
|
259
545
|
if (!source.authRef)
|
|
@@ -268,16 +554,16 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
268
554
|
return resolved;
|
|
269
555
|
}
|
|
270
556
|
/**
|
|
271
|
-
*
|
|
557
|
+
* ?VirtualDocument 写入索引?/ 规范化后文本?
|
|
272
558
|
*
|
|
273
|
-
* @returns
|
|
559
|
+
* @returns 成功写入
|
|
274
560
|
*/
|
|
275
|
-
indexVirtualDoc(sourceId, doc) {
|
|
561
|
+
async indexVirtualDoc(sourceId, doc, authRef) {
|
|
276
562
|
if (!doc.content?.trim()) {
|
|
277
563
|
this.index.markFileSkipped(sourceId, doc.path, 'empty_content');
|
|
278
564
|
return false;
|
|
279
565
|
}
|
|
280
|
-
//
|
|
566
|
+
// 逻辑能无扩展名(connector id / URL path):按 MIME / 内容启发式
|
|
281
567
|
let adapter = this.adapters.match(doc.path, doc.contentType);
|
|
282
568
|
if (!adapter) {
|
|
283
569
|
if (looksLikeHtml(doc.content, doc.contentType))
|
|
@@ -292,8 +578,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
292
578
|
return true;
|
|
293
579
|
}
|
|
294
580
|
try {
|
|
295
|
-
const chunks =
|
|
296
|
-
|
|
581
|
+
const chunks = await this.chunkTextInWorker(doc.content, doc.path, undefined, adapter.id);
|
|
582
|
+
let identityKey;
|
|
583
|
+
if (doc.externalUrl) {
|
|
584
|
+
identityKey = urlIdentityKey(doc.externalUrl, authRef);
|
|
585
|
+
}
|
|
586
|
+
else {
|
|
587
|
+
try {
|
|
588
|
+
identityKey = `path:${normalizePathLexical(doc.path)}`;
|
|
589
|
+
}
|
|
590
|
+
catch {
|
|
591
|
+
identityKey = `path:${doc.path.replace(/\\/g, '/')}`;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
await this.index.upsertFile({
|
|
297
595
|
sourceId,
|
|
298
596
|
path: doc.path,
|
|
299
597
|
contentHash,
|
|
@@ -301,6 +599,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
301
599
|
mtime: Math.floor(Date.now() / 1000),
|
|
302
600
|
adapterId: adapter.id,
|
|
303
601
|
chunks,
|
|
602
|
+
identityKey,
|
|
304
603
|
externalUrl: doc.externalUrl,
|
|
305
604
|
etag: doc.etag,
|
|
306
605
|
lastModified: doc.lastModified,
|
|
@@ -313,13 +612,129 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
313
612
|
}
|
|
314
613
|
}
|
|
315
614
|
/**
|
|
316
|
-
*
|
|
615
|
+
* 单文(P0 道;完成 parse,不 await 全库?
|
|
317
616
|
*/
|
|
318
617
|
async ingestFileNow(sourceId, filePath) {
|
|
319
618
|
const ok = await this.parseOne(sourceId, filePath);
|
|
320
619
|
this.refreshCoverage(sourceId);
|
|
321
620
|
return ok;
|
|
322
621
|
}
|
|
622
|
+
/**
|
|
623
|
+
* 重做指定文件:强制重?解析 ?分块 ?向量
|
|
624
|
+
*
|
|
625
|
+
* - 必须归属(防越权读盘);外源逻辑 fetch_doc 重取
|
|
626
|
+
* - 失效 contentHash(忽?isFresh),?chunks 保留?upsert 成功
|
|
627
|
+
* - **于中 resume**:显式重?= 意图要跑活,不允许静默丢?
|
|
628
|
+
* - 等非文件:直接清,不入队
|
|
629
|
+
* - 完成后走既有 embed 链路
|
|
630
|
+
*
|
|
631
|
+
* @param sourceId - ?id
|
|
632
|
+
* @param paths - 文件列表
|
|
633
|
+
* @returns queued=新入队;alreadyActive=已在队列/;cleanedNonFiles=清掉的目录脏行;rejected=越权/不属
|
|
634
|
+
*/
|
|
635
|
+
async reprocessFiles(sourceId, paths) {
|
|
636
|
+
if (!paths.length) {
|
|
637
|
+
return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: 0 };
|
|
638
|
+
}
|
|
639
|
+
const source = this.sources.get(sourceId);
|
|
640
|
+
if (!source) {
|
|
641
|
+
return { queued: 0, alreadyActive: 0, cleanedNonFiles: 0, resumed: false, rejected: paths.length };
|
|
642
|
+
}
|
|
643
|
+
let resumed = false;
|
|
644
|
+
if (this.isAborted(sourceId)) {
|
|
645
|
+
this.beginAbortEpoch(sourceId);
|
|
646
|
+
resumed = true;
|
|
647
|
+
}
|
|
648
|
+
let queued = 0;
|
|
649
|
+
let alreadyActive = 0;
|
|
650
|
+
let cleanedNonFiles = 0;
|
|
651
|
+
let rejected = 0;
|
|
652
|
+
const isRemote = source.kind === 'url' || source.kind === 'connector';
|
|
653
|
+
for (const p of paths) {
|
|
654
|
+
if (!this.sourceOwnsPath(source, p)) {
|
|
655
|
+
rejected += 1;
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
if (isRemote) {
|
|
659
|
+
// 外源:path 重做 = 重新 fetch + 索引(不读本地盘?
|
|
660
|
+
this.index.invalidateFileForReparse(sourceId, p);
|
|
661
|
+
if (await this.enqueue(sourceId, 'fetch_doc', p, 1)) {
|
|
662
|
+
queued += 1;
|
|
663
|
+
}
|
|
664
|
+
else if (this.hasActiveParseJob(sourceId, p)) {
|
|
665
|
+
alreadyActive += 1;
|
|
666
|
+
}
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
// /节点:清即可,没?
|
|
670
|
+
try {
|
|
671
|
+
const st = statSync(p);
|
|
672
|
+
if (!st.isFile()) {
|
|
673
|
+
this.index.removeFile(sourceId, p);
|
|
674
|
+
cleanedNonFiles += 1;
|
|
675
|
+
continue;
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
catch {
|
|
679
|
+
// 不存仍按文件流程 parse ?
|
|
680
|
+
}
|
|
681
|
+
// 总是失效 hash:即使已在队列,跑起来也不能 isFresh 跳过
|
|
682
|
+
this.index.invalidateFileForReparse(sourceId, p);
|
|
683
|
+
if (await this.enqueue(sourceId, 'parse_file', p, 1)) {
|
|
684
|
+
queued += 1;
|
|
685
|
+
}
|
|
686
|
+
else if (this.hasActiveParseJob(sourceId, p)) {
|
|
687
|
+
alreadyActive += 1;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
this.kick();
|
|
691
|
+
return { queued, alreadyActive, cleanedNonFiles, resumed, rejected };
|
|
692
|
+
}
|
|
693
|
+
/** 上是否已?queued/running ?parse/fetch 任务 */
|
|
694
|
+
hasActiveParseJob(sourceId, path) {
|
|
695
|
+
return this.jobQueue.hasActivePathJob(sourceId, path);
|
|
696
|
+
}
|
|
697
|
+
/**
|
|
698
|
+
* 级任?文件UI 跟踪重做完成?
|
|
699
|
+
*
|
|
700
|
+
* @param sourceId - ?id
|
|
701
|
+
* @param paths - 列表
|
|
702
|
+
*/
|
|
703
|
+
jobStateForPaths(sourceId, paths) {
|
|
704
|
+
return paths.map((p) => {
|
|
705
|
+
const jobs = this.sources.database.raw
|
|
706
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs
|
|
707
|
+
WHERE source_id = ? AND path = ? AND status IN ('queued','running')`)
|
|
708
|
+
.get(sourceId, p);
|
|
709
|
+
const f = this.index.getFile(sourceId, p);
|
|
710
|
+
let exists = false;
|
|
711
|
+
try {
|
|
712
|
+
exists = statSync(p).isFile();
|
|
713
|
+
}
|
|
714
|
+
catch {
|
|
715
|
+
exists = false;
|
|
716
|
+
}
|
|
717
|
+
return {
|
|
718
|
+
path: p,
|
|
719
|
+
jobsActive: Number(jobs?.n ?? 0),
|
|
720
|
+
fileStatus: f?.status ?? null,
|
|
721
|
+
chunkCount: f?.chunkCount ?? 0,
|
|
722
|
+
error: f?.error ?? null,
|
|
723
|
+
exists,
|
|
724
|
+
};
|
|
725
|
+
});
|
|
726
|
+
}
|
|
727
|
+
/**
|
|
728
|
+
* 按筛选批量重做(当前列表?status/ext/q 条件?
|
|
729
|
+
*
|
|
730
|
+
* @param sourceId - ?id
|
|
731
|
+
* @param opts - 与文件列表筛选一?
|
|
732
|
+
* @returns 入队条数
|
|
733
|
+
*/
|
|
734
|
+
async reprocessByFilter(sourceId, opts) {
|
|
735
|
+
const paths = this.index.listFilePathsFiltered(sourceId, opts);
|
|
736
|
+
return this.reprocessFiles(sourceId, paths);
|
|
737
|
+
}
|
|
323
738
|
/**
|
|
324
739
|
* 处理 fs 变更(debounce 后入队)
|
|
325
740
|
*/
|
|
@@ -332,30 +747,33 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
332
747
|
clearTimeout(prev);
|
|
333
748
|
this.debounceTimers.set(key, setTimeout(() => {
|
|
334
749
|
this.debounceTimers.delete(key);
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
750
|
+
// 统一?stat:目?change 时不?parse ?no_adapter
|
|
751
|
+
void stat(filePath)
|
|
752
|
+
.then(async (st) => {
|
|
753
|
+
if (st.isFile()) {
|
|
754
|
+
await this.enqueue(sourceId, 'parse_file', filePath, 1);
|
|
755
|
+
}
|
|
756
|
+
else if (event === 'rename' || !st.isDirectory()) {
|
|
757
|
+
await this.enqueue(sourceId, 'drop_file', filePath, 1);
|
|
758
|
+
}
|
|
759
|
+
else {
|
|
760
|
+
this.index.removeFile(sourceId, filePath);
|
|
761
|
+
}
|
|
762
|
+
this.requestGapScan(sourceId);
|
|
763
|
+
this.kick();
|
|
764
|
+
})
|
|
765
|
+
.catch(() => {
|
|
766
|
+
void this.enqueue(sourceId, 'drop_file', filePath, 1)
|
|
767
|
+
.then(() => {
|
|
768
|
+
this.requestGapScan(sourceId);
|
|
345
769
|
this.kick();
|
|
346
770
|
})
|
|
347
|
-
.catch(() =>
|
|
348
|
-
|
|
349
|
-
this.kick();
|
|
350
|
-
});
|
|
351
|
-
return;
|
|
352
|
-
}
|
|
353
|
-
this.enqueue(sourceId, 'parse_file', filePath, 1, filePath);
|
|
354
|
-
this.kick();
|
|
771
|
+
.catch(() => undefined);
|
|
772
|
+
});
|
|
355
773
|
}, this.debounceMs));
|
|
356
774
|
}
|
|
357
775
|
/**
|
|
358
|
-
*
|
|
776
|
+
* watch(幂等)
|
|
359
777
|
*/
|
|
360
778
|
startWatch(sourceId) {
|
|
361
779
|
const source = this.sources.get(sourceId);
|
|
@@ -399,11 +817,11 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
399
817
|
}
|
|
400
818
|
}
|
|
401
819
|
/**
|
|
402
|
-
*
|
|
820
|
+
* 等待队列排空(测?管理?
|
|
403
821
|
*/
|
|
404
822
|
async idle(timeoutMs = 30_000) {
|
|
405
823
|
const start = Date.now();
|
|
406
|
-
while (this.
|
|
824
|
+
while (this.runningParse > 0 || this.runningEmbed > 0 || this.hasQueuedJobs()) {
|
|
407
825
|
if (Date.now() - start > timeoutMs) {
|
|
408
826
|
throw new Error('knowledge ingest idle timeout');
|
|
409
827
|
}
|
|
@@ -414,11 +832,436 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
414
832
|
dispose() {
|
|
415
833
|
this.disposed = true;
|
|
416
834
|
this.stopPolling();
|
|
835
|
+
this.stopReconciler();
|
|
836
|
+
if (this.gapScanTimer) {
|
|
837
|
+
clearTimeout(this.gapScanTimer);
|
|
838
|
+
this.gapScanTimer = null;
|
|
839
|
+
}
|
|
840
|
+
this.gapScanUrgent.clear();
|
|
841
|
+
void this.abortJobs();
|
|
417
842
|
for (const id of this.watchers.keys())
|
|
418
843
|
this.stopWatch(id);
|
|
419
844
|
}
|
|
420
845
|
/**
|
|
421
|
-
*
|
|
846
|
+
* 手动索引任务(业务完整:停收 + 清队 + 打断在跑?
|
|
847
|
+
*
|
|
848
|
+
* @param opts.sourceId - 仅中源;省略则全部源
|
|
849
|
+
* @returns 取消
|
|
850
|
+
*/
|
|
851
|
+
abortJobs(opts) {
|
|
852
|
+
const sid = opts?.sourceId?.trim();
|
|
853
|
+
const ids = sid
|
|
854
|
+
? [sid]
|
|
855
|
+
: [
|
|
856
|
+
...this.jobControl.listAbortedSourceIds(),
|
|
857
|
+
...this.sources.list().map((s) => s.id),
|
|
858
|
+
];
|
|
859
|
+
const unique = [...new Set(ids)];
|
|
860
|
+
let cancelledQueued = 0;
|
|
861
|
+
let abortedRunning = 0;
|
|
862
|
+
for (const id of unique) {
|
|
863
|
+
// 1) DB 落中跨重+ 打断程信?
|
|
864
|
+
const newly = this.jobControl.markAborted(id);
|
|
865
|
+
if (newly)
|
|
866
|
+
abortedRunning += 1;
|
|
867
|
+
// 2) 清空 queued ?cancelled(共?File ?parse 不杀:仍领则保留?
|
|
868
|
+
const res = this.sources.database.raw
|
|
869
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
870
|
+
WHERE source_id = ? AND status = 'queued'
|
|
871
|
+
AND (
|
|
872
|
+
file_id IS NULL
|
|
873
|
+
OR kind IN ('walk_source', 'embed_source')
|
|
874
|
+
OR NOT EXISTS (
|
|
875
|
+
SELECT 1 FROM knowledge_memberships m
|
|
876
|
+
WHERE m.file_id = knowledge_jobs.file_id AND m.source_id != ?
|
|
877
|
+
)
|
|
878
|
+
)`)
|
|
879
|
+
.run(Date.now(), id, id);
|
|
880
|
+
cancelledQueued += Number(res.changes ?? 0);
|
|
881
|
+
}
|
|
882
|
+
// 3) 看门狗不要再?aborted running 收回 queued(限定本次涉及的源)
|
|
883
|
+
if (sid) {
|
|
884
|
+
this.sources.database.raw
|
|
885
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
886
|
+
WHERE source_id = ? AND status = 'running' AND last_error = 'aborted'`)
|
|
887
|
+
.run(Date.now(), sid);
|
|
888
|
+
}
|
|
889
|
+
else {
|
|
890
|
+
this.sources.database.raw
|
|
891
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
892
|
+
WHERE status = 'running' AND last_error = 'aborted'`)
|
|
893
|
+
.run(Date.now());
|
|
894
|
+
}
|
|
895
|
+
const runningRow = this.sources.database.raw
|
|
896
|
+
.prepare(sid
|
|
897
|
+
? `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE source_id = ? AND status = 'running'`
|
|
898
|
+
: `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'running'`)
|
|
899
|
+
.get(...(sid ? [sid] : []));
|
|
900
|
+
const runningJobs = Number(runningRow?.n ?? 0);
|
|
901
|
+
this.emitProgress({
|
|
902
|
+
type: 'source',
|
|
903
|
+
sourceId: sid ?? '*',
|
|
904
|
+
status: 'aborted',
|
|
905
|
+
detail: `cancelledQueued=${cancelledQueued} runningTail=${runningJobs}`,
|
|
906
|
+
});
|
|
907
|
+
return { cancelledQueued, abortedRunning, runningJobs };
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
910
|
+
* /继续索引(相轻)
|
|
911
|
+
*
|
|
912
|
+
* - 清除该源?
|
|
913
|
+
* - ?cancelled 任务放回 queued
|
|
914
|
+
* - 若仍缺向量则排队 embed
|
|
915
|
+
* - 不重?walk 全量;需要全 reindex
|
|
916
|
+
*
|
|
917
|
+
* @param opts.sourceId - 仅恢源;省略则全部源
|
|
918
|
+
* @returns
|
|
919
|
+
*/
|
|
920
|
+
resumeJobs(opts) {
|
|
921
|
+
const sid = opts?.sourceId?.trim();
|
|
922
|
+
const ids = sid
|
|
923
|
+
? [sid]
|
|
924
|
+
: [
|
|
925
|
+
...new Set([
|
|
926
|
+
...this.jobControl.listAbortedSourceIds(),
|
|
927
|
+
...this.sources.list().map((s) => s.id),
|
|
928
|
+
]),
|
|
929
|
+
];
|
|
930
|
+
let restoredCancelled = 0;
|
|
931
|
+
let embedQueued = 0;
|
|
932
|
+
for (const id of ids) {
|
|
933
|
+
// 1) 清除态(DB + 新纪元信号)
|
|
934
|
+
this.beginAbortEpoch(id);
|
|
935
|
+
// 2) cancelled ?queued(用户中止时的活?
|
|
936
|
+
const res = this.sources.database.raw
|
|
937
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', last_error = NULL, updated_at = ?
|
|
938
|
+
WHERE source_id = ? AND status = 'cancelled' AND (last_error IS NULL OR last_error = 'aborted')`)
|
|
939
|
+
.run(Date.now(), id);
|
|
940
|
+
restoredCancelled += Number(res.changes ?? 0);
|
|
941
|
+
// 3) 仍缺向量则确?embed 在队
|
|
942
|
+
if (this.embedding) {
|
|
943
|
+
const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
|
|
944
|
+
const active = this.countActiveJobs(id);
|
|
945
|
+
if (missing && active.embed === 0) {
|
|
946
|
+
this.ensureEmbedJob(id);
|
|
947
|
+
embedQueued += 1;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
this.kick();
|
|
952
|
+
this.emitProgress({
|
|
953
|
+
type: 'source',
|
|
954
|
+
sourceId: sid ?? '*',
|
|
955
|
+
status: 'resumed',
|
|
956
|
+
detail: `restoredCancelled=${restoredCancelled} embedQueued=${embedQueued}`,
|
|
957
|
+
});
|
|
958
|
+
return { restoredCancelled, embedQueued };
|
|
959
|
+
}
|
|
960
|
+
/** 的未(resume / supersede 后用;清 DB 态) */
|
|
961
|
+
beginAbortEpoch(sourceId) {
|
|
962
|
+
this.jobControl.beginEpoch(sourceId);
|
|
963
|
+
}
|
|
964
|
+
/**
|
|
965
|
+
* 重建前作废本源未完成任务(supersede?
|
|
966
|
+
*
|
|
967
|
+
* - queued ?cancelled(superseded_by_reindex?
|
|
968
|
+
* - running 打断(worker/出)
|
|
969
|
+
* - 随后 beginAbortEpoch,本建可继续?
|
|
970
|
+
*/
|
|
971
|
+
supersedeSourceWork(sourceId) {
|
|
972
|
+
this.sources.database.raw
|
|
973
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
|
|
974
|
+
WHERE source_id = ? AND status = 'queued'`)
|
|
975
|
+
.run(Date.now(), sourceId);
|
|
976
|
+
// 打断在跑 parse/embed;再新纪元(清中,避?walk ?abort
|
|
977
|
+
this.jobControl.markAborted(sourceId);
|
|
978
|
+
this.beginAbortEpoch(sourceId);
|
|
979
|
+
this.emitProgress({
|
|
980
|
+
type: 'source',
|
|
981
|
+
sourceId,
|
|
982
|
+
status: 'superseded',
|
|
983
|
+
detail: 'reindex will resync from disk',
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
async ensureParseCoverage(sourceId) {
|
|
987
|
+
const source = this.sources.get(sourceId);
|
|
988
|
+
if (!source || source.status === 'removed' || source.status === 'disabled')
|
|
989
|
+
return 0;
|
|
990
|
+
if (source.kind !== 'directory' && source.kind !== 'workspace')
|
|
991
|
+
return 0;
|
|
992
|
+
if (this.isAborted(sourceId))
|
|
993
|
+
return 0;
|
|
994
|
+
// 仅跳过「全量 walk 在途」:individual parse_file 排队 **不得** 挡住缺口补扫。
|
|
995
|
+
// 否则 watch 漏事件后,当前批次 parse 会一直占着门闩,排空才一次入队数百文件(84 → 726)。
|
|
996
|
+
// 不用 sourceLocks:与 reconcile 同轮 fire-and-forget ingestSource 竞态,会误伤补扫。
|
|
997
|
+
if (this.countQueuedKinds(['walk_source'], sourceId) > 0)
|
|
998
|
+
return 0;
|
|
999
|
+
let found;
|
|
1000
|
+
try {
|
|
1001
|
+
found = await this.localFetcher.discover(source);
|
|
1002
|
+
}
|
|
1003
|
+
catch {
|
|
1004
|
+
return 0;
|
|
1005
|
+
}
|
|
1006
|
+
const byPath = new Map(this.index.listFiles(sourceId).map((f) => [f.path.replace(/\\/g, '/'), f]));
|
|
1007
|
+
let added = 0;
|
|
1008
|
+
let retriedSkipped = 0;
|
|
1009
|
+
let retriedIncomplete = 0;
|
|
1010
|
+
for (const f of found.docs) {
|
|
1011
|
+
const logical = f.path.replace(/\\/g, '/');
|
|
1012
|
+
const existing = byPath.get(logical);
|
|
1013
|
+
// 缺文件必补;indexing=半写入/崩溃残留必补;skipped 且原因可重试时补
|
|
1014
|
+
const need = !existing ||
|
|
1015
|
+
existing.status === 'indexing' ||
|
|
1016
|
+
(existing.status === 'skipped' &&
|
|
1017
|
+
isRetryableSkipReason(existing.error ?? null, existing.size, this.fileLimits));
|
|
1018
|
+
if (!need)
|
|
1019
|
+
continue;
|
|
1020
|
+
if (await this.enqueue(sourceId, 'parse_file', f.path, 2)) {
|
|
1021
|
+
added += 1;
|
|
1022
|
+
if (!existing)
|
|
1023
|
+
continue;
|
|
1024
|
+
if (existing.status === 'indexing')
|
|
1025
|
+
retriedIncomplete += 1;
|
|
1026
|
+
else
|
|
1027
|
+
retriedSkipped += 1;
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
if (added > 0) {
|
|
1031
|
+
this.emitProgress({
|
|
1032
|
+
type: 'source',
|
|
1033
|
+
sourceId,
|
|
1034
|
+
status: 'parse_gap_found',
|
|
1035
|
+
detail: `missing_files=${added - retriedSkipped - retriedIncomplete} retry_skipped=${retriedSkipped} retry_indexing=${retriedIncomplete}`,
|
|
1036
|
+
});
|
|
1037
|
+
this.kick();
|
|
1038
|
+
}
|
|
1039
|
+
return added;
|
|
1040
|
+
}
|
|
1041
|
+
/** watch 突发/目录事件后:尽快做一次缺口补扫(默认 3s,不等 60s 节流) */
|
|
1042
|
+
requestGapScan(sourceId, delayMs = 3_000) {
|
|
1043
|
+
if (this.disposed || this.isAborted(sourceId))
|
|
1044
|
+
return;
|
|
1045
|
+
this.gapScanUrgent.add(sourceId);
|
|
1046
|
+
if (this.gapScanTimer)
|
|
1047
|
+
return;
|
|
1048
|
+
this.gapScanTimer = setTimeout(() => {
|
|
1049
|
+
this.gapScanTimer = null;
|
|
1050
|
+
const ids = [...this.gapScanUrgent];
|
|
1051
|
+
this.gapScanUrgent.clear();
|
|
1052
|
+
for (const id of ids) {
|
|
1053
|
+
this.lastParseGapScanAt.set(id, Date.now());
|
|
1054
|
+
void this.ensureParseCoverage(id).catch(() => undefined);
|
|
1055
|
+
}
|
|
1056
|
+
}, delayMs);
|
|
1057
|
+
this.gapScanTimer.unref?.();
|
|
1058
|
+
}
|
|
1059
|
+
/**
|
|
1060
|
+
* 磁盘↔索齐:删掉已不存在的文件(?chunks/embeddings?
|
|
1061
|
+
*
|
|
1062
|
+
* 源用?discover walk ?keep 集(;失回并?stat?
|
|
1063
|
+
* embed 前调避免给已移出文件的残 chunks 上向?
|
|
1064
|
+
*
|
|
1065
|
+
* @param sourceId - 知识?id
|
|
1066
|
+
* @returns 删除的文件数
|
|
1067
|
+
*/
|
|
1068
|
+
async pruneMissingOnDisk(sourceId) {
|
|
1069
|
+
const source = this.sources.get(sourceId);
|
|
1070
|
+
if (!source || source.kind === 'url' || source.kind === 'connector')
|
|
1071
|
+
return 0;
|
|
1072
|
+
let removed = 0;
|
|
1073
|
+
if (source.kind === 'directory' || source.kind === 'workspace') {
|
|
1074
|
+
try {
|
|
1075
|
+
const found = await this.localFetcher.discover(source);
|
|
1076
|
+
if (!found.complete) {
|
|
1077
|
+
// walk 不完整:?stat止用残缺 keep 清库
|
|
1078
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
1079
|
+
}
|
|
1080
|
+
else {
|
|
1081
|
+
const keep = new Set(found.docs.map((f) => f.path));
|
|
1082
|
+
// walk 完整:即使空也可清空(与逐文?stat 致)
|
|
1083
|
+
removed = await this.index.pruneMissingAsync(sourceId, keep, { allowEmptyKeep: true });
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
catch {
|
|
1087
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
else {
|
|
1091
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
1092
|
+
}
|
|
1093
|
+
if (removed > 0) {
|
|
1094
|
+
this.emitProgress({
|
|
1095
|
+
type: 'source',
|
|
1096
|
+
sourceId,
|
|
1097
|
+
status: 'pruned_missing',
|
|
1098
|
+
detail: `removed_files=${removed}`,
|
|
1099
|
+
});
|
|
1100
|
+
this.refreshCoverage(sourceId, { force: true });
|
|
1101
|
+
}
|
|
1102
|
+
return removed;
|
|
1103
|
+
}
|
|
1104
|
+
/** 逐文?stat(file ?/ walk 失败);限流 + 单文件 removeFile(勿对每个 gone 做 removePathTree LIKE) */
|
|
1105
|
+
async pruneByStatScan(sourceId, concurrency = 4) {
|
|
1106
|
+
const files = this.index.listFiles(sourceId);
|
|
1107
|
+
let removed = 0;
|
|
1108
|
+
let i = 0;
|
|
1109
|
+
const worker = async () => {
|
|
1110
|
+
for (;;) {
|
|
1111
|
+
if (this.isAborted(sourceId))
|
|
1112
|
+
return;
|
|
1113
|
+
const idx = i++;
|
|
1114
|
+
if (idx >= files.length)
|
|
1115
|
+
return;
|
|
1116
|
+
const f = files[idx];
|
|
1117
|
+
let exists = true;
|
|
1118
|
+
try {
|
|
1119
|
+
const st = await stat(f.path);
|
|
1120
|
+
exists = st.isFile();
|
|
1121
|
+
}
|
|
1122
|
+
catch {
|
|
1123
|
+
exists = false;
|
|
1124
|
+
}
|
|
1125
|
+
if (!exists) {
|
|
1126
|
+
await this.index.removeFileAsync(sourceId, f.path);
|
|
1127
|
+
removed += 1;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
};
|
|
1131
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, files.length || 1) }, () => worker()));
|
|
1132
|
+
return removed;
|
|
1133
|
+
}
|
|
1134
|
+
abortSignalFor(sourceId) {
|
|
1135
|
+
return this.jobControl.signalFor(sourceId);
|
|
1136
|
+
}
|
|
1137
|
+
isAborted(sourceId) {
|
|
1138
|
+
return this.jobControl.isAborted(sourceId);
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* /继续 控制数(互斥按钮
|
|
1142
|
+
*
|
|
1143
|
+
* @param sourceId - ?id
|
|
1144
|
+
* @returns 队列计数 + 已中?+ 仍缺向量
|
|
1145
|
+
*/
|
|
1146
|
+
jobControlState(sourceId) {
|
|
1147
|
+
return readJobControlState(this.sources.database, this.index, sourceId, {
|
|
1148
|
+
embeddingEnabled: Boolean(this.embedding),
|
|
1149
|
+
});
|
|
1150
|
+
}
|
|
1151
|
+
/**
|
|
1152
|
+
* 任务对账(看门狗?不依赖单?ensureEmbedJob
|
|
1153
|
+
*
|
|
1154
|
+
* 解决「续去重吞掉 / 进程 / partial」等静默停摆?
|
|
1155
|
+
* 1. 回收超时 running(崩?孤儿?
|
|
1156
|
+
* 2. 有缺向量且无 active embed ?排队
|
|
1157
|
+
* 3. 无队缺口 ??coverage/status
|
|
1158
|
+
*/
|
|
1159
|
+
startReconciler(intervalMs = 15_000) {
|
|
1160
|
+
if (this.reconcileTimer || this.disposed)
|
|
1161
|
+
return;
|
|
1162
|
+
this.reconcileTimer = setInterval(() => {
|
|
1163
|
+
void this.reconcileJobs().catch((err) => {
|
|
1164
|
+
this.emitProgress({
|
|
1165
|
+
type: 'error',
|
|
1166
|
+
sourceId: '*',
|
|
1167
|
+
status: 'reconcile_failed',
|
|
1168
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
1169
|
+
});
|
|
1170
|
+
});
|
|
1171
|
+
}, intervalMs);
|
|
1172
|
+
this.reconcileTimer.unref?.();
|
|
1173
|
+
}
|
|
1174
|
+
stopReconciler() {
|
|
1175
|
+
if (this.reconcileTimer) {
|
|
1176
|
+
clearInterval(this.reconcileTimer);
|
|
1177
|
+
this.reconcileTimer = null;
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
/** 对账入口(也工调 */
|
|
1181
|
+
async reconcileJobs() {
|
|
1182
|
+
if (this.disposed)
|
|
1183
|
+
return;
|
|
1184
|
+
// stale ?**大于** 时,否则合法的长 parse 收回后双?
|
|
1185
|
+
this.reclaimStaleRunningJobs(this.fileLimits.maxParseTimeoutMs + 60_000);
|
|
1186
|
+
// path: 与 win:/unix: 身份分裂的零认领残留
|
|
1187
|
+
this.index.purgeOrphanPathIdentityFiles();
|
|
1188
|
+
// 等历入项(轻量;非每
|
|
1189
|
+
this.cleanupNonFileIndexRows();
|
|
1190
|
+
// ?job 定期清理,防表无?
|
|
1191
|
+
this.cleanupTerminalJobs();
|
|
1192
|
+
// 存量 FTS 后台补齐(分出事件循 rebuild?
|
|
1193
|
+
void this.index.ensureFtsBackfill();
|
|
1194
|
+
// 存量向量回填:小批量 + 让出事件?2000 条同步写(会 abort?
|
|
1195
|
+
await this.index.backfillVecFromEmbeddings(25);
|
|
1196
|
+
for (const s of this.sources.list()) {
|
|
1197
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
1198
|
+
continue;
|
|
1199
|
+
if (this.isAborted(s.id))
|
|
1200
|
+
continue;
|
|
1201
|
+
const active = this.countActiveJobs(s.id);
|
|
1202
|
+
// discovering 卡死(注册 kick 失败 / file 源 walk 中断):无任务则 re-kick,不 supersede
|
|
1203
|
+
if (active.total === 0 &&
|
|
1204
|
+
(s.status === 'pending' || s.status === 'discovering') &&
|
|
1205
|
+
(s.kind === 'file' || s.kind === 'directory' || s.kind === 'workspace')) {
|
|
1206
|
+
await this.ingestSource(s.id, { fromQueue: true }).catch(() => undefined);
|
|
1207
|
+
}
|
|
1208
|
+
// 稳定态收尾(auto-describe 等)关键词部署同触发
|
|
1209
|
+
// 先看 active,再 gap ;避免刚入队?gap job 挡住 settled
|
|
1210
|
+
if (active.total === 0) {
|
|
1211
|
+
this.refreshCoverage(s.id, { force: true });
|
|
1212
|
+
try {
|
|
1213
|
+
this.onSourceSettled?.(s.id);
|
|
1214
|
+
}
|
|
1215
|
+
catch {
|
|
1216
|
+
/* ignore settle hook errors */
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
else {
|
|
1220
|
+
this.flushCoverageDirty(s.id);
|
|
1221
|
+
}
|
|
1222
|
+
// watch 漏事件 / skip 重试时补齐 parse(与配置 embedding 无关)
|
|
1223
|
+
// urgent(watch 突发)不走 60s 节流;否则批量拷贝后要等一整轮才补
|
|
1224
|
+
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
1225
|
+
const gapDue = this.gapScanUrgent.has(s.id) || Date.now() - lastGap > 60_000;
|
|
1226
|
+
if (gapDue) {
|
|
1227
|
+
this.lastParseGapScanAt.set(s.id, Date.now());
|
|
1228
|
+
this.gapScanUrgent.delete(s.id);
|
|
1229
|
+
await this.ensureParseCoverage(s.id);
|
|
1230
|
+
// drop_file 在途时勿再 prune:双路径会同时删同一棵子树,放大写压力
|
|
1231
|
+
const dropsInFlight = this.countQueuedKinds(['drop_file'], s.id) > 0 || this.hasRunningKind(s.id, 'drop_file');
|
|
1232
|
+
if (!dropsInFlight) {
|
|
1233
|
+
await this.pruneMissingOnDisk(s.id);
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
const missingEmbed = this.embedding
|
|
1237
|
+
? this.index.hasChunksMissingEmbedding(s.id)
|
|
1238
|
+
: false;
|
|
1239
|
+
if (this.embedding && missingEmbed && active.embed === 0) {
|
|
1240
|
+
this.ensureEmbedJob(s.id);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
this.kick();
|
|
1244
|
+
}
|
|
1245
|
+
/** ?job 清理(done/failed/cancelled 超过保留期) */
|
|
1246
|
+
cleanupTerminalJobs(retainMs = 24 * 60 * 60_000) {
|
|
1247
|
+
this.jobQueue.cleanupTerminal(retainMs);
|
|
1248
|
+
}
|
|
1249
|
+
/** running 超时(无 heartbeat)→ 收回 queued次失败的直接?failed,防毒文 */
|
|
1250
|
+
reclaimStaleRunningJobs(staleMs) {
|
|
1251
|
+
this.jobQueue.reclaimStaleRunning(staleMs);
|
|
1252
|
+
}
|
|
1253
|
+
countActiveJobs(sourceId) {
|
|
1254
|
+
return this.jobQueue.countActiveJobs(sourceId);
|
|
1255
|
+
}
|
|
1256
|
+
hasRunningKind(sourceId, kind) {
|
|
1257
|
+
const row = this.sources.database.raw
|
|
1258
|
+
.prepare(`SELECT 1 AS ok FROM knowledge_jobs
|
|
1259
|
+
WHERE source_id = ? AND kind = ? AND status = 'running' LIMIT 1`)
|
|
1260
|
+
.get(sourceId, kind);
|
|
1261
|
+
return Boolean(row?.ok);
|
|
1262
|
+
}
|
|
1263
|
+
/**
|
|
1264
|
+
* poll 调度(幂等);仅处理 sync.strategy=poll ?enabled 的源
|
|
422
1265
|
*/
|
|
423
1266
|
startPolling() {
|
|
424
1267
|
if (this.pollTimer || this.disposed)
|
|
@@ -433,7 +1276,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
433
1276
|
});
|
|
434
1277
|
});
|
|
435
1278
|
}, this.pollTickMs);
|
|
436
|
-
//
|
|
1279
|
+
// 不阻止进?
|
|
437
1280
|
this.pollTimer.unref?.();
|
|
438
1281
|
}
|
|
439
1282
|
stopPolling() {
|
|
@@ -443,9 +1286,9 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
443
1286
|
}
|
|
444
1287
|
}
|
|
445
1288
|
/**
|
|
446
|
-
*
|
|
1289
|
+
* 到期 poll 源并入队刷新(内容未变由 content-hash / 304 ?
|
|
447
1290
|
*
|
|
448
|
-
* @returns
|
|
1291
|
+
* @returns 实际触发?sourceId 列表
|
|
449
1292
|
*/
|
|
450
1293
|
async pollDueSources() {
|
|
451
1294
|
if (this.disposed)
|
|
@@ -475,8 +1318,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
475
1318
|
status: 'poll_due',
|
|
476
1319
|
detail: source.sync.intervalMs?.toString(),
|
|
477
1320
|
});
|
|
478
|
-
//
|
|
479
|
-
void this.ingestSource(source.id).catch((err) => {
|
|
1321
|
+
// 入队,不 tick量:?active 则跳过, supersede 打断
|
|
1322
|
+
void this.ingestSource(source.id, { incremental: true }).catch((err) => {
|
|
480
1323
|
this.emitProgress({
|
|
481
1324
|
type: 'error',
|
|
482
1325
|
sourceId: source.id,
|
|
@@ -487,79 +1330,126 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
487
1330
|
}
|
|
488
1331
|
return triggered;
|
|
489
1332
|
}
|
|
490
|
-
//
|
|
1333
|
+
// 内部
|
|
491
1334
|
hasQueuedJobs() {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
const
|
|
1335
|
+
return this.jobQueue.hasQueuedJobs();
|
|
1336
|
+
}
|
|
1337
|
+
/**
|
|
1338
|
+
* 入队(同步调度用;失败不抛——DB 已关/中止时静默丢弃)。
|
|
1339
|
+
* 需要计数的路径请 `await enqueueStrict`。
|
|
1340
|
+
*/
|
|
1341
|
+
enqueue(sourceId, kind, path, priority, fileIdHint) {
|
|
1342
|
+
return this.enqueueStrict(sourceId, kind, path, priority, fileIdHint).catch(() => false);
|
|
1343
|
+
}
|
|
1344
|
+
/** 入队并绑定真实 file_id(JobQueue 去重 / purge 清 job / abort 共享 File 守卫)。
|
|
1345
|
+
*
|
|
1346
|
+
* 共享 File:同 file_id 已有 parse 在途时只补 Membership,禁止双 parse。
|
|
1347
|
+
*/
|
|
1348
|
+
async enqueueStrict(sourceId, kind, path, priority, fileIdHint) {
|
|
1349
|
+
if (this.disposed)
|
|
1350
|
+
return false;
|
|
1351
|
+
const fileId = fileIdHint ?? (await this.resolveJobFileId(sourceId, kind, path));
|
|
1352
|
+
if (fileId && (kind === 'parse_file' || kind === 'fetch_doc')) {
|
|
1353
|
+
// 共享 File 已有在途 parse:补认领后跳过,避免双写
|
|
1354
|
+
const busy = this.sources.database.raw
|
|
512
1355
|
.prepare(`SELECT id FROM knowledge_jobs
|
|
513
|
-
WHERE
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
|
|
521
|
-
.get();
|
|
522
|
-
if ((depth?.n ?? 0) >= this.maxQueueDepth) {
|
|
523
|
-
this.emitProgress({
|
|
524
|
-
type: 'error',
|
|
525
|
-
sourceId,
|
|
526
|
-
status: 'queue_backpressure',
|
|
527
|
-
detail: `queued=${depth.n} (delay only, jobs kept)`,
|
|
528
|
-
});
|
|
1356
|
+
WHERE file_id = ? AND kind IN ('parse_file','fetch_doc') AND status IN ('queued','running')
|
|
1357
|
+
LIMIT 1`)
|
|
1358
|
+
.get(fileId);
|
|
1359
|
+
if (busy?.id && path) {
|
|
1360
|
+
this.index.attachMembership(sourceId, fileId, path);
|
|
1361
|
+
return false;
|
|
1362
|
+
}
|
|
529
1363
|
}
|
|
530
|
-
|
|
1364
|
+
const ok = this.jobQueue.enqueue(sourceId, kind, path, priority, fileId);
|
|
1365
|
+
if (ok && this.diskWatermarkAlert) {
|
|
531
1366
|
void this.warnDiskWatermark(sourceId);
|
|
532
1367
|
}
|
|
1368
|
+
return ok;
|
|
1369
|
+
}
|
|
1370
|
+
/** parse/fetch 成功后回填 job.file_id(新建 File 的首条 job 入队时尚无 id) */
|
|
1371
|
+
bindJobFileId(job) {
|
|
1372
|
+
if (job.file_id || !job.path)
|
|
1373
|
+
return;
|
|
1374
|
+
const rec = this.index.getFile(job.source_id, job.path);
|
|
1375
|
+
if (!rec?.id)
|
|
1376
|
+
return;
|
|
533
1377
|
this.sources.database.raw
|
|
534
|
-
.prepare(`
|
|
535
|
-
|
|
536
|
-
|
|
1378
|
+
.prepare(`UPDATE knowledge_jobs SET file_id = ? WHERE id = ? AND file_id IS NULL`)
|
|
1379
|
+
.run(rec.id, job.id);
|
|
1380
|
+
job.file_id = rec.id;
|
|
1381
|
+
}
|
|
1382
|
+
/** job 绑定 file_id:membership → identity;本地文件可 identify */
|
|
1383
|
+
async resolveJobFileId(sourceId, kind, path) {
|
|
1384
|
+
if (!path)
|
|
1385
|
+
return null;
|
|
1386
|
+
if (kind !== 'parse_file' && kind !== 'fetch_doc' && kind !== 'drop_file')
|
|
1387
|
+
return null;
|
|
1388
|
+
const logical = path.replace(/\\/g, '/');
|
|
1389
|
+
const existing = this.index.getFile(sourceId, logical);
|
|
1390
|
+
if (existing?.id)
|
|
1391
|
+
return existing.id;
|
|
1392
|
+
try {
|
|
1393
|
+
const ident = await identifyLocalFile(path);
|
|
1394
|
+
return this.index.findFileIdByIdentity(ident.key);
|
|
1395
|
+
}
|
|
1396
|
+
catch {
|
|
1397
|
+
return null;
|
|
1398
|
+
}
|
|
537
1399
|
}
|
|
538
1400
|
kick() {
|
|
539
|
-
if (this.disposed
|
|
1401
|
+
if (this.disposed)
|
|
1402
|
+
return;
|
|
1403
|
+
if (this.draining) {
|
|
1404
|
+
// 等待 drain 会再 kick;标记一次避免丢信号
|
|
1405
|
+
this.kickAgain = true;
|
|
540
1406
|
return;
|
|
1407
|
+
}
|
|
541
1408
|
this.draining = true;
|
|
542
1409
|
queueMicrotask(() => {
|
|
543
1410
|
this.draining = false;
|
|
544
1411
|
this.drain();
|
|
1412
|
+
if (this.kickAgain) {
|
|
1413
|
+
this.kickAgain = false;
|
|
1414
|
+
this.kick();
|
|
1415
|
+
}
|
|
545
1416
|
});
|
|
546
1417
|
}
|
|
547
1418
|
drain() {
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
1419
|
+
if (this.disposed)
|
|
1420
|
+
return;
|
|
1421
|
+
for (;;) {
|
|
1422
|
+
if (this.disposed)
|
|
551
1423
|
return;
|
|
552
|
-
const
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
1424
|
+
const allowParse = this.runningParse < this.parseConcurrency;
|
|
1425
|
+
const allowEmbed = this.runningEmbed < this.embedConcurrency;
|
|
1426
|
+
if (!allowParse && !allowEmbed)
|
|
1427
|
+
return;
|
|
1428
|
+
const kinds = [];
|
|
1429
|
+
if (allowParse)
|
|
1430
|
+
kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
|
|
1431
|
+
// parse 优先(priority 2 < 3),但 **不得** 等全部 parse 结束才 embed:
|
|
1432
|
+
// 大库 parse 可能跑数小时,否则向量永远 0。embed 与 parse 并行,仅受 embed 槽限制。
|
|
1433
|
+
if (allowEmbed)
|
|
1434
|
+
kinds.push('embed_source');
|
|
1435
|
+
if (kinds.length === 0) {
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
const job = this.claimJob(kinds, (j) => this.acceptParseSlot(j));
|
|
1439
|
+
if (!job) {
|
|
1440
|
+
// 队列清空:合并刷 parse/drop 标脏?coverage,避免每文件?COUNT
|
|
1441
|
+
this.flushCoverageDirty();
|
|
558
1442
|
return;
|
|
559
1443
|
}
|
|
560
|
-
|
|
1444
|
+
const isEmbed = job.kind === 'embed_source';
|
|
1445
|
+
const isDocParse = this.isDocParseJob(job);
|
|
561
1446
|
if (isEmbed)
|
|
562
1447
|
this.runningEmbed += 1;
|
|
1448
|
+
else {
|
|
1449
|
+
this.runningParse += 1;
|
|
1450
|
+
if (isDocParse)
|
|
1451
|
+
this.runningDocParse += 1;
|
|
1452
|
+
}
|
|
563
1453
|
void this.runJob(job)
|
|
564
1454
|
.catch((err) => {
|
|
565
1455
|
this.emitProgress({
|
|
@@ -571,58 +1461,116 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
571
1461
|
});
|
|
572
1462
|
})
|
|
573
1463
|
.finally(() => {
|
|
574
|
-
this.running -= 1;
|
|
575
1464
|
if (isEmbed)
|
|
576
1465
|
this.runningEmbed -= 1;
|
|
1466
|
+
else {
|
|
1467
|
+
this.runningParse -= 1;
|
|
1468
|
+
if (isDocParse)
|
|
1469
|
+
this.runningDocParse -= 1;
|
|
1470
|
+
}
|
|
577
1471
|
this.kick();
|
|
578
1472
|
});
|
|
579
1473
|
}
|
|
580
1474
|
}
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
.
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
1475
|
+
/** 文档抽取单独限流:多份 PDF/Office 同时抽会打爆内存与 Engine */
|
|
1476
|
+
isDocParseJob(job) {
|
|
1477
|
+
return job.kind === 'parse_file' && !!job.path && isDocumentPath(job.path);
|
|
1478
|
+
}
|
|
1479
|
+
acceptParseSlot(job) {
|
|
1480
|
+
if (this.isDocParseJob(job)) {
|
|
1481
|
+
return this.runningDocParse < this.documentParseConcurrency;
|
|
1482
|
+
}
|
|
1483
|
+
return true;
|
|
1484
|
+
}
|
|
1485
|
+
countQueuedKinds(kinds, sourceId) {
|
|
1486
|
+
if (this.disposed)
|
|
1487
|
+
return 0;
|
|
1488
|
+
return this.jobQueue.countQueuedKinds(kinds, sourceId);
|
|
1489
|
+
}
|
|
1490
|
+
claimJob(kinds, accept) {
|
|
1491
|
+
return this.jobQueue.claimJob(kinds, accept);
|
|
597
1492
|
}
|
|
598
1493
|
async runJob(job) {
|
|
599
1494
|
const now = Date.now();
|
|
1495
|
+
if (this.isAborted(job.source_id)) {
|
|
1496
|
+
this.sources.database.raw
|
|
1497
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
1498
|
+
WHERE id = ? AND status = 'running'`)
|
|
1499
|
+
.run(Date.now(), job.id);
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
600
1502
|
try {
|
|
1503
|
+
// 归属闸门:任何会碰本地盘 / 外源逻辑 path 任务都必须属于本?
|
|
1504
|
+
if (job.path && (job.kind === 'parse_file' || job.kind === 'fetch_doc')) {
|
|
1505
|
+
const owner = this.sources.get(job.source_id);
|
|
1506
|
+
if (owner && !this.sourceOwnsPath(owner, job.path)) {
|
|
1507
|
+
throw new Error(`path not owned by source: ${job.path}`);
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
601
1510
|
if (job.kind === 'parse_file' && job.path) {
|
|
602
|
-
|
|
603
|
-
this.
|
|
1511
|
+
// 单文件解析硬超时:大 xlsx/pdf 抽取会堵死事件循超时随体?
|
|
1512
|
+
let parseTimeoutMs = this.parseTimeoutMs;
|
|
1513
|
+
try {
|
|
1514
|
+
const st = await stat(job.path);
|
|
1515
|
+
if (st.isFile())
|
|
1516
|
+
parseTimeoutMs = parseTimeoutForSize(st.size, this.fileLimits);
|
|
1517
|
+
}
|
|
1518
|
+
catch {
|
|
1519
|
+
// 已删时交?parseOne ?
|
|
1520
|
+
}
|
|
1521
|
+
this.heartbeatJob(job.id);
|
|
1522
|
+
await this.withTimeout((signal) => this.parseOne(job.source_id, job.path, signal), parseTimeoutMs, `parse_timeout:${job.path}`, this.abortSignalFor(job.source_id));
|
|
1523
|
+
this.bindJobFileId(job);
|
|
1524
|
+
if (this.isAborted(job.source_id)) {
|
|
1525
|
+
throw new Error('aborted');
|
|
1526
|
+
}
|
|
1527
|
+
this.markCoverageDirty(job.source_id);
|
|
604
1528
|
if (this.embedding) {
|
|
605
|
-
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
1529
|
+
await this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
606
1530
|
this.kick();
|
|
607
1531
|
}
|
|
608
1532
|
}
|
|
609
1533
|
else if (job.kind === 'fetch_doc' && job.path) {
|
|
610
|
-
|
|
611
|
-
this.
|
|
1534
|
+
this.heartbeatJob(job.id);
|
|
1535
|
+
await this.withTimeout((signal) => this.fetchDocJob(job.source_id, job.path, signal), this.parseTimeoutMs, `fetch_timeout:${job.path}`, this.abortSignalFor(job.source_id));
|
|
1536
|
+
this.bindJobFileId(job);
|
|
1537
|
+
if (this.isAborted(job.source_id)) {
|
|
1538
|
+
throw new Error('aborted');
|
|
1539
|
+
}
|
|
1540
|
+
this.markCoverageDirty(job.source_id);
|
|
612
1541
|
if (this.embedding) {
|
|
613
|
-
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
1542
|
+
await this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
614
1543
|
this.kick();
|
|
615
1544
|
}
|
|
616
1545
|
}
|
|
617
1546
|
else if (job.kind === 'drop_file' && job.path) {
|
|
618
|
-
|
|
619
|
-
this.
|
|
1547
|
+
// 移出/删除:连同子树异步清(分批让出,禁止同步连环 DELETE 堵死 Engine)
|
|
1548
|
+
await this.index.removePathTreeAsync(job.source_id, job.path);
|
|
1549
|
+
this.markCoverageDirty(job.source_id);
|
|
620
1550
|
}
|
|
621
1551
|
else if (job.kind === 'walk_source') {
|
|
622
|
-
await this.ingestSource(job.source_id);
|
|
1552
|
+
await this.ingestSource(job.source_id, { fromQueue: true });
|
|
1553
|
+
if (this.isAborted(job.source_id)) {
|
|
1554
|
+
throw new Error('aborted');
|
|
1555
|
+
}
|
|
623
1556
|
}
|
|
624
1557
|
else if (job.kind === 'embed_source') {
|
|
625
|
-
|
|
1558
|
+
if (!this.embedding) {
|
|
1559
|
+
const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
|
|
1560
|
+
if (stillMissing) {
|
|
1561
|
+
// 空跑?done:迁移后缺向量却显示 queue=0
|
|
1562
|
+
throw new Error('embedding_provider_missing');
|
|
1563
|
+
}
|
|
1564
|
+
return;
|
|
1565
|
+
}
|
|
1566
|
+
await this.embedPending(job.source_id, 50, job.id);
|
|
1567
|
+
if (this.isAborted(job.source_id)) {
|
|
1568
|
+
throw new Error('aborted');
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
// 统一收尾闸门:任?kind 后都不得?done
|
|
1572
|
+
if (this.isAborted(job.source_id)) {
|
|
1573
|
+
throw new Error('aborted');
|
|
626
1574
|
}
|
|
627
1575
|
this.sources.database.raw
|
|
628
1576
|
.prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
|
|
@@ -633,24 +1581,66 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
633
1581
|
path: job.path ?? undefined,
|
|
634
1582
|
status: 'done',
|
|
635
1583
|
});
|
|
1584
|
+
// ?done 后再:否?enqueue 去重会把「自?active 而丢掉下?
|
|
1585
|
+
if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
|
|
1586
|
+
this.ensureEmbedJob(job.source_id);
|
|
1587
|
+
this.refreshCoverage(job.source_id, { force: true });
|
|
1588
|
+
}
|
|
636
1589
|
}
|
|
637
1590
|
catch (err) {
|
|
638
1591
|
const message = err instanceof Error ? err.message : String(err);
|
|
1592
|
+
const aborted = message === 'aborted' || this.isAborted(job.source_id);
|
|
1593
|
+
// 归属拒绝不是「文件解析失不得写入 knowledge_files
|
|
1594
|
+
const ownership = message.includes('not owned by source');
|
|
1595
|
+
if (job.kind === 'parse_file' && job.path && !aborted && !ownership) {
|
|
1596
|
+
this.index.markFileError(job.source_id, job.path, message);
|
|
1597
|
+
}
|
|
639
1598
|
this.sources.database.raw
|
|
640
1599
|
.prepare(`UPDATE knowledge_jobs
|
|
641
|
-
SET status =
|
|
1600
|
+
SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
|
|
642
1601
|
WHERE id = ?`)
|
|
643
|
-
.run(message, now, job.id);
|
|
1602
|
+
.run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, now, job.id);
|
|
1603
|
+
if (job.kind === 'embed_source' && !aborted) {
|
|
1604
|
+
this.ensureEmbedJob(job.source_id);
|
|
1605
|
+
}
|
|
1606
|
+
throw err;
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
/**
|
|
1610
|
+
* 带截止时间的消执行:超时?abort signal,工作侧必须在副作用?
|
|
1611
|
+
* 再用 Promise.race 包不消的 Promise会在超时后留下僵尸写?
|
|
1612
|
+
*/
|
|
1613
|
+
async withTimeout(work, ms, label, parent) {
|
|
1614
|
+
const controller = new AbortController();
|
|
1615
|
+
const onParentAbort = () => controller.abort();
|
|
1616
|
+
parent?.addEventListener('abort', onParentAbort, { once: true });
|
|
1617
|
+
const timer = setTimeout(() => controller.abort(), ms);
|
|
1618
|
+
try {
|
|
1619
|
+
return await work(controller.signal);
|
|
1620
|
+
}
|
|
1621
|
+
catch (err) {
|
|
1622
|
+
if (controller.signal.aborted) {
|
|
1623
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1624
|
+
if (msg === 'aborted' || msg.includes('abort'))
|
|
1625
|
+
throw new Error(label);
|
|
1626
|
+
throw new Error(label);
|
|
1627
|
+
}
|
|
644
1628
|
throw err;
|
|
645
1629
|
}
|
|
1630
|
+
finally {
|
|
1631
|
+
clearTimeout(timer);
|
|
1632
|
+
parent?.removeEventListener('abort', onParentAbort);
|
|
1633
|
+
}
|
|
646
1634
|
}
|
|
647
1635
|
/**
|
|
648
|
-
*
|
|
1636
|
+
* 外源单文?job:按逻辑?fetch + 索引
|
|
649
1637
|
*/
|
|
650
|
-
async fetchDocJob(sourceId, path) {
|
|
1638
|
+
async fetchDocJob(sourceId, path, signal) {
|
|
651
1639
|
const source = this.sources.get(sourceId);
|
|
652
1640
|
if (!source || source.status === 'removed')
|
|
653
1641
|
return;
|
|
1642
|
+
if (signal?.aborted)
|
|
1643
|
+
throw new Error('aborted');
|
|
654
1644
|
const cred = await this.resolveCred(source);
|
|
655
1645
|
const existing = this.index.getFile(sourceId, path);
|
|
656
1646
|
const ref = {
|
|
@@ -666,21 +1656,21 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
666
1656
|
: this.localFetcher;
|
|
667
1657
|
const doc = await fetcher.fetch(source, ref, cred);
|
|
668
1658
|
if (doc)
|
|
669
|
-
this.indexVirtualDoc(sourceId, doc);
|
|
1659
|
+
await this.indexVirtualDoc(sourceId, doc, source.authRef);
|
|
670
1660
|
}
|
|
671
|
-
async parseOne(sourceId, filePath) {
|
|
1661
|
+
async parseOne(sourceId, filePath, signal) {
|
|
1662
|
+
const source = this.sources.get(sourceId);
|
|
1663
|
+
// 防御深度:即使队污染,也?root 外的文件
|
|
1664
|
+
if (source && !this.sourceOwnsPath(source, filePath)) {
|
|
1665
|
+
throw new Error(`path not owned by source: ${filePath}`);
|
|
1666
|
+
}
|
|
1667
|
+
if (signal?.aborted)
|
|
1668
|
+
throw new Error('aborted');
|
|
672
1669
|
if (this.shouldSkipFile(filePath)) {
|
|
673
1670
|
this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
|
|
674
1671
|
return false;
|
|
675
1672
|
}
|
|
676
|
-
|
|
677
|
-
if (!adapter) {
|
|
678
|
-
if (this.documentPort && isDocumentPath(filePath)) {
|
|
679
|
-
return this.parseDocumentFile(sourceId, filePath);
|
|
680
|
-
}
|
|
681
|
-
this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
|
|
682
|
-
return false;
|
|
683
|
-
}
|
|
1673
|
+
// 先确认是件:/节点不得?knowledge_files(历史上?no_adapter?
|
|
684
1674
|
let st;
|
|
685
1675
|
try {
|
|
686
1676
|
st = await stat(filePath);
|
|
@@ -689,47 +1679,83 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
689
1679
|
this.index.removeFile(sourceId, filePath);
|
|
690
1680
|
return false;
|
|
691
1681
|
}
|
|
692
|
-
if (!st.isFile())
|
|
1682
|
+
if (!st.isFile()) {
|
|
1683
|
+
this.index.removeFile(sourceId, filePath);
|
|
693
1684
|
return false;
|
|
694
|
-
|
|
695
|
-
|
|
1685
|
+
}
|
|
1686
|
+
const adapter = this.adapters.match(filePath);
|
|
1687
|
+
if (!adapter) {
|
|
1688
|
+
// worker DocumentPort;注?port 仅作 worker 不可用时的回
|
|
1689
|
+
if (isDocumentPath(filePath)) {
|
|
1690
|
+
return this.parseDocumentFile(sourceId, filePath, signal);
|
|
1691
|
+
}
|
|
1692
|
+
this.index.markFileSkipped(sourceId, filePath, 'no_adapter', st.size, await this.identityKeyFor(filePath));
|
|
696
1693
|
return false;
|
|
697
1694
|
}
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
1695
|
+
// Axure 导出 data.js/document.js:内容嗅探后 skip,避免误伤普通 data.js
|
|
1696
|
+
if (/\.(js|cjs|mjs)$/i.test(filePath)) {
|
|
1697
|
+
const head = await this.readFileHead(filePath, 256);
|
|
1698
|
+
if (head && looksLikeAxureExportJs(head)) {
|
|
1699
|
+
this.index.markFileSkipped(sourceId, filePath, 'axure_export_sidecar', st.size, await this.identityKeyFor(filePath));
|
|
1700
|
+
return false;
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
const kind = classifyFileKind(filePath);
|
|
1704
|
+
const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
|
|
1705
|
+
if (sizeDecision.action === 'skip') {
|
|
1706
|
+
this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size, await this.identityKeyFor(filePath));
|
|
1707
|
+
return false;
|
|
702
1708
|
}
|
|
1709
|
+
// 读盘 + hash + 切块 + FTS token 全在 worker(Service 事件循环只做写库)
|
|
1710
|
+
const maxTextChars = this.fileLimits.partial.maxTextChars;
|
|
1711
|
+
if (signal?.aborted)
|
|
1712
|
+
throw new Error('aborted');
|
|
703
1713
|
try {
|
|
704
|
-
const
|
|
705
|
-
|
|
1714
|
+
const { parseTextInWorker } = await import('./parse-text-in-worker.js');
|
|
1715
|
+
const parsed = await parseTextInWorker(filePath, {
|
|
1716
|
+
sizeDecision: sizeDecision.action === 'partial' ? 'partial' : 'ok',
|
|
1717
|
+
maxTextChars,
|
|
1718
|
+
timeoutMs: parseTimeoutForSize(st.size, this.fileLimits),
|
|
1719
|
+
signal,
|
|
1720
|
+
});
|
|
1721
|
+
if (this.index.isFresh(sourceId, filePath, parsed.contentHash)) {
|
|
1722
|
+
return true;
|
|
1723
|
+
}
|
|
1724
|
+
if (signal?.aborted)
|
|
1725
|
+
throw new Error('aborted');
|
|
1726
|
+
const ident = await identifyLocalFile(filePath);
|
|
1727
|
+
await this.index.upsertFile({
|
|
706
1728
|
sourceId,
|
|
707
1729
|
path: filePath,
|
|
708
|
-
contentHash,
|
|
1730
|
+
contentHash: parsed.contentHash,
|
|
709
1731
|
size: st.size,
|
|
710
1732
|
mtime: Math.floor(st.mtimeMs),
|
|
711
|
-
adapterId:
|
|
712
|
-
chunks,
|
|
713
|
-
|
|
1733
|
+
adapterId: parsed.adapterId,
|
|
1734
|
+
chunks: parsed.chunks,
|
|
1735
|
+
identityKey: ident.key,
|
|
1736
|
+
}, { signal });
|
|
714
1737
|
return true;
|
|
715
1738
|
}
|
|
716
1739
|
catch (err) {
|
|
717
|
-
|
|
1740
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1741
|
+
if (msg === 'aborted' || msg.startsWith('text_parse_timeout') || signal?.aborted) {
|
|
1742
|
+
throw err;
|
|
1743
|
+
}
|
|
1744
|
+
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
|
|
718
1745
|
return false;
|
|
719
1746
|
}
|
|
720
1747
|
}
|
|
721
1748
|
/**
|
|
722
|
-
* PDF/Office:DocumentPort
|
|
723
|
-
* 新鲜度以原件字节 hash
|
|
1749
|
+
* PDF/Office:DocumentPort 抽取?Markdown 后走 markdownAdapter 切块?
|
|
1750
|
+
* 新鲜度以原件字节 hash 为准?
|
|
724
1751
|
*
|
|
725
|
-
* @param sourceId -
|
|
726
|
-
* @param filePath -
|
|
727
|
-
* @returns
|
|
1752
|
+
* @param sourceId - 知识?id
|
|
1753
|
+
* @param filePath - 文件
|
|
1754
|
+
* @returns 成功入索?
|
|
728
1755
|
*/
|
|
729
|
-
async parseDocumentFile(sourceId, filePath) {
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
return false;
|
|
1756
|
+
async parseDocumentFile(sourceId, filePath, signal) {
|
|
1757
|
+
if (signal?.aborted)
|
|
1758
|
+
throw new Error('aborted');
|
|
733
1759
|
let st;
|
|
734
1760
|
try {
|
|
735
1761
|
st = await stat(filePath);
|
|
@@ -740,96 +1766,167 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
740
1766
|
}
|
|
741
1767
|
if (!st.isFile())
|
|
742
1768
|
return false;
|
|
743
|
-
|
|
744
|
-
|
|
1769
|
+
const kind = classifyFileKind(filePath);
|
|
1770
|
+
const sizeDecision = decideBySize(st.size, kind, this.fileLimits);
|
|
1771
|
+
if (sizeDecision.action === 'skip') {
|
|
1772
|
+
this.index.markFileSkipped(sourceId, filePath, sizeDecision.reason, st.size);
|
|
745
1773
|
return false;
|
|
746
1774
|
}
|
|
747
|
-
const raw = await readFile(filePath);
|
|
748
|
-
const contentHash = hashContent(raw);
|
|
749
|
-
if (this.index.isFresh(sourceId, filePath, contentHash)) {
|
|
750
|
-
return true;
|
|
751
|
-
}
|
|
752
1775
|
try {
|
|
753
|
-
|
|
754
|
-
const
|
|
755
|
-
|
|
756
|
-
|
|
1776
|
+
// worker 抽取 + 流式 hash + 切块 + ftsToks:xlsx/pdf 不堵 Service 事件循环
|
|
1777
|
+
const extracted = await this.extractDocument(filePath, sourceId, st.size, signal);
|
|
1778
|
+
const contentHash = extracted.contentHash;
|
|
1779
|
+
if (this.index.isFresh(sourceId, filePath, contentHash)) {
|
|
1780
|
+
return true;
|
|
1781
|
+
}
|
|
1782
|
+
const markdown = extracted.result.markdown?.trim();
|
|
1783
|
+
if (!markdown || extracted.chunks.length === 0) {
|
|
1784
|
+
this.index.markFileSkipped(sourceId, filePath, 'empty_content', st.size);
|
|
757
1785
|
return false;
|
|
758
1786
|
}
|
|
759
|
-
|
|
760
|
-
|
|
1787
|
+
if (signal?.aborted)
|
|
1788
|
+
throw new Error('aborted');
|
|
1789
|
+
const ident = await identifyLocalFile(filePath);
|
|
1790
|
+
await this.index.upsertFile({
|
|
761
1791
|
sourceId,
|
|
762
1792
|
path: filePath,
|
|
763
1793
|
contentHash,
|
|
764
1794
|
size: st.size,
|
|
765
1795
|
mtime: Math.floor(st.mtimeMs),
|
|
766
|
-
adapterId: `document:${result.backend}`,
|
|
767
|
-
chunks,
|
|
768
|
-
|
|
1796
|
+
adapterId: `document:${extracted.result.backend}`,
|
|
1797
|
+
chunks: extracted.chunks,
|
|
1798
|
+
identityKey: ident.key,
|
|
1799
|
+
}, { signal });
|
|
769
1800
|
return true;
|
|
770
1801
|
}
|
|
771
1802
|
catch (err) {
|
|
1803
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1804
|
+
// /超时不是「这件解析失必须上抛?job ?failed止静?done
|
|
1805
|
+
if (msg === 'aborted' || msg.startsWith('parse_timeout') || signal?.aborted) {
|
|
1806
|
+
throw err;
|
|
1807
|
+
}
|
|
772
1808
|
if (isDocumentExtractError(err)) {
|
|
773
|
-
//
|
|
774
|
-
this.
|
|
1809
|
+
// 结构化跳过(密码/老格?超大等),不挡整?
|
|
1810
|
+
const identKey = await this.identityKeyFor(filePath);
|
|
1811
|
+
this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase(), st.size, identKey);
|
|
775
1812
|
return false;
|
|
776
1813
|
}
|
|
777
|
-
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
|
|
1814
|
+
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err), st.size, await this.identityKeyFor(filePath));
|
|
778
1815
|
return false;
|
|
779
1816
|
}
|
|
780
1817
|
}
|
|
781
|
-
/**
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
1818
|
+
/** worker 抽取 + 流式 hash + 切块;注入 port 时(测试)由调用侧流式 hash */
|
|
1819
|
+
async extractDocument(filePath, sourceId, fileSize = 0, extraSignal) {
|
|
1820
|
+
const sourceSignal = this.abortSignalFor(sourceId);
|
|
1821
|
+
const controller = new AbortController();
|
|
1822
|
+
const abort = () => controller.abort();
|
|
1823
|
+
sourceSignal.addEventListener('abort', abort, { once: true });
|
|
1824
|
+
extraSignal?.addEventListener('abort', abort, { once: true });
|
|
1825
|
+
const signal = controller.signal;
|
|
1826
|
+
const timeoutMs = parseTimeoutForSize(fileSize, this.fileLimits);
|
|
1827
|
+
const extractOpts = {
|
|
1828
|
+
timeoutMs,
|
|
1829
|
+
// knowledge hardMax:软超限仍可 partial 抽取
|
|
1830
|
+
maxFileBytes: this.fileLimits.hardMaxFileBytes,
|
|
1831
|
+
maxSheets: this.fileLimits.partial.maxSheets,
|
|
1832
|
+
maxRowsPerSheet: this.fileLimits.partial.maxRowsPerSheet,
|
|
1833
|
+
maxPages: this.fileLimits.partial.maxPdfPages,
|
|
1834
|
+
maxTextChars: this.fileLimits.partial.maxTextChars,
|
|
1835
|
+
};
|
|
795
1836
|
try {
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
const
|
|
799
|
-
|
|
800
|
-
|
|
1837
|
+
// 注入的 documentPort 是调用方选定的抽取实现(测试 stub / 自定义后端)
|
|
1838
|
+
if (this.documentPort) {
|
|
1839
|
+
const result = await this.documentPort.extract({ path: filePath, name: filePath }, { ...extractOpts, signal });
|
|
1840
|
+
const markdown = result.markdown?.trim() ?? '';
|
|
1841
|
+
const chunks = markdown
|
|
1842
|
+
? await this.chunkTextInWorker(markdown, filePath, signal)
|
|
1843
|
+
: [];
|
|
1844
|
+
const contentHash = await hashFileStreaming(filePath, signal);
|
|
1845
|
+
return { result, chunks, contentHash };
|
|
801
1846
|
}
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
detail: `embedded=${pending.length}`,
|
|
1847
|
+
const { extractDocumentInWorker } = await import('./extract-document.js');
|
|
1848
|
+
return await extractDocumentInWorker(filePath, {
|
|
1849
|
+
...extractOpts,
|
|
1850
|
+
documentConfig: this.documentConfig,
|
|
1851
|
+
signal,
|
|
808
1852
|
});
|
|
809
1853
|
}
|
|
810
1854
|
catch (err) {
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
});
|
|
817
|
-
return;
|
|
1855
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1856
|
+
if (msg === 'extract_aborted' || signal.aborted) {
|
|
1857
|
+
throw new Error('aborted');
|
|
1858
|
+
}
|
|
1859
|
+
throw err;
|
|
818
1860
|
}
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
this.kick();
|
|
1861
|
+
finally {
|
|
1862
|
+
sourceSignal.removeEventListener('abort', abort);
|
|
1863
|
+
extraSignal?.removeEventListener('abort', abort);
|
|
823
1864
|
}
|
|
824
|
-
this.refreshCoverage(sourceId);
|
|
825
1865
|
}
|
|
826
|
-
|
|
1866
|
+
/** 纯文本切块 + ftsToks(虚拟文档 / port 回退共用) */
|
|
1867
|
+
async chunkTextInWorker(text, path, signal, adapterId) {
|
|
1868
|
+
const { chunkTextWithFtsToks } = await import('./parse-text-in-worker.js');
|
|
1869
|
+
return chunkTextWithFtsToks(text, path, signal, adapterId);
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* Phase B:嵌入待处理 chunk
|
|
1873
|
+
*
|
|
1874
|
+
* ?job 内按 embedConcurrency ****多批调用 API?
|
|
1875
|
+
* 失败/超时?re-enqueue,避?job ?done ?coverage 永久卡住?
|
|
1876
|
+
*/
|
|
1877
|
+
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
1878
|
+
await this.embedRunner.embedPending(sourceId, maxRounds, jobId);
|
|
1879
|
+
}
|
|
1880
|
+
recordEmbedSecretHit(input) {
|
|
1881
|
+
this.sources.database.raw
|
|
1882
|
+
.prepare(`INSERT INTO knowledge_embed_secret_log (id, source_id, chunk_id, action, hits_json, created_at)
|
|
1883
|
+
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
1884
|
+
.run(`kes_${randomUUID().slice(0, 12)}`, input.sourceId, input.chunkId, input.action, JSON.stringify(input.hits), Date.now());
|
|
1885
|
+
}
|
|
1886
|
+
/** 仍有缺向量则排队下一?embed_source(须在当?job ?done 之后调用?*/
|
|
1887
|
+
ensureEmbedJob(sourceId) {
|
|
1888
|
+
if (!this.embedding)
|
|
1889
|
+
return;
|
|
1890
|
+
if (this.isAborted(sourceId))
|
|
1891
|
+
return;
|
|
1892
|
+
if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
|
|
1893
|
+
return;
|
|
1894
|
+
void this.enqueue(sourceId, 'embed_source', null, 3)
|
|
1895
|
+
.then(() => this.kick())
|
|
1896
|
+
.catch(() => undefined);
|
|
1897
|
+
}
|
|
1898
|
+
heartbeatJob(jobId) {
|
|
1899
|
+
this.jobQueue.heartbeat(jobId);
|
|
1900
|
+
}
|
|
1901
|
+
/** parse 完成不立刻刷 coverage(O(N) 写放大);标脏,drain/reconcile 时合并刷?*/
|
|
1902
|
+
coverageDirty = new Set();
|
|
1903
|
+
markCoverageDirty(sourceId) {
|
|
1904
|
+
this.coverageDirty.add(sourceId);
|
|
1905
|
+
}
|
|
1906
|
+
flushCoverageDirty(sourceId) {
|
|
1907
|
+
const ids = sourceId ? [sourceId] : [...this.coverageDirty];
|
|
1908
|
+
if (sourceId)
|
|
1909
|
+
this.coverageDirty.delete(sourceId);
|
|
1910
|
+
else
|
|
1911
|
+
this.coverageDirty.clear();
|
|
1912
|
+
for (const id of ids)
|
|
1913
|
+
this.refreshCoverage(id, { force: true });
|
|
1914
|
+
}
|
|
1915
|
+
lastCoverageRefreshAt = new Map();
|
|
1916
|
+
refreshCoverage(sourceId, opts) {
|
|
1917
|
+
const now = Date.now();
|
|
1918
|
+
const last = this.lastCoverageRefreshAt.get(sourceId) ?? 0;
|
|
1919
|
+
// 大库 sourceStats/embeddingCoverage 是多表 COUNT;解析/嵌入高频调用时节流,
|
|
1920
|
+
// 避免 Engine 线程被统计查询堵住导致 HTTP list 超时。收尾路径 force。
|
|
1921
|
+
if (!opts?.force && now - last < 2_000)
|
|
1922
|
+
return;
|
|
1923
|
+
this.lastCoverageRefreshAt.set(sourceId, now);
|
|
827
1924
|
const stats = this.index.sourceStats(sourceId);
|
|
828
1925
|
const source = this.sources.get(sourceId);
|
|
829
1926
|
if (!source)
|
|
830
1927
|
return;
|
|
831
1928
|
const discovered = this.discoveredFiles.get(sourceId) ?? 0;
|
|
832
|
-
// Phase A:已落库文件(indexed+skipped+error
|
|
1929
|
+
// Phase A:已落库文件(indexed+skipped+error? walk 发现?
|
|
833
1930
|
const phaseA = discovered > 0
|
|
834
1931
|
? Math.min(1, stats.files / discovered)
|
|
835
1932
|
: stats.files > 0
|
|
@@ -865,10 +1962,29 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
865
1962
|
});
|
|
866
1963
|
}
|
|
867
1964
|
emitProgress(evt) {
|
|
868
|
-
|
|
869
|
-
|
|
1965
|
+
let out = evt;
|
|
1966
|
+
if (evt.sourceId && evt.registeredBy === undefined) {
|
|
1967
|
+
try {
|
|
1968
|
+
const row = this.sources.database.raw
|
|
1969
|
+
.prepare('SELECT tenant_id, registered_by, visibility FROM knowledge_sources WHERE id = ?')
|
|
1970
|
+
.get(evt.sourceId);
|
|
1971
|
+
if (row) {
|
|
1972
|
+
out = {
|
|
1973
|
+
...evt,
|
|
1974
|
+
tenantId: row.tenant_id,
|
|
1975
|
+
registeredBy: row.registered_by,
|
|
1976
|
+
visibility: row.visibility,
|
|
1977
|
+
};
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
catch {
|
|
1981
|
+
/* 源已删时保持原事件;SSE 侧按无归属放行到 fallback */
|
|
1982
|
+
}
|
|
1983
|
+
}
|
|
1984
|
+
this.emit('progress', out);
|
|
1985
|
+
this.emit('knowledge.index.progress', out);
|
|
870
1986
|
}
|
|
871
|
-
/**
|
|
1987
|
+
/** 磁盘水位(仅,不务) */
|
|
872
1988
|
async warnDiskWatermark(sourceId) {
|
|
873
1989
|
try {
|
|
874
1990
|
const { statfs } = await import('node:fs/promises');
|
|
@@ -881,7 +1997,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
881
1997
|
type: 'error',
|
|
882
1998
|
sourceId,
|
|
883
1999
|
status: 'disk_watermark',
|
|
884
|
-
detail: `free
|
|
2000
|
+
detail: `free?{Math.round(freeMb)}MB (jobs kept)`,
|
|
885
2001
|
});
|
|
886
2002
|
}
|
|
887
2003
|
}
|