@octopi-agent/engine 0.55.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +38 -0
- package/dist/config-schema/context.js +39 -0
- package/dist/config-schema/engine.d.ts +42 -0
- package/dist/config-schema/engine.js +3 -1
- package/dist/config-schema/substrate.d.ts +5 -0
- package/dist/config-schema/substrate.js +11 -0
- package/dist/config.d.ts +44 -0
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/context/capabilities/document/backends/docx.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/docx.js +67 -0
- package/dist/harness/context/capabilities/document/backends/office-parser.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/office-parser.js +76 -0
- package/dist/harness/context/capabilities/document/backends/pdf.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/pdf.js +86 -0
- package/dist/harness/context/capabilities/document/backends/plain.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/plain.js +42 -0
- package/dist/harness/context/capabilities/document/backends/sheet.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/sheet.js +90 -0
- package/dist/harness/context/capabilities/document/backends/xmind.d.ts +20 -0
- package/dist/harness/context/capabilities/document/backends/xmind.js +198 -0
- package/dist/harness/context/capabilities/document/errors.d.ts +27 -0
- package/dist/harness/context/capabilities/document/errors.js +33 -0
- package/dist/harness/context/capabilities/document/format.d.ts +38 -0
- package/dist/harness/context/capabilities/document/format.js +113 -0
- package/dist/harness/context/capabilities/document/html-to-md.d.ts +14 -0
- package/dist/harness/context/capabilities/document/html-to-md.js +97 -0
- package/dist/harness/context/capabilities/document/index.d.ts +21 -0
- package/dist/harness/context/capabilities/document/index.js +20 -0
- package/dist/harness/context/capabilities/document/legacy/factory.d.ts +13 -0
- package/dist/harness/context/capabilities/document/legacy/factory.js +23 -0
- package/dist/harness/context/capabilities/document/legacy/soffice.d.ts +32 -0
- package/dist/harness/context/capabilities/document/legacy/soffice.js +186 -0
- package/dist/harness/context/capabilities/document/port.d.ts +19 -0
- package/dist/harness/context/capabilities/document/port.js +400 -0
- package/dist/harness/context/capabilities/document/types.d.ts +149 -0
- package/dist/harness/context/capabilities/document/types.js +8 -0
- package/dist/harness/context/capabilities/index.d.ts +1 -0
- package/dist/harness/context/capabilities/index.js +1 -0
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.d.ts +64 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +155 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +4 -1
- package/dist/harness/index.d.ts +3 -3
- package/dist/harness/index.js +2 -2
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/db.d.ts +4 -0
- package/dist/harness/knowledge/db.js +58 -3
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +25 -0
- package/dist/harness/knowledge/extract-document.d.ts +17 -0
- package/dist/harness/knowledge/extract-document.js +62 -0
- package/dist/harness/knowledge/index-store.d.ts +49 -5
- package/dist/harness/knowledge/index-store.js +240 -32
- package/dist/harness/knowledge/index.d.ts +2 -0
- package/dist/harness/knowledge/index.js +1 -0
- package/dist/harness/knowledge/ingest.d.ts +123 -2
- package/dist/harness/knowledge/ingest.js +661 -59
- package/dist/harness/knowledge/retriever.d.ts +8 -0
- package/dist/harness/knowledge/retriever.js +33 -1
- package/dist/harness/knowledge/source-store.d.ts +4 -0
- package/dist/harness/knowledge/source-store.js +62 -6
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
- package/dist/harness/memory/sqlite/keyword-search.js +40 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/extract.d.ts +3 -2
- package/dist/harness/session/attachments/extract.js +19 -6
- package/dist/harness/session/attachments/service.d.ts +5 -1
- package/dist/harness/session/attachments/service.js +44 -2
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/package.json +8 -3
|
@@ -14,6 +14,8 @@ import { hashContent, KnowledgeIndexStore } from './index-store.js';
|
|
|
14
14
|
import { LocalFsFetcher, UrlFetcher } from './fetchers.js';
|
|
15
15
|
import { ConnectorRegistry } from './connectors.js';
|
|
16
16
|
import { ConnectorFetcher } from './connector-fetcher.js';
|
|
17
|
+
import { isDocumentPath } from '../context/capabilities/document/format.js';
|
|
18
|
+
import { isDocumentExtractError } from '../context/capabilities/document/errors.js';
|
|
17
19
|
export class KnowledgeIngest extends EventEmitter {
|
|
18
20
|
sources;
|
|
19
21
|
index;
|
|
@@ -22,6 +24,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
22
24
|
maxQueueDepth;
|
|
23
25
|
debounceMs;
|
|
24
26
|
maxFileBytes;
|
|
27
|
+
/** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
|
|
28
|
+
parseTimeoutMs;
|
|
25
29
|
embedding;
|
|
26
30
|
embedBatch;
|
|
27
31
|
embedMinIntervalMs;
|
|
@@ -34,11 +38,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
34
38
|
pollTickMs;
|
|
35
39
|
pollMinIntervalMs;
|
|
36
40
|
maxPollPerTick;
|
|
41
|
+
documentPort;
|
|
37
42
|
lastEmbedAt = 0;
|
|
38
43
|
pollTimer = null;
|
|
39
|
-
|
|
44
|
+
reconcileTimer = null;
|
|
45
|
+
/** 源级中止控制器(手动 abort / 重建前清场) */
|
|
46
|
+
abortBySource = new Map();
|
|
47
|
+
/** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
|
|
48
|
+
onSourceSettled;
|
|
49
|
+
/** 上次 parse 缺口扫描时间 */
|
|
50
|
+
lastParseGapScanAt = new Map();
|
|
51
|
+
runningParse = 0;
|
|
40
52
|
runningEmbed = 0;
|
|
41
53
|
draining = false;
|
|
54
|
+
kickAgain = false;
|
|
42
55
|
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
43
56
|
sourceLocks = new Set();
|
|
44
57
|
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
@@ -51,18 +64,22 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
51
64
|
this.sources = options.sourceStore;
|
|
52
65
|
this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
|
|
53
66
|
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
54
|
-
this.parseConcurrency = options.parseConcurrency ??
|
|
67
|
+
this.parseConcurrency = options.parseConcurrency ?? 8;
|
|
55
68
|
this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
|
|
56
|
-
|
|
69
|
+
// watch 批量落盘时缩短静默窗,新增文件更快入队
|
|
70
|
+
this.debounceMs = options.debounceMs ?? 800;
|
|
57
71
|
this.maxFileBytes = options.maxFileBytes ?? 5_000_000;
|
|
72
|
+
this.parseTimeoutMs = options.parseTimeoutMs ?? 45_000;
|
|
58
73
|
this.embedding = options.embeddingProvider ?? null;
|
|
59
74
|
this.embedBatch = options.embedBatch ?? 32;
|
|
60
75
|
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
61
|
-
|
|
76
|
+
// embed 与 parse 分槽;缺省 4 路并行打 embedding API
|
|
77
|
+
this.embedConcurrency = options.embedConcurrency ?? 4;
|
|
62
78
|
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
63
79
|
this.credentials = options.credentials ?? null;
|
|
80
|
+
this.documentPort = options.documentPort ?? null;
|
|
64
81
|
this.localFetcher =
|
|
65
|
-
options.fetchers?.local ?? new LocalFsFetcher((p) => this.
|
|
82
|
+
options.fetchers?.local ?? new LocalFsFetcher((p) => this.shouldSkipFile(p));
|
|
66
83
|
this.urlFetcher = options.fetchers?.url ?? new UrlFetcher();
|
|
67
84
|
this.connectorFetcher =
|
|
68
85
|
options.fetchers?.connector ??
|
|
@@ -70,14 +87,38 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
70
87
|
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
71
88
|
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
72
89
|
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
90
|
+
// 崩溃/重启遗留的 running 会堵住 enqueue 去重并永久显示 indexing
|
|
91
|
+
this.reclaimOrphanRunningJobs();
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
|
|
95
|
+
* 仅构造时调用——运行中同进程的 running 是合法的。
|
|
96
|
+
*/
|
|
97
|
+
reclaimOrphanRunningJobs() {
|
|
98
|
+
this.sources.database.raw
|
|
99
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ? WHERE status = 'running'`)
|
|
100
|
+
.run(Date.now());
|
|
73
101
|
}
|
|
74
102
|
get indexStore() {
|
|
75
103
|
return this.index;
|
|
76
104
|
}
|
|
105
|
+
/**
|
|
106
|
+
* 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
|
|
107
|
+
*
|
|
108
|
+
* @param p - 文件路径
|
|
109
|
+
* @returns 是否跳过
|
|
110
|
+
*/
|
|
111
|
+
shouldSkipFile(p) {
|
|
112
|
+
if (this.documentPort && isDocumentPath(p))
|
|
113
|
+
return false;
|
|
114
|
+
return this.adapters.shouldSkipPath(p);
|
|
115
|
+
}
|
|
77
116
|
/**
|
|
78
117
|
* 全量/增量索引某源(Phase A)
|
|
79
118
|
*/
|
|
80
119
|
async ingestSource(sourceId, opts) {
|
|
120
|
+
// 重建 = supersede:以本次磁盘扫描为准,作废本源未完成任务
|
|
121
|
+
this.supersedeSourceWork(sourceId);
|
|
81
122
|
const source = this.sources.get(sourceId);
|
|
82
123
|
if (!source || source.status === 'removed') {
|
|
83
124
|
throw new Error(`knowledge source not found: ${sourceId}`);
|
|
@@ -355,7 +396,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
355
396
|
if (!filename)
|
|
356
397
|
return;
|
|
357
398
|
const abs = join(source.location, filename.toString().split(sep).join(sep));
|
|
358
|
-
if (this.
|
|
399
|
+
if (this.shouldSkipFile(abs))
|
|
359
400
|
return;
|
|
360
401
|
this.handleFsChange(sourceId, abs, event === 'rename' ? 'rename' : 'change');
|
|
361
402
|
});
|
|
@@ -388,7 +429,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
388
429
|
*/
|
|
389
430
|
async idle(timeoutMs = 30_000) {
|
|
390
431
|
const start = Date.now();
|
|
391
|
-
while (this.
|
|
432
|
+
while (this.runningParse > 0 || this.runningEmbed > 0 || this.hasQueuedJobs()) {
|
|
392
433
|
if (Date.now() - start > timeoutMs) {
|
|
393
434
|
throw new Error('knowledge ingest idle timeout');
|
|
394
435
|
}
|
|
@@ -399,9 +440,344 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
399
440
|
dispose() {
|
|
400
441
|
this.disposed = true;
|
|
401
442
|
this.stopPolling();
|
|
443
|
+
this.stopReconciler();
|
|
444
|
+
void this.abortJobs();
|
|
402
445
|
for (const id of this.watchers.keys())
|
|
403
446
|
this.stopWatch(id);
|
|
404
447
|
}
|
|
448
|
+
/**
|
|
449
|
+
* 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
|
|
450
|
+
*
|
|
451
|
+
* @param opts.sourceId - 仅中止该源;省略则全部源
|
|
452
|
+
* @returns 取消统计
|
|
453
|
+
*/
|
|
454
|
+
abortJobs(opts) {
|
|
455
|
+
const sid = opts?.sourceId?.trim();
|
|
456
|
+
const ids = sid ? [sid] : [...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)];
|
|
457
|
+
const unique = [...new Set(ids)];
|
|
458
|
+
let cancelledQueued = 0;
|
|
459
|
+
let abortedRunning = 0;
|
|
460
|
+
// 1) 清空 queued → cancelled
|
|
461
|
+
for (const id of unique) {
|
|
462
|
+
const res = this.sources.database.raw
|
|
463
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
464
|
+
WHERE source_id = ? AND status = 'queued'`)
|
|
465
|
+
.run(Date.now(), id);
|
|
466
|
+
cancelledQueued += Number(res.changes ?? 0);
|
|
467
|
+
// 2) 打断在跑(worker terminate / embed 循环退出);保持 aborted 直到下次 reindex
|
|
468
|
+
let ctrl = this.abortBySource.get(id);
|
|
469
|
+
if (!ctrl) {
|
|
470
|
+
ctrl = new AbortController();
|
|
471
|
+
this.abortBySource.set(id, ctrl);
|
|
472
|
+
}
|
|
473
|
+
if (!ctrl.signal.aborted) {
|
|
474
|
+
ctrl.abort();
|
|
475
|
+
abortedRunning += 1;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
// 3) 看门狗不要再把 aborted running 收回 queued
|
|
479
|
+
this.sources.database.raw
|
|
480
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
481
|
+
WHERE status = 'running' AND last_error = 'aborted'`)
|
|
482
|
+
.run(Date.now());
|
|
483
|
+
this.emitProgress({
|
|
484
|
+
type: 'source',
|
|
485
|
+
sourceId: sid ?? '*',
|
|
486
|
+
status: 'aborted',
|
|
487
|
+
detail: `cancelledQueued=${cancelledQueued}`,
|
|
488
|
+
});
|
|
489
|
+
return { cancelledQueued, abortedRunning };
|
|
490
|
+
}
|
|
491
|
+
/**
|
|
492
|
+
* 恢复/继续索引(相对「重建」更轻)
|
|
493
|
+
*
|
|
494
|
+
* - 清除该源中止态
|
|
495
|
+
* - 把 cancelled 任务放回 queued
|
|
496
|
+
* - 若仍缺向量则排队 embed
|
|
497
|
+
* - 不重新 walk 全量;需要全量对齐请 reindex
|
|
498
|
+
*
|
|
499
|
+
* @param opts.sourceId - 仅恢复该源;省略则全部源
|
|
500
|
+
* @returns 恢复统计
|
|
501
|
+
*/
|
|
502
|
+
resumeJobs(opts) {
|
|
503
|
+
const sid = opts?.sourceId?.trim();
|
|
504
|
+
const ids = sid
|
|
505
|
+
? [sid]
|
|
506
|
+
: [...new Set([...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)])];
|
|
507
|
+
let restoredCancelled = 0;
|
|
508
|
+
let embedQueued = 0;
|
|
509
|
+
for (const id of ids) {
|
|
510
|
+
// 1) 清除中止纪元
|
|
511
|
+
this.beginAbortEpoch(id);
|
|
512
|
+
// 2) cancelled → queued(用户中止时被杀的活)
|
|
513
|
+
const res = this.sources.database.raw
|
|
514
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
|
|
515
|
+
WHERE source_id = ? AND status = 'cancelled'`)
|
|
516
|
+
.run(Date.now(), id);
|
|
517
|
+
restoredCancelled += Number(res.changes ?? 0);
|
|
518
|
+
// 3) 仍缺向量则确保 embed 在队
|
|
519
|
+
if (this.embedding) {
|
|
520
|
+
const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
|
|
521
|
+
const active = this.countActiveJobs(id);
|
|
522
|
+
if (missing && active.embed === 0) {
|
|
523
|
+
this.ensureEmbedJob(id);
|
|
524
|
+
embedQueued += 1;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
this.kick();
|
|
529
|
+
this.emitProgress({
|
|
530
|
+
type: 'source',
|
|
531
|
+
sourceId: sid ?? '*',
|
|
532
|
+
status: 'resumed',
|
|
533
|
+
detail: `restoredCancelled=${restoredCancelled} embedQueued=${embedQueued}`,
|
|
534
|
+
});
|
|
535
|
+
return { restoredCancelled, embedQueued };
|
|
536
|
+
}
|
|
537
|
+
/** 开启新的中止纪元(abort 后新任务不受旧信号影响) */
|
|
538
|
+
beginAbortEpoch(sourceId) {
|
|
539
|
+
this.abortBySource.set(sourceId, new AbortController());
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* 重建前作废本源未完成任务(supersede)
|
|
543
|
+
*
|
|
544
|
+
* - queued → cancelled(superseded_by_reindex)
|
|
545
|
+
* - running 打断(worker/循环退出)
|
|
546
|
+
* - 随后 beginAbortEpoch,本轮重建可继续写
|
|
547
|
+
*/
|
|
548
|
+
supersedeSourceWork(sourceId) {
|
|
549
|
+
this.sources.database.raw
|
|
550
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
|
|
551
|
+
WHERE source_id = ? AND status = 'queued'`)
|
|
552
|
+
.run(Date.now(), sourceId);
|
|
553
|
+
// 打断在跑 parse/embed;再开新纪元,避免 walk 自己被 abort 卡死
|
|
554
|
+
let ctrl = this.abortBySource.get(sourceId);
|
|
555
|
+
if (!ctrl) {
|
|
556
|
+
ctrl = new AbortController();
|
|
557
|
+
this.abortBySource.set(sourceId, ctrl);
|
|
558
|
+
}
|
|
559
|
+
if (!ctrl.signal.aborted) {
|
|
560
|
+
ctrl.abort();
|
|
561
|
+
}
|
|
562
|
+
this.beginAbortEpoch(sourceId);
|
|
563
|
+
this.emitProgress({
|
|
564
|
+
type: 'source',
|
|
565
|
+
sourceId,
|
|
566
|
+
status: 'superseded',
|
|
567
|
+
detail: 'reindex will resync from disk',
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
async ensureParseCoverage(sourceId) {
|
|
571
|
+
const source = this.sources.get(sourceId);
|
|
572
|
+
if (!source || source.status === 'removed' || source.status === 'disabled')
|
|
573
|
+
return 0;
|
|
574
|
+
if (source.kind !== 'directory' && source.kind !== 'workspace')
|
|
575
|
+
return 0;
|
|
576
|
+
if (this.isAborted(sourceId))
|
|
577
|
+
return 0;
|
|
578
|
+
// 已有 parse 积压则不重复扫
|
|
579
|
+
if (this.countQueuedKinds(['parse_file', 'walk_source']) > 0)
|
|
580
|
+
return 0;
|
|
581
|
+
let found;
|
|
582
|
+
try {
|
|
583
|
+
found = await this.localFetcher.discover(source);
|
|
584
|
+
}
|
|
585
|
+
catch {
|
|
586
|
+
return 0;
|
|
587
|
+
}
|
|
588
|
+
const indexed = new Set(this.index.listFiles(sourceId).map((f) => f.path));
|
|
589
|
+
let added = 0;
|
|
590
|
+
for (const f of found) {
|
|
591
|
+
if (!indexed.has(f.path)) {
|
|
592
|
+
this.enqueue(sourceId, 'parse_file', f.path, 2, f.path);
|
|
593
|
+
added += 1;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (added > 0) {
|
|
597
|
+
this.emitProgress({
|
|
598
|
+
type: 'source',
|
|
599
|
+
sourceId,
|
|
600
|
+
status: 'parse_gap_found',
|
|
601
|
+
detail: `missing_files=${added}`,
|
|
602
|
+
});
|
|
603
|
+
this.kick();
|
|
604
|
+
}
|
|
605
|
+
return added;
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
|
|
609
|
+
*
|
|
610
|
+
* 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
|
|
611
|
+
* embed 前调用,避免给已移出文件的残 chunks 上向量。
|
|
612
|
+
*
|
|
613
|
+
* @param sourceId - 知识源 id
|
|
614
|
+
* @returns 删除的文件数
|
|
615
|
+
*/
|
|
616
|
+
async pruneMissingOnDisk(sourceId) {
|
|
617
|
+
const source = this.sources.get(sourceId);
|
|
618
|
+
if (!source || source.kind === 'url' || source.kind === 'connector')
|
|
619
|
+
return 0;
|
|
620
|
+
let removed = 0;
|
|
621
|
+
if (source.kind === 'directory' || source.kind === 'workspace') {
|
|
622
|
+
try {
|
|
623
|
+
const found = await this.localFetcher.discover(source);
|
|
624
|
+
const keep = new Set(found.map((f) => f.path));
|
|
625
|
+
// walk 成功:即使空目录也可清空(与逐文件 stat 语义一致)
|
|
626
|
+
removed = this.index.pruneMissing(sourceId, keep, { allowEmptyKeep: true });
|
|
627
|
+
}
|
|
628
|
+
catch {
|
|
629
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
else {
|
|
633
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
634
|
+
}
|
|
635
|
+
if (removed > 0) {
|
|
636
|
+
this.emitProgress({
|
|
637
|
+
type: 'source',
|
|
638
|
+
sourceId,
|
|
639
|
+
status: 'pruned_missing',
|
|
640
|
+
detail: `removed_files=${removed}`,
|
|
641
|
+
});
|
|
642
|
+
this.refreshCoverage(sourceId);
|
|
643
|
+
}
|
|
644
|
+
return removed;
|
|
645
|
+
}
|
|
646
|
+
/** 逐文件 stat(file 源 / walk 失败回退);并行限流 */
|
|
647
|
+
async pruneByStatScan(sourceId, concurrency = 32) {
|
|
648
|
+
const files = this.index.listFiles(sourceId);
|
|
649
|
+
let removed = 0;
|
|
650
|
+
let i = 0;
|
|
651
|
+
const worker = async () => {
|
|
652
|
+
for (;;) {
|
|
653
|
+
if (this.isAborted(sourceId))
|
|
654
|
+
return;
|
|
655
|
+
const idx = i++;
|
|
656
|
+
if (idx >= files.length)
|
|
657
|
+
return;
|
|
658
|
+
const f = files[idx];
|
|
659
|
+
let exists = true;
|
|
660
|
+
try {
|
|
661
|
+
const st = await stat(f.path);
|
|
662
|
+
exists = st.isFile();
|
|
663
|
+
}
|
|
664
|
+
catch {
|
|
665
|
+
exists = false;
|
|
666
|
+
}
|
|
667
|
+
if (!exists) {
|
|
668
|
+
this.index.removePathTree(sourceId, f.path);
|
|
669
|
+
removed += 1;
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
};
|
|
673
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, files.length || 1) }, () => worker()));
|
|
674
|
+
return removed;
|
|
675
|
+
}
|
|
676
|
+
abortSignalFor(sourceId) {
|
|
677
|
+
let ac = this.abortBySource.get(sourceId);
|
|
678
|
+
if (!ac) {
|
|
679
|
+
ac = new AbortController();
|
|
680
|
+
this.abortBySource.set(sourceId, ac);
|
|
681
|
+
}
|
|
682
|
+
return ac.signal;
|
|
683
|
+
}
|
|
684
|
+
isAborted(sourceId) {
|
|
685
|
+
return this.abortSignalFor(sourceId).aborted;
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
|
|
689
|
+
*
|
|
690
|
+
* 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
|
|
691
|
+
* 1. 回收超时 running(崩溃/挂死孤儿)
|
|
692
|
+
* 2. 有缺向量且无 active embed → 自动排队
|
|
693
|
+
* 3. 无队列、无缺口 → 刷 coverage/status
|
|
694
|
+
*/
|
|
695
|
+
startReconciler(intervalMs = 15_000) {
|
|
696
|
+
if (this.reconcileTimer || this.disposed)
|
|
697
|
+
return;
|
|
698
|
+
this.reconcileTimer = setInterval(() => {
|
|
699
|
+
void this.reconcileJobs().catch((err) => {
|
|
700
|
+
this.emitProgress({
|
|
701
|
+
type: 'error',
|
|
702
|
+
sourceId: '*',
|
|
703
|
+
status: 'reconcile_failed',
|
|
704
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
705
|
+
});
|
|
706
|
+
});
|
|
707
|
+
}, intervalMs);
|
|
708
|
+
this.reconcileTimer.unref?.();
|
|
709
|
+
}
|
|
710
|
+
stopReconciler() {
|
|
711
|
+
if (this.reconcileTimer) {
|
|
712
|
+
clearInterval(this.reconcileTimer);
|
|
713
|
+
this.reconcileTimer = null;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
/** 对账入口(也可手工调用) */
|
|
717
|
+
async reconcileJobs() {
|
|
718
|
+
if (this.disposed)
|
|
719
|
+
return;
|
|
720
|
+
this.reclaimStaleRunningJobs(5 * 60_000);
|
|
721
|
+
if (!this.embedding) {
|
|
722
|
+
for (const s of this.sources.list()) {
|
|
723
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
724
|
+
continue;
|
|
725
|
+
this.refreshCoverage(s.id);
|
|
726
|
+
}
|
|
727
|
+
return;
|
|
728
|
+
}
|
|
729
|
+
for (const s of this.sources.list()) {
|
|
730
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
731
|
+
continue;
|
|
732
|
+
if (this.isAborted(s.id))
|
|
733
|
+
continue;
|
|
734
|
+
// watch 漏事件时补齐 parse:磁盘有、索引无 → 入队(解析优先于 embed)
|
|
735
|
+
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
736
|
+
if (Date.now() - lastGap > 60_000) {
|
|
737
|
+
this.lastParseGapScanAt.set(s.id, Date.now());
|
|
738
|
+
await this.ensureParseCoverage(s.id);
|
|
739
|
+
}
|
|
740
|
+
const missing = this.index.listChunksMissingEmbedding(s.id, 1).length > 0;
|
|
741
|
+
const active = this.countActiveJobs(s.id);
|
|
742
|
+
if (missing && active.embed === 0) {
|
|
743
|
+
this.ensureEmbedJob(s.id);
|
|
744
|
+
}
|
|
745
|
+
else if (!missing && active.total === 0) {
|
|
746
|
+
this.refreshCoverage(s.id);
|
|
747
|
+
// 稳定态收尾(auto-describe 等);回调不得抛出打断对账
|
|
748
|
+
try {
|
|
749
|
+
this.onSourceSettled?.(s.id);
|
|
750
|
+
}
|
|
751
|
+
catch {
|
|
752
|
+
/* ignore settle hook errors */
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
this.kick();
|
|
757
|
+
}
|
|
758
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
|
|
759
|
+
reclaimStaleRunningJobs(staleMs) {
|
|
760
|
+
const cutoff = Date.now() - staleMs;
|
|
761
|
+
// 先放弃重试过多的
|
|
762
|
+
this.sources.database.raw
|
|
763
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
|
|
764
|
+
WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
|
|
765
|
+
.run(Date.now(), cutoff);
|
|
766
|
+
this.sources.database.raw
|
|
767
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
|
|
768
|
+
WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
|
|
769
|
+
.run(Date.now(), cutoff);
|
|
770
|
+
}
|
|
771
|
+
countActiveJobs(sourceId) {
|
|
772
|
+
const row = this.sources.database.raw
|
|
773
|
+
.prepare(`SELECT
|
|
774
|
+
COUNT(*) AS total,
|
|
775
|
+
SUM(CASE WHEN kind = 'embed_source' THEN 1 ELSE 0 END) AS embed
|
|
776
|
+
FROM knowledge_jobs
|
|
777
|
+
WHERE source_id = ? AND status IN ('queued', 'running')`)
|
|
778
|
+
.get(sourceId);
|
|
779
|
+
return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
|
|
780
|
+
}
|
|
405
781
|
/**
|
|
406
782
|
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
407
783
|
*/
|
|
@@ -480,6 +856,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
480
856
|
return (row?.n ?? 0) > 0;
|
|
481
857
|
}
|
|
482
858
|
enqueue(sourceId, kind, path, priority, _detail) {
|
|
859
|
+
if (this.isAborted(sourceId))
|
|
860
|
+
return;
|
|
483
861
|
// 同 source+kind 去重(queued + running,避免并发重复 embed)
|
|
484
862
|
const activeStatuses = ['queued', 'running'];
|
|
485
863
|
if (path != null) {
|
|
@@ -521,30 +899,57 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
521
899
|
.run(`kj_${randomUUID().slice(0, 12)}`, sourceId, kind, path, priority, now, now);
|
|
522
900
|
}
|
|
523
901
|
kick() {
|
|
524
|
-
if (this.disposed
|
|
902
|
+
if (this.disposed)
|
|
903
|
+
return;
|
|
904
|
+
if (this.draining) {
|
|
905
|
+
// 等待中的 drain 会再 kick;标记一次避免丢信号
|
|
906
|
+
this.kickAgain = true;
|
|
525
907
|
return;
|
|
908
|
+
}
|
|
526
909
|
this.draining = true;
|
|
527
910
|
queueMicrotask(() => {
|
|
528
911
|
this.draining = false;
|
|
529
912
|
this.drain();
|
|
913
|
+
if (this.kickAgain) {
|
|
914
|
+
this.kickAgain = false;
|
|
915
|
+
this.kick();
|
|
916
|
+
}
|
|
530
917
|
});
|
|
531
918
|
}
|
|
532
919
|
drain() {
|
|
533
|
-
|
|
534
|
-
const
|
|
535
|
-
|
|
920
|
+
for (;;) {
|
|
921
|
+
const allowParse = this.runningParse < this.parseConcurrency;
|
|
922
|
+
const allowEmbed = this.runningEmbed < this.embedConcurrency;
|
|
923
|
+
if (!allowParse && !allowEmbed)
|
|
536
924
|
return;
|
|
537
|
-
const
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
925
|
+
const parsePending = this.countQueuedKinds([
|
|
926
|
+
'parse_file',
|
|
927
|
+
'walk_source',
|
|
928
|
+
'drop_file',
|
|
929
|
+
'fetch_doc',
|
|
930
|
+
]);
|
|
931
|
+
const kinds = [];
|
|
932
|
+
if (allowParse)
|
|
933
|
+
kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
|
|
934
|
+
// 业务优先级:解析/分块做完即可关键词搜;embedding 仅在无 parse 积压时占槽
|
|
935
|
+
if (allowEmbed && parsePending === 0)
|
|
936
|
+
kinds.push('embed_source');
|
|
937
|
+
if (kinds.length === 0) {
|
|
938
|
+
if (!allowParse && allowEmbed && parsePending === 0)
|
|
939
|
+
return;
|
|
940
|
+
// parse 槽满但仍有 parse 积压 → 等 parse,不抢 embed
|
|
941
|
+
if (parsePending > 0 && !allowParse)
|
|
942
|
+
return;
|
|
543
943
|
return;
|
|
544
944
|
}
|
|
545
|
-
this.
|
|
945
|
+
const job = this.claimJob(kinds);
|
|
946
|
+
if (!job)
|
|
947
|
+
return;
|
|
948
|
+
const isEmbed = job.kind === 'embed_source';
|
|
546
949
|
if (isEmbed)
|
|
547
950
|
this.runningEmbed += 1;
|
|
951
|
+
else
|
|
952
|
+
this.runningParse += 1;
|
|
548
953
|
void this.runJob(job)
|
|
549
954
|
.catch((err) => {
|
|
550
955
|
this.emitProgress({
|
|
@@ -556,20 +961,33 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
556
961
|
});
|
|
557
962
|
})
|
|
558
963
|
.finally(() => {
|
|
559
|
-
this.running -= 1;
|
|
560
964
|
if (isEmbed)
|
|
561
965
|
this.runningEmbed -= 1;
|
|
966
|
+
else
|
|
967
|
+
this.runningParse -= 1;
|
|
562
968
|
this.kick();
|
|
563
969
|
});
|
|
564
970
|
}
|
|
565
971
|
}
|
|
566
|
-
|
|
972
|
+
countQueuedKinds(kinds) {
|
|
973
|
+
if (kinds.length === 0)
|
|
974
|
+
return 0;
|
|
975
|
+
const placeholders = kinds.map(() => '?').join(',');
|
|
976
|
+
const row = this.sources.database.raw
|
|
977
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`)
|
|
978
|
+
.get(...kinds);
|
|
979
|
+
return row?.n ?? 0;
|
|
980
|
+
}
|
|
981
|
+
claimJob(kinds) {
|
|
982
|
+
if (kinds.length === 0)
|
|
983
|
+
return null;
|
|
984
|
+
const placeholders = kinds.map(() => '?').join(',');
|
|
567
985
|
const row = this.sources.database.raw
|
|
568
986
|
.prepare(`SELECT * FROM knowledge_jobs
|
|
569
|
-
WHERE status = 'queued'
|
|
987
|
+
WHERE status = 'queued' AND kind IN (${placeholders})
|
|
570
988
|
ORDER BY priority ASC, created_at ASC
|
|
571
989
|
LIMIT 1`)
|
|
572
|
-
.get();
|
|
990
|
+
.get(...kinds);
|
|
573
991
|
if (!row)
|
|
574
992
|
return null;
|
|
575
993
|
const res = this.sources.database.raw
|
|
@@ -582,9 +1000,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
582
1000
|
}
|
|
583
1001
|
async runJob(job) {
|
|
584
1002
|
const now = Date.now();
|
|
1003
|
+
if (this.isAborted(job.source_id)) {
|
|
1004
|
+
this.sources.database.raw
|
|
1005
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
1006
|
+
WHERE id = ? AND status = 'running'`)
|
|
1007
|
+
.run(Date.now(), job.id);
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
585
1010
|
try {
|
|
586
1011
|
if (job.kind === 'parse_file' && job.path) {
|
|
587
|
-
|
|
1012
|
+
// 单文件解析硬超时:大 xlsx/pdf 同步抽取会堵死事件循环
|
|
1013
|
+
await this.withTimeout(this.parseOne(job.source_id, job.path), this.parseTimeoutMs, `parse_timeout:${job.path}`);
|
|
1014
|
+
if (this.isAborted(job.source_id)) {
|
|
1015
|
+
throw new Error('aborted');
|
|
1016
|
+
}
|
|
588
1017
|
this.refreshCoverage(job.source_id);
|
|
589
1018
|
if (this.embedding) {
|
|
590
1019
|
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
@@ -592,7 +1021,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
592
1021
|
}
|
|
593
1022
|
}
|
|
594
1023
|
else if (job.kind === 'fetch_doc' && job.path) {
|
|
595
|
-
await this.fetchDocJob(job.source_id, job.path);
|
|
1024
|
+
await this.withTimeout(this.fetchDocJob(job.source_id, job.path), this.parseTimeoutMs, `fetch_timeout:${job.path}`);
|
|
596
1025
|
this.refreshCoverage(job.source_id);
|
|
597
1026
|
if (this.embedding) {
|
|
598
1027
|
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
@@ -600,14 +1029,23 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
600
1029
|
}
|
|
601
1030
|
}
|
|
602
1031
|
else if (job.kind === 'drop_file' && job.path) {
|
|
603
|
-
|
|
1032
|
+
// 目录移出/删除:连同子路径一并清(否则只剩空目录节点)
|
|
1033
|
+
this.index.removePathTree(job.source_id, job.path);
|
|
604
1034
|
this.refreshCoverage(job.source_id);
|
|
605
1035
|
}
|
|
606
1036
|
else if (job.kind === 'walk_source') {
|
|
607
1037
|
await this.ingestSource(job.source_id);
|
|
608
1038
|
}
|
|
609
1039
|
else if (job.kind === 'embed_source') {
|
|
610
|
-
|
|
1040
|
+
if (!this.embedding) {
|
|
1041
|
+
const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
|
|
1042
|
+
if (stillMissing) {
|
|
1043
|
+
// 禁止空跑标 done:迁移后缺向量却显示 queue=0
|
|
1044
|
+
throw new Error('embedding_provider_missing');
|
|
1045
|
+
}
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
await this.embedPending(job.source_id, 50, job.id);
|
|
611
1049
|
}
|
|
612
1050
|
this.sources.database.raw
|
|
613
1051
|
.prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
|
|
@@ -618,17 +1056,44 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
618
1056
|
path: job.path ?? undefined,
|
|
619
1057
|
status: 'done',
|
|
620
1058
|
});
|
|
1059
|
+
// 标 done 后再续跑:否则 enqueue 去重会把「自己」当成 active 而丢掉下一棒
|
|
1060
|
+
if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
|
|
1061
|
+
this.ensureEmbedJob(job.source_id);
|
|
1062
|
+
this.refreshCoverage(job.source_id);
|
|
1063
|
+
}
|
|
621
1064
|
}
|
|
622
1065
|
catch (err) {
|
|
623
1066
|
const message = err instanceof Error ? err.message : String(err);
|
|
1067
|
+
const aborted = message === 'aborted' || this.isAborted(job.source_id);
|
|
1068
|
+
if (job.kind === 'parse_file' && job.path && !aborted) {
|
|
1069
|
+
this.index.markFileError(job.source_id, job.path, message);
|
|
1070
|
+
}
|
|
624
1071
|
this.sources.database.raw
|
|
625
1072
|
.prepare(`UPDATE knowledge_jobs
|
|
626
|
-
SET status =
|
|
1073
|
+
SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
|
|
627
1074
|
WHERE id = ?`)
|
|
628
|
-
.run(message, now, job.id);
|
|
1075
|
+
.run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, now, job.id);
|
|
1076
|
+
if (job.kind === 'embed_source' && !aborted) {
|
|
1077
|
+
this.ensureEmbedJob(job.source_id);
|
|
1078
|
+
}
|
|
629
1079
|
throw err;
|
|
630
1080
|
}
|
|
631
1081
|
}
|
|
1082
|
+
async withTimeout(p, ms, label) {
|
|
1083
|
+
let timer;
|
|
1084
|
+
try {
|
|
1085
|
+
return await Promise.race([
|
|
1086
|
+
p,
|
|
1087
|
+
new Promise((_, reject) => {
|
|
1088
|
+
timer = setTimeout(() => reject(new Error(label)), ms);
|
|
1089
|
+
}),
|
|
1090
|
+
]);
|
|
1091
|
+
}
|
|
1092
|
+
finally {
|
|
1093
|
+
if (timer)
|
|
1094
|
+
clearTimeout(timer);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
632
1097
|
/**
|
|
633
1098
|
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
634
1099
|
*/
|
|
@@ -654,12 +1119,15 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
654
1119
|
this.indexVirtualDoc(sourceId, doc);
|
|
655
1120
|
}
|
|
656
1121
|
async parseOne(sourceId, filePath) {
|
|
657
|
-
if (this.
|
|
1122
|
+
if (this.shouldSkipFile(filePath)) {
|
|
658
1123
|
this.index.markFileSkipped(sourceId, filePath, 'ignored_path');
|
|
659
1124
|
return false;
|
|
660
1125
|
}
|
|
661
1126
|
const adapter = this.adapters.match(filePath);
|
|
662
1127
|
if (!adapter) {
|
|
1128
|
+
if (this.documentPort && isDocumentPath(filePath)) {
|
|
1129
|
+
return this.parseDocumentFile(sourceId, filePath);
|
|
1130
|
+
}
|
|
663
1131
|
this.index.markFileSkipped(sourceId, filePath, 'no_adapter');
|
|
664
1132
|
return false;
|
|
665
1133
|
}
|
|
@@ -701,50 +1169,184 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
701
1169
|
}
|
|
702
1170
|
}
|
|
703
1171
|
/**
|
|
704
|
-
*
|
|
1172
|
+
* PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
|
|
1173
|
+
* 新鲜度以原件字节 hash 为准。
|
|
1174
|
+
*
|
|
1175
|
+
* @param sourceId - 知识源 id
|
|
1176
|
+
* @param filePath - 本地文件路径
|
|
1177
|
+
* @returns 是否成功入索引
|
|
705
1178
|
*/
|
|
706
|
-
async
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
1179
|
+
async parseDocumentFile(sourceId, filePath) {
|
|
1180
|
+
const port = this.documentPort;
|
|
1181
|
+
if (!port)
|
|
1182
|
+
return false;
|
|
1183
|
+
let st;
|
|
1184
|
+
try {
|
|
1185
|
+
st = await stat(filePath);
|
|
1186
|
+
}
|
|
1187
|
+
catch {
|
|
1188
|
+
this.index.removeFile(sourceId, filePath);
|
|
1189
|
+
return false;
|
|
1190
|
+
}
|
|
1191
|
+
if (!st.isFile())
|
|
1192
|
+
return false;
|
|
1193
|
+
if (st.size > this.maxFileBytes) {
|
|
1194
|
+
this.index.markFileSkipped(sourceId, filePath, 'oversize');
|
|
1195
|
+
return false;
|
|
1196
|
+
}
|
|
1197
|
+
const raw = await readFile(filePath);
|
|
1198
|
+
const contentHash = hashContent(raw);
|
|
1199
|
+
if (this.index.isFresh(sourceId, filePath, contentHash)) {
|
|
1200
|
+
return true;
|
|
716
1201
|
}
|
|
717
1202
|
try {
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
1203
|
+
// worker 抽取:同步 xlsx/pdf 不堵主循环;超时/中止可 terminate
|
|
1204
|
+
const result = await this.extractDocument(filePath, sourceId);
|
|
1205
|
+
const markdown = result.markdown?.trim();
|
|
1206
|
+
if (!markdown) {
|
|
1207
|
+
this.index.markFileSkipped(sourceId, filePath, 'empty_content');
|
|
1208
|
+
return false;
|
|
723
1209
|
}
|
|
724
|
-
|
|
725
|
-
this.
|
|
726
|
-
type: 'embed',
|
|
1210
|
+
const chunks = markdownAdapter.chunk(markdown, filePath);
|
|
1211
|
+
this.index.upsertFile({
|
|
727
1212
|
sourceId,
|
|
728
|
-
|
|
729
|
-
|
|
1213
|
+
path: filePath,
|
|
1214
|
+
contentHash,
|
|
1215
|
+
size: st.size,
|
|
1216
|
+
mtime: Math.floor(st.mtimeMs),
|
|
1217
|
+
adapterId: `document:${result.backend}`,
|
|
1218
|
+
chunks,
|
|
730
1219
|
});
|
|
1220
|
+
return true;
|
|
731
1221
|
}
|
|
732
1222
|
catch (err) {
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
sourceId,
|
|
736
|
-
|
|
737
|
-
|
|
1223
|
+
if (isDocumentExtractError(err)) {
|
|
1224
|
+
// 结构化跳过(密码/老格式/超大等),不挡整库
|
|
1225
|
+
this.index.markFileSkipped(sourceId, filePath, err.code.toLowerCase());
|
|
1226
|
+
return false;
|
|
1227
|
+
}
|
|
1228
|
+
this.index.markFileError(sourceId, filePath, err instanceof Error ? err.message : String(err));
|
|
1229
|
+
return false;
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
/** 优先 worker 抽取;worker 不可用时退回进程内 port */
|
|
1233
|
+
async extractDocument(filePath, sourceId) {
|
|
1234
|
+
const signal = this.abortSignalFor(sourceId);
|
|
1235
|
+
try {
|
|
1236
|
+
const { extractDocumentInWorker } = await import('./extract-document.js');
|
|
1237
|
+
return await extractDocumentInWorker(filePath, {
|
|
1238
|
+
timeoutMs: this.parseTimeoutMs,
|
|
1239
|
+
maxFileBytes: this.maxFileBytes,
|
|
1240
|
+
signal,
|
|
738
1241
|
});
|
|
1242
|
+
}
|
|
1243
|
+
catch (err) {
|
|
1244
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1245
|
+
if (msg === 'extract_aborted' || signal.aborted) {
|
|
1246
|
+
throw new Error('aborted');
|
|
1247
|
+
}
|
|
1248
|
+
if (msg.includes('Cannot find module') || msg.includes('ERR_MODULE')) {
|
|
1249
|
+
return this.documentPort.extract({ path: filePath, name: filePath }, { timeoutMs: this.parseTimeoutMs });
|
|
1250
|
+
}
|
|
1251
|
+
throw err;
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Phase B:嵌入待处理 chunk
|
|
1256
|
+
*
|
|
1257
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
1258
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
1259
|
+
*/
|
|
1260
|
+
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
1261
|
+
if (!this.embedding) {
|
|
1262
|
+
if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
|
|
1263
|
+
throw new Error('embedding_provider_missing');
|
|
1264
|
+
}
|
|
739
1265
|
return;
|
|
740
1266
|
}
|
|
741
|
-
//
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
1267
|
+
// 磁盘对齐在 reindex/walk 做;embed 任务内不再全量 prune(曾拖慢解析)
|
|
1268
|
+
const lanes = Math.max(1, this.embedConcurrency);
|
|
1269
|
+
const batchSize = Math.max(1, this.embedBatch);
|
|
1270
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
1271
|
+
if (this.isAborted(sourceId))
|
|
1272
|
+
return;
|
|
1273
|
+
const pending = this.index.listChunksMissingEmbedding(sourceId, batchSize * lanes);
|
|
1274
|
+
if (pending.length === 0) {
|
|
1275
|
+
this.refreshCoverage(sourceId);
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
if (jobId)
|
|
1279
|
+
this.heartbeatJob(jobId);
|
|
1280
|
+
// 轮次限速(并行片共享同一闸门)
|
|
1281
|
+
if (this.embedMinIntervalMs > 0) {
|
|
1282
|
+
const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
|
|
1283
|
+
if (wait > 0)
|
|
1284
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
1285
|
+
}
|
|
1286
|
+
const slices = [];
|
|
1287
|
+
for (let i = 0; i < pending.length; i += batchSize) {
|
|
1288
|
+
slices.push(pending.slice(i, i + batchSize));
|
|
1289
|
+
}
|
|
1290
|
+
const settled = await Promise.allSettled(slices.map((slice) => this.embedOneSlice(sourceId, slice)));
|
|
1291
|
+
let ok = 0;
|
|
1292
|
+
let failed = false;
|
|
1293
|
+
for (const s of settled) {
|
|
1294
|
+
if (s.status === 'fulfilled')
|
|
1295
|
+
ok += s.value;
|
|
1296
|
+
else
|
|
1297
|
+
failed = true;
|
|
1298
|
+
}
|
|
1299
|
+
this.lastEmbedAt = Date.now();
|
|
1300
|
+
this.emitProgress({
|
|
1301
|
+
type: 'embed',
|
|
1302
|
+
sourceId,
|
|
1303
|
+
status: 'batch_done',
|
|
1304
|
+
detail: `embedded=${ok}/${pending.length} lanes=${slices.length}`,
|
|
1305
|
+
});
|
|
1306
|
+
if (failed) {
|
|
1307
|
+
// 至少一片失败:交给 done 后的 ensureEmbedJob 续跑,不把整 job 打成 failed
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
745
1310
|
}
|
|
746
1311
|
this.refreshCoverage(sourceId);
|
|
747
1312
|
}
|
|
1313
|
+
/** 单片 embedding;返回成功写入条数 */
|
|
1314
|
+
async embedOneSlice(sourceId, slice) {
|
|
1315
|
+
if (slice.length === 0)
|
|
1316
|
+
return 0;
|
|
1317
|
+
const vectors = await Promise.race([
|
|
1318
|
+
this.embedding.embedBatch(slice.map((p) => p.text)),
|
|
1319
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('embed_batch_timeout')), 120_000)),
|
|
1320
|
+
]);
|
|
1321
|
+
let ok = 0;
|
|
1322
|
+
const batch = [];
|
|
1323
|
+
for (let i = 0; i < slice.length; i++) {
|
|
1324
|
+
const vec = vectors[i];
|
|
1325
|
+
if (vec?.length) {
|
|
1326
|
+
batch.push([slice[i].id, vec]);
|
|
1327
|
+
ok += 1;
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
if (batch.length > 0)
|
|
1331
|
+
this.index.setChunkEmbeddings(batch);
|
|
1332
|
+
return ok;
|
|
1333
|
+
}
|
|
1334
|
+
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
1335
|
+
ensureEmbedJob(sourceId) {
|
|
1336
|
+
if (!this.embedding)
|
|
1337
|
+
return;
|
|
1338
|
+
if (this.isAborted(sourceId))
|
|
1339
|
+
return;
|
|
1340
|
+
if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
|
|
1341
|
+
return;
|
|
1342
|
+
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
1343
|
+
this.kick();
|
|
1344
|
+
}
|
|
1345
|
+
heartbeatJob(jobId) {
|
|
1346
|
+
this.sources.database.raw
|
|
1347
|
+
.prepare(`UPDATE knowledge_jobs SET updated_at = ? WHERE id = ? AND status = 'running'`)
|
|
1348
|
+
.run(Date.now(), jobId);
|
|
1349
|
+
}
|
|
748
1350
|
refreshCoverage(sourceId) {
|
|
749
1351
|
const stats = this.index.sourceStats(sourceId);
|
|
750
1352
|
const source = this.sources.get(sourceId);
|