@octopi-agent/engine 0.56.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/engine.d.ts +5 -0
- package/dist/config-schema/substrate.d.ts +5 -0
- package/dist/config-schema/substrate.js +11 -0
- package/dist/config.d.ts +9 -0
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +1 -1
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/db.d.ts +4 -0
- package/dist/harness/knowledge/db.js +58 -3
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +25 -0
- package/dist/harness/knowledge/extract-document.d.ts +17 -0
- package/dist/harness/knowledge/extract-document.js +62 -0
- package/dist/harness/knowledge/index-store.d.ts +42 -4
- package/dist/harness/knowledge/index-store.js +234 -32
- package/dist/harness/knowledge/index.d.ts +2 -0
- package/dist/harness/knowledge/index.js +1 -0
- package/dist/harness/knowledge/ingest.d.ts +100 -2
- package/dist/harness/knowledge/ingest.js +582 -58
- package/dist/harness/knowledge/retriever.d.ts +8 -0
- package/dist/harness/knowledge/retriever.js +33 -1
- package/dist/harness/knowledge/source-store.d.ts +4 -0
- package/dist/harness/knowledge/source-store.js +62 -6
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
- package/dist/harness/memory/sqlite/keyword-search.js +40 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/package.json +2 -2
|
@@ -24,6 +24,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
24
24
|
maxQueueDepth;
|
|
25
25
|
debounceMs;
|
|
26
26
|
maxFileBytes;
|
|
27
|
+
/** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
|
|
28
|
+
parseTimeoutMs;
|
|
27
29
|
embedding;
|
|
28
30
|
embedBatch;
|
|
29
31
|
embedMinIntervalMs;
|
|
@@ -39,9 +41,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
39
41
|
documentPort;
|
|
40
42
|
lastEmbedAt = 0;
|
|
41
43
|
pollTimer = null;
|
|
42
|
-
|
|
44
|
+
reconcileTimer = null;
|
|
45
|
+
/** 源级中止控制器(手动 abort / 重建前清场) */
|
|
46
|
+
abortBySource = new Map();
|
|
47
|
+
/** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
|
|
48
|
+
onSourceSettled;
|
|
49
|
+
/** 上次 parse 缺口扫描时间 */
|
|
50
|
+
lastParseGapScanAt = new Map();
|
|
51
|
+
runningParse = 0;
|
|
43
52
|
runningEmbed = 0;
|
|
44
53
|
draining = false;
|
|
54
|
+
kickAgain = false;
|
|
45
55
|
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
46
56
|
sourceLocks = new Set();
|
|
47
57
|
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
@@ -54,14 +64,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
54
64
|
this.sources = options.sourceStore;
|
|
55
65
|
this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
|
|
56
66
|
this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
|
|
57
|
-
this.parseConcurrency = options.parseConcurrency ??
|
|
67
|
+
this.parseConcurrency = options.parseConcurrency ?? 8;
|
|
58
68
|
this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
|
|
59
|
-
|
|
69
|
+
// watch 批量落盘时缩短静默窗,新增文件更快入队
|
|
70
|
+
this.debounceMs = options.debounceMs ?? 800;
|
|
60
71
|
this.maxFileBytes = options.maxFileBytes ?? 5_000_000;
|
|
72
|
+
this.parseTimeoutMs = options.parseTimeoutMs ?? 45_000;
|
|
61
73
|
this.embedding = options.embeddingProvider ?? null;
|
|
62
74
|
this.embedBatch = options.embedBatch ?? 32;
|
|
63
75
|
this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
|
|
64
|
-
|
|
76
|
+
// embed 与 parse 分槽;缺省 4 路并行打 embedding API
|
|
77
|
+
this.embedConcurrency = options.embedConcurrency ?? 4;
|
|
65
78
|
this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
|
|
66
79
|
this.credentials = options.credentials ?? null;
|
|
67
80
|
this.documentPort = options.documentPort ?? null;
|
|
@@ -74,6 +87,17 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
74
87
|
this.pollTickMs = options.pollTickMs ?? 60_000;
|
|
75
88
|
this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
|
|
76
89
|
this.maxPollPerTick = options.maxPollPerTick ?? 8;
|
|
90
|
+
// 崩溃/重启遗留的 running 会堵住 enqueue 去重并永久显示 indexing
|
|
91
|
+
this.reclaimOrphanRunningJobs();
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
|
|
95
|
+
* 仅构造时调用——运行中同进程的 running 是合法的。
|
|
96
|
+
*/
|
|
97
|
+
reclaimOrphanRunningJobs() {
|
|
98
|
+
this.sources.database.raw
|
|
99
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ? WHERE status = 'running'`)
|
|
100
|
+
.run(Date.now());
|
|
77
101
|
}
|
|
78
102
|
get indexStore() {
|
|
79
103
|
return this.index;
|
|
@@ -93,6 +117,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
93
117
|
* 全量/增量索引某源(Phase A)
|
|
94
118
|
*/
|
|
95
119
|
async ingestSource(sourceId, opts) {
|
|
120
|
+
// 重建 = supersede:以本次磁盘扫描为准,作废本源未完成任务
|
|
121
|
+
this.supersedeSourceWork(sourceId);
|
|
96
122
|
const source = this.sources.get(sourceId);
|
|
97
123
|
if (!source || source.status === 'removed') {
|
|
98
124
|
throw new Error(`knowledge source not found: ${sourceId}`);
|
|
@@ -403,7 +429,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
403
429
|
*/
|
|
404
430
|
async idle(timeoutMs = 30_000) {
|
|
405
431
|
const start = Date.now();
|
|
406
|
-
while (this.
|
|
432
|
+
while (this.runningParse > 0 || this.runningEmbed > 0 || this.hasQueuedJobs()) {
|
|
407
433
|
if (Date.now() - start > timeoutMs) {
|
|
408
434
|
throw new Error('knowledge ingest idle timeout');
|
|
409
435
|
}
|
|
@@ -414,9 +440,344 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
414
440
|
dispose() {
|
|
415
441
|
this.disposed = true;
|
|
416
442
|
this.stopPolling();
|
|
443
|
+
this.stopReconciler();
|
|
444
|
+
void this.abortJobs();
|
|
417
445
|
for (const id of this.watchers.keys())
|
|
418
446
|
this.stopWatch(id);
|
|
419
447
|
}
|
|
448
|
+
/**
|
|
449
|
+
* 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
|
|
450
|
+
*
|
|
451
|
+
* @param opts.sourceId - 仅中止该源;省略则全部源
|
|
452
|
+
* @returns 取消统计
|
|
453
|
+
*/
|
|
454
|
+
abortJobs(opts) {
|
|
455
|
+
const sid = opts?.sourceId?.trim();
|
|
456
|
+
const ids = sid ? [sid] : [...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)];
|
|
457
|
+
const unique = [...new Set(ids)];
|
|
458
|
+
let cancelledQueued = 0;
|
|
459
|
+
let abortedRunning = 0;
|
|
460
|
+
// 1) 清空 queued → cancelled
|
|
461
|
+
for (const id of unique) {
|
|
462
|
+
const res = this.sources.database.raw
|
|
463
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
464
|
+
WHERE source_id = ? AND status = 'queued'`)
|
|
465
|
+
.run(Date.now(), id);
|
|
466
|
+
cancelledQueued += Number(res.changes ?? 0);
|
|
467
|
+
// 2) 打断在跑(worker terminate / embed 循环退出);保持 aborted 直到下次 reindex
|
|
468
|
+
let ctrl = this.abortBySource.get(id);
|
|
469
|
+
if (!ctrl) {
|
|
470
|
+
ctrl = new AbortController();
|
|
471
|
+
this.abortBySource.set(id, ctrl);
|
|
472
|
+
}
|
|
473
|
+
if (!ctrl.signal.aborted) {
|
|
474
|
+
ctrl.abort();
|
|
475
|
+
abortedRunning += 1;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
// 3) 看门狗不要再把 aborted running 收回 queued
|
|
479
|
+
this.sources.database.raw
|
|
480
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
481
|
+
WHERE status = 'running' AND last_error = 'aborted'`)
|
|
482
|
+
.run(Date.now());
|
|
483
|
+
this.emitProgress({
|
|
484
|
+
type: 'source',
|
|
485
|
+
sourceId: sid ?? '*',
|
|
486
|
+
status: 'aborted',
|
|
487
|
+
detail: `cancelledQueued=${cancelledQueued}`,
|
|
488
|
+
});
|
|
489
|
+
return { cancelledQueued, abortedRunning };
|
|
490
|
+
}
|
|
491
|
+
/**
|
|
492
|
+
* 恢复/继续索引(相对「重建」更轻)
|
|
493
|
+
*
|
|
494
|
+
* - 清除该源中止态
|
|
495
|
+
* - 把 cancelled 任务放回 queued
|
|
496
|
+
* - 若仍缺向量则排队 embed
|
|
497
|
+
* - 不重新 walk 全量;需要全量对齐请 reindex
|
|
498
|
+
*
|
|
499
|
+
* @param opts.sourceId - 仅恢复该源;省略则全部源
|
|
500
|
+
* @returns 恢复统计
|
|
501
|
+
*/
|
|
502
|
+
resumeJobs(opts) {
|
|
503
|
+
const sid = opts?.sourceId?.trim();
|
|
504
|
+
const ids = sid
|
|
505
|
+
? [sid]
|
|
506
|
+
: [...new Set([...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)])];
|
|
507
|
+
let restoredCancelled = 0;
|
|
508
|
+
let embedQueued = 0;
|
|
509
|
+
for (const id of ids) {
|
|
510
|
+
// 1) 清除中止纪元
|
|
511
|
+
this.beginAbortEpoch(id);
|
|
512
|
+
// 2) cancelled → queued(用户中止时被杀的活)
|
|
513
|
+
const res = this.sources.database.raw
|
|
514
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
|
|
515
|
+
WHERE source_id = ? AND status = 'cancelled'`)
|
|
516
|
+
.run(Date.now(), id);
|
|
517
|
+
restoredCancelled += Number(res.changes ?? 0);
|
|
518
|
+
// 3) 仍缺向量则确保 embed 在队
|
|
519
|
+
if (this.embedding) {
|
|
520
|
+
const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
|
|
521
|
+
const active = this.countActiveJobs(id);
|
|
522
|
+
if (missing && active.embed === 0) {
|
|
523
|
+
this.ensureEmbedJob(id);
|
|
524
|
+
embedQueued += 1;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
this.kick();
|
|
529
|
+
this.emitProgress({
|
|
530
|
+
type: 'source',
|
|
531
|
+
sourceId: sid ?? '*',
|
|
532
|
+
status: 'resumed',
|
|
533
|
+
detail: `restoredCancelled=${restoredCancelled} embedQueued=${embedQueued}`,
|
|
534
|
+
});
|
|
535
|
+
return { restoredCancelled, embedQueued };
|
|
536
|
+
}
|
|
537
|
+
/** 开启新的中止纪元(abort 后新任务不受旧信号影响) */
|
|
538
|
+
beginAbortEpoch(sourceId) {
|
|
539
|
+
this.abortBySource.set(sourceId, new AbortController());
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* 重建前作废本源未完成任务(supersede)
|
|
543
|
+
*
|
|
544
|
+
* - queued → cancelled(superseded_by_reindex)
|
|
545
|
+
* - running 打断(worker/循环退出)
|
|
546
|
+
* - 随后 beginAbortEpoch,本轮重建可继续写
|
|
547
|
+
*/
|
|
548
|
+
supersedeSourceWork(sourceId) {
|
|
549
|
+
this.sources.database.raw
|
|
550
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
|
|
551
|
+
WHERE source_id = ? AND status = 'queued'`)
|
|
552
|
+
.run(Date.now(), sourceId);
|
|
553
|
+
// 打断在跑 parse/embed;再开新纪元,避免 walk 自己被 abort 卡死
|
|
554
|
+
let ctrl = this.abortBySource.get(sourceId);
|
|
555
|
+
if (!ctrl) {
|
|
556
|
+
ctrl = new AbortController();
|
|
557
|
+
this.abortBySource.set(sourceId, ctrl);
|
|
558
|
+
}
|
|
559
|
+
if (!ctrl.signal.aborted) {
|
|
560
|
+
ctrl.abort();
|
|
561
|
+
}
|
|
562
|
+
this.beginAbortEpoch(sourceId);
|
|
563
|
+
this.emitProgress({
|
|
564
|
+
type: 'source',
|
|
565
|
+
sourceId,
|
|
566
|
+
status: 'superseded',
|
|
567
|
+
detail: 'reindex will resync from disk',
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
async ensureParseCoverage(sourceId) {
|
|
571
|
+
const source = this.sources.get(sourceId);
|
|
572
|
+
if (!source || source.status === 'removed' || source.status === 'disabled')
|
|
573
|
+
return 0;
|
|
574
|
+
if (source.kind !== 'directory' && source.kind !== 'workspace')
|
|
575
|
+
return 0;
|
|
576
|
+
if (this.isAborted(sourceId))
|
|
577
|
+
return 0;
|
|
578
|
+
// 已有 parse 积压则不重复扫
|
|
579
|
+
if (this.countQueuedKinds(['parse_file', 'walk_source']) > 0)
|
|
580
|
+
return 0;
|
|
581
|
+
let found;
|
|
582
|
+
try {
|
|
583
|
+
found = await this.localFetcher.discover(source);
|
|
584
|
+
}
|
|
585
|
+
catch {
|
|
586
|
+
return 0;
|
|
587
|
+
}
|
|
588
|
+
const indexed = new Set(this.index.listFiles(sourceId).map((f) => f.path));
|
|
589
|
+
let added = 0;
|
|
590
|
+
for (const f of found) {
|
|
591
|
+
if (!indexed.has(f.path)) {
|
|
592
|
+
this.enqueue(sourceId, 'parse_file', f.path, 2, f.path);
|
|
593
|
+
added += 1;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (added > 0) {
|
|
597
|
+
this.emitProgress({
|
|
598
|
+
type: 'source',
|
|
599
|
+
sourceId,
|
|
600
|
+
status: 'parse_gap_found',
|
|
601
|
+
detail: `missing_files=${added}`,
|
|
602
|
+
});
|
|
603
|
+
this.kick();
|
|
604
|
+
}
|
|
605
|
+
return added;
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
|
|
609
|
+
*
|
|
610
|
+
* 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
|
|
611
|
+
* embed 前调用,避免给已移出文件的残 chunks 上向量。
|
|
612
|
+
*
|
|
613
|
+
* @param sourceId - 知识源 id
|
|
614
|
+
* @returns 删除的文件数
|
|
615
|
+
*/
|
|
616
|
+
async pruneMissingOnDisk(sourceId) {
|
|
617
|
+
const source = this.sources.get(sourceId);
|
|
618
|
+
if (!source || source.kind === 'url' || source.kind === 'connector')
|
|
619
|
+
return 0;
|
|
620
|
+
let removed = 0;
|
|
621
|
+
if (source.kind === 'directory' || source.kind === 'workspace') {
|
|
622
|
+
try {
|
|
623
|
+
const found = await this.localFetcher.discover(source);
|
|
624
|
+
const keep = new Set(found.map((f) => f.path));
|
|
625
|
+
// walk 成功:即使空目录也可清空(与逐文件 stat 语义一致)
|
|
626
|
+
removed = this.index.pruneMissing(sourceId, keep, { allowEmptyKeep: true });
|
|
627
|
+
}
|
|
628
|
+
catch {
|
|
629
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
else {
|
|
633
|
+
removed = await this.pruneByStatScan(sourceId);
|
|
634
|
+
}
|
|
635
|
+
if (removed > 0) {
|
|
636
|
+
this.emitProgress({
|
|
637
|
+
type: 'source',
|
|
638
|
+
sourceId,
|
|
639
|
+
status: 'pruned_missing',
|
|
640
|
+
detail: `removed_files=${removed}`,
|
|
641
|
+
});
|
|
642
|
+
this.refreshCoverage(sourceId);
|
|
643
|
+
}
|
|
644
|
+
return removed;
|
|
645
|
+
}
|
|
646
|
+
/** 逐文件 stat(file 源 / walk 失败回退);并行限流 */
|
|
647
|
+
async pruneByStatScan(sourceId, concurrency = 32) {
|
|
648
|
+
const files = this.index.listFiles(sourceId);
|
|
649
|
+
let removed = 0;
|
|
650
|
+
let i = 0;
|
|
651
|
+
const worker = async () => {
|
|
652
|
+
for (;;) {
|
|
653
|
+
if (this.isAborted(sourceId))
|
|
654
|
+
return;
|
|
655
|
+
const idx = i++;
|
|
656
|
+
if (idx >= files.length)
|
|
657
|
+
return;
|
|
658
|
+
const f = files[idx];
|
|
659
|
+
let exists = true;
|
|
660
|
+
try {
|
|
661
|
+
const st = await stat(f.path);
|
|
662
|
+
exists = st.isFile();
|
|
663
|
+
}
|
|
664
|
+
catch {
|
|
665
|
+
exists = false;
|
|
666
|
+
}
|
|
667
|
+
if (!exists) {
|
|
668
|
+
this.index.removePathTree(sourceId, f.path);
|
|
669
|
+
removed += 1;
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
};
|
|
673
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, files.length || 1) }, () => worker()));
|
|
674
|
+
return removed;
|
|
675
|
+
}
|
|
676
|
+
abortSignalFor(sourceId) {
|
|
677
|
+
let ac = this.abortBySource.get(sourceId);
|
|
678
|
+
if (!ac) {
|
|
679
|
+
ac = new AbortController();
|
|
680
|
+
this.abortBySource.set(sourceId, ac);
|
|
681
|
+
}
|
|
682
|
+
return ac.signal;
|
|
683
|
+
}
|
|
684
|
+
isAborted(sourceId) {
|
|
685
|
+
return this.abortSignalFor(sourceId).aborted;
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
|
|
689
|
+
*
|
|
690
|
+
* 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
|
|
691
|
+
* 1. 回收超时 running(崩溃/挂死孤儿)
|
|
692
|
+
* 2. 有缺向量且无 active embed → 自动排队
|
|
693
|
+
* 3. 无队列、无缺口 → 刷 coverage/status
|
|
694
|
+
*/
|
|
695
|
+
startReconciler(intervalMs = 15_000) {
|
|
696
|
+
if (this.reconcileTimer || this.disposed)
|
|
697
|
+
return;
|
|
698
|
+
this.reconcileTimer = setInterval(() => {
|
|
699
|
+
void this.reconcileJobs().catch((err) => {
|
|
700
|
+
this.emitProgress({
|
|
701
|
+
type: 'error',
|
|
702
|
+
sourceId: '*',
|
|
703
|
+
status: 'reconcile_failed',
|
|
704
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
705
|
+
});
|
|
706
|
+
});
|
|
707
|
+
}, intervalMs);
|
|
708
|
+
this.reconcileTimer.unref?.();
|
|
709
|
+
}
|
|
710
|
+
stopReconciler() {
|
|
711
|
+
if (this.reconcileTimer) {
|
|
712
|
+
clearInterval(this.reconcileTimer);
|
|
713
|
+
this.reconcileTimer = null;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
/** 对账入口(也可手工调用) */
|
|
717
|
+
async reconcileJobs() {
|
|
718
|
+
if (this.disposed)
|
|
719
|
+
return;
|
|
720
|
+
this.reclaimStaleRunningJobs(5 * 60_000);
|
|
721
|
+
if (!this.embedding) {
|
|
722
|
+
for (const s of this.sources.list()) {
|
|
723
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
724
|
+
continue;
|
|
725
|
+
this.refreshCoverage(s.id);
|
|
726
|
+
}
|
|
727
|
+
return;
|
|
728
|
+
}
|
|
729
|
+
for (const s of this.sources.list()) {
|
|
730
|
+
if (s.status === 'removed' || s.status === 'disabled')
|
|
731
|
+
continue;
|
|
732
|
+
if (this.isAborted(s.id))
|
|
733
|
+
continue;
|
|
734
|
+
// watch 漏事件时补齐 parse:磁盘有、索引无 → 入队(解析优先于 embed)
|
|
735
|
+
const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
|
|
736
|
+
if (Date.now() - lastGap > 60_000) {
|
|
737
|
+
this.lastParseGapScanAt.set(s.id, Date.now());
|
|
738
|
+
await this.ensureParseCoverage(s.id);
|
|
739
|
+
}
|
|
740
|
+
const missing = this.index.listChunksMissingEmbedding(s.id, 1).length > 0;
|
|
741
|
+
const active = this.countActiveJobs(s.id);
|
|
742
|
+
if (missing && active.embed === 0) {
|
|
743
|
+
this.ensureEmbedJob(s.id);
|
|
744
|
+
}
|
|
745
|
+
else if (!missing && active.total === 0) {
|
|
746
|
+
this.refreshCoverage(s.id);
|
|
747
|
+
// 稳定态收尾(auto-describe 等);回调不得抛出打断对账
|
|
748
|
+
try {
|
|
749
|
+
this.onSourceSettled?.(s.id);
|
|
750
|
+
}
|
|
751
|
+
catch {
|
|
752
|
+
/* ignore settle hook errors */
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
this.kick();
|
|
757
|
+
}
|
|
758
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
|
|
759
|
+
reclaimStaleRunningJobs(staleMs) {
|
|
760
|
+
const cutoff = Date.now() - staleMs;
|
|
761
|
+
// 先放弃重试过多的
|
|
762
|
+
this.sources.database.raw
|
|
763
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
|
|
764
|
+
WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
|
|
765
|
+
.run(Date.now(), cutoff);
|
|
766
|
+
this.sources.database.raw
|
|
767
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
|
|
768
|
+
WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
|
|
769
|
+
.run(Date.now(), cutoff);
|
|
770
|
+
}
|
|
771
|
+
countActiveJobs(sourceId) {
|
|
772
|
+
const row = this.sources.database.raw
|
|
773
|
+
.prepare(`SELECT
|
|
774
|
+
COUNT(*) AS total,
|
|
775
|
+
SUM(CASE WHEN kind = 'embed_source' THEN 1 ELSE 0 END) AS embed
|
|
776
|
+
FROM knowledge_jobs
|
|
777
|
+
WHERE source_id = ? AND status IN ('queued', 'running')`)
|
|
778
|
+
.get(sourceId);
|
|
779
|
+
return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
|
|
780
|
+
}
|
|
420
781
|
/**
|
|
421
782
|
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
422
783
|
*/
|
|
@@ -495,6 +856,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
495
856
|
return (row?.n ?? 0) > 0;
|
|
496
857
|
}
|
|
497
858
|
enqueue(sourceId, kind, path, priority, _detail) {
|
|
859
|
+
if (this.isAborted(sourceId))
|
|
860
|
+
return;
|
|
498
861
|
// 同 source+kind 去重(queued + running,避免并发重复 embed)
|
|
499
862
|
const activeStatuses = ['queued', 'running'];
|
|
500
863
|
if (path != null) {
|
|
@@ -536,30 +899,57 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
536
899
|
.run(`kj_${randomUUID().slice(0, 12)}`, sourceId, kind, path, priority, now, now);
|
|
537
900
|
}
|
|
538
901
|
kick() {
|
|
539
|
-
if (this.disposed
|
|
902
|
+
if (this.disposed)
|
|
540
903
|
return;
|
|
904
|
+
if (this.draining) {
|
|
905
|
+
// 等待中的 drain 会再 kick;标记一次避免丢信号
|
|
906
|
+
this.kickAgain = true;
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
541
909
|
this.draining = true;
|
|
542
910
|
queueMicrotask(() => {
|
|
543
911
|
this.draining = false;
|
|
544
912
|
this.drain();
|
|
913
|
+
if (this.kickAgain) {
|
|
914
|
+
this.kickAgain = false;
|
|
915
|
+
this.kick();
|
|
916
|
+
}
|
|
545
917
|
});
|
|
546
918
|
}
|
|
547
919
|
drain() {
|
|
548
|
-
|
|
549
|
-
const
|
|
550
|
-
|
|
920
|
+
for (;;) {
|
|
921
|
+
const allowParse = this.runningParse < this.parseConcurrency;
|
|
922
|
+
const allowEmbed = this.runningEmbed < this.embedConcurrency;
|
|
923
|
+
if (!allowParse && !allowEmbed)
|
|
551
924
|
return;
|
|
552
|
-
const
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
925
|
+
const parsePending = this.countQueuedKinds([
|
|
926
|
+
'parse_file',
|
|
927
|
+
'walk_source',
|
|
928
|
+
'drop_file',
|
|
929
|
+
'fetch_doc',
|
|
930
|
+
]);
|
|
931
|
+
const kinds = [];
|
|
932
|
+
if (allowParse)
|
|
933
|
+
kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
|
|
934
|
+
// 业务优先级:解析/分块做完即可关键词搜;embedding 仅在无 parse 积压时占槽
|
|
935
|
+
if (allowEmbed && parsePending === 0)
|
|
936
|
+
kinds.push('embed_source');
|
|
937
|
+
if (kinds.length === 0) {
|
|
938
|
+
if (!allowParse && allowEmbed && parsePending === 0)
|
|
939
|
+
return;
|
|
940
|
+
// parse 槽满但仍有 parse 积压 → 等 parse,不抢 embed
|
|
941
|
+
if (parsePending > 0 && !allowParse)
|
|
942
|
+
return;
|
|
558
943
|
return;
|
|
559
944
|
}
|
|
560
|
-
this.
|
|
945
|
+
const job = this.claimJob(kinds);
|
|
946
|
+
if (!job)
|
|
947
|
+
return;
|
|
948
|
+
const isEmbed = job.kind === 'embed_source';
|
|
561
949
|
if (isEmbed)
|
|
562
950
|
this.runningEmbed += 1;
|
|
951
|
+
else
|
|
952
|
+
this.runningParse += 1;
|
|
563
953
|
void this.runJob(job)
|
|
564
954
|
.catch((err) => {
|
|
565
955
|
this.emitProgress({
|
|
@@ -571,20 +961,33 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
571
961
|
});
|
|
572
962
|
})
|
|
573
963
|
.finally(() => {
|
|
574
|
-
this.running -= 1;
|
|
575
964
|
if (isEmbed)
|
|
576
965
|
this.runningEmbed -= 1;
|
|
966
|
+
else
|
|
967
|
+
this.runningParse -= 1;
|
|
577
968
|
this.kick();
|
|
578
969
|
});
|
|
579
970
|
}
|
|
580
971
|
}
|
|
581
|
-
|
|
972
|
+
countQueuedKinds(kinds) {
|
|
973
|
+
if (kinds.length === 0)
|
|
974
|
+
return 0;
|
|
975
|
+
const placeholders = kinds.map(() => '?').join(',');
|
|
976
|
+
const row = this.sources.database.raw
|
|
977
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`)
|
|
978
|
+
.get(...kinds);
|
|
979
|
+
return row?.n ?? 0;
|
|
980
|
+
}
|
|
981
|
+
claimJob(kinds) {
|
|
982
|
+
if (kinds.length === 0)
|
|
983
|
+
return null;
|
|
984
|
+
const placeholders = kinds.map(() => '?').join(',');
|
|
582
985
|
const row = this.sources.database.raw
|
|
583
986
|
.prepare(`SELECT * FROM knowledge_jobs
|
|
584
|
-
WHERE status = 'queued'
|
|
987
|
+
WHERE status = 'queued' AND kind IN (${placeholders})
|
|
585
988
|
ORDER BY priority ASC, created_at ASC
|
|
586
989
|
LIMIT 1`)
|
|
587
|
-
.get();
|
|
990
|
+
.get(...kinds);
|
|
588
991
|
if (!row)
|
|
589
992
|
return null;
|
|
590
993
|
const res = this.sources.database.raw
|
|
@@ -597,9 +1000,20 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
597
1000
|
}
|
|
598
1001
|
async runJob(job) {
|
|
599
1002
|
const now = Date.now();
|
|
1003
|
+
if (this.isAborted(job.source_id)) {
|
|
1004
|
+
this.sources.database.raw
|
|
1005
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
|
|
1006
|
+
WHERE id = ? AND status = 'running'`)
|
|
1007
|
+
.run(Date.now(), job.id);
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
600
1010
|
try {
|
|
601
1011
|
if (job.kind === 'parse_file' && job.path) {
|
|
602
|
-
|
|
1012
|
+
// 单文件解析硬超时:大 xlsx/pdf 同步抽取会堵死事件循环
|
|
1013
|
+
await this.withTimeout(this.parseOne(job.source_id, job.path), this.parseTimeoutMs, `parse_timeout:${job.path}`);
|
|
1014
|
+
if (this.isAborted(job.source_id)) {
|
|
1015
|
+
throw new Error('aborted');
|
|
1016
|
+
}
|
|
603
1017
|
this.refreshCoverage(job.source_id);
|
|
604
1018
|
if (this.embedding) {
|
|
605
1019
|
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
@@ -607,7 +1021,7 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
607
1021
|
}
|
|
608
1022
|
}
|
|
609
1023
|
else if (job.kind === 'fetch_doc' && job.path) {
|
|
610
|
-
await this.fetchDocJob(job.source_id, job.path);
|
|
1024
|
+
await this.withTimeout(this.fetchDocJob(job.source_id, job.path), this.parseTimeoutMs, `fetch_timeout:${job.path}`);
|
|
611
1025
|
this.refreshCoverage(job.source_id);
|
|
612
1026
|
if (this.embedding) {
|
|
613
1027
|
this.enqueue(job.source_id, 'embed_source', null, 3);
|
|
@@ -615,14 +1029,23 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
615
1029
|
}
|
|
616
1030
|
}
|
|
617
1031
|
else if (job.kind === 'drop_file' && job.path) {
|
|
618
|
-
|
|
1032
|
+
// 目录移出/删除:连同子路径一并清(否则只剩空目录节点)
|
|
1033
|
+
this.index.removePathTree(job.source_id, job.path);
|
|
619
1034
|
this.refreshCoverage(job.source_id);
|
|
620
1035
|
}
|
|
621
1036
|
else if (job.kind === 'walk_source') {
|
|
622
1037
|
await this.ingestSource(job.source_id);
|
|
623
1038
|
}
|
|
624
1039
|
else if (job.kind === 'embed_source') {
|
|
625
|
-
|
|
1040
|
+
if (!this.embedding) {
|
|
1041
|
+
const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
|
|
1042
|
+
if (stillMissing) {
|
|
1043
|
+
// 禁止空跑标 done:迁移后缺向量却显示 queue=0
|
|
1044
|
+
throw new Error('embedding_provider_missing');
|
|
1045
|
+
}
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
await this.embedPending(job.source_id, 50, job.id);
|
|
626
1049
|
}
|
|
627
1050
|
this.sources.database.raw
|
|
628
1051
|
.prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
|
|
@@ -633,17 +1056,44 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
633
1056
|
path: job.path ?? undefined,
|
|
634
1057
|
status: 'done',
|
|
635
1058
|
});
|
|
1059
|
+
// 标 done 后再续跑:否则 enqueue 去重会把「自己」当成 active 而丢掉下一棒
|
|
1060
|
+
if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
|
|
1061
|
+
this.ensureEmbedJob(job.source_id);
|
|
1062
|
+
this.refreshCoverage(job.source_id);
|
|
1063
|
+
}
|
|
636
1064
|
}
|
|
637
1065
|
catch (err) {
|
|
638
1066
|
const message = err instanceof Error ? err.message : String(err);
|
|
1067
|
+
const aborted = message === 'aborted' || this.isAborted(job.source_id);
|
|
1068
|
+
if (job.kind === 'parse_file' && job.path && !aborted) {
|
|
1069
|
+
this.index.markFileError(job.source_id, job.path, message);
|
|
1070
|
+
}
|
|
639
1071
|
this.sources.database.raw
|
|
640
1072
|
.prepare(`UPDATE knowledge_jobs
|
|
641
|
-
SET status =
|
|
1073
|
+
SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
|
|
642
1074
|
WHERE id = ?`)
|
|
643
|
-
.run(message, now, job.id);
|
|
1075
|
+
.run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, now, job.id);
|
|
1076
|
+
if (job.kind === 'embed_source' && !aborted) {
|
|
1077
|
+
this.ensureEmbedJob(job.source_id);
|
|
1078
|
+
}
|
|
644
1079
|
throw err;
|
|
645
1080
|
}
|
|
646
1081
|
}
|
|
1082
|
+
async withTimeout(p, ms, label) {
|
|
1083
|
+
let timer;
|
|
1084
|
+
try {
|
|
1085
|
+
return await Promise.race([
|
|
1086
|
+
p,
|
|
1087
|
+
new Promise((_, reject) => {
|
|
1088
|
+
timer = setTimeout(() => reject(new Error(label)), ms);
|
|
1089
|
+
}),
|
|
1090
|
+
]);
|
|
1091
|
+
}
|
|
1092
|
+
finally {
|
|
1093
|
+
if (timer)
|
|
1094
|
+
clearTimeout(timer);
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
647
1097
|
/**
|
|
648
1098
|
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
649
1099
|
*/
|
|
@@ -750,7 +1200,8 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
750
1200
|
return true;
|
|
751
1201
|
}
|
|
752
1202
|
try {
|
|
753
|
-
|
|
1203
|
+
// worker 抽取:同步 xlsx/pdf 不堵主循环;超时/中止可 terminate
|
|
1204
|
+
const result = await this.extractDocument(filePath, sourceId);
|
|
754
1205
|
const markdown = result.markdown?.trim();
|
|
755
1206
|
if (!markdown) {
|
|
756
1207
|
this.index.markFileSkipped(sourceId, filePath, 'empty_content');
|
|
@@ -778,51 +1229,124 @@ export class KnowledgeIngest extends EventEmitter {
|
|
|
778
1229
|
return false;
|
|
779
1230
|
}
|
|
780
1231
|
}
|
|
1232
|
+
/** 优先 worker 抽取;worker 不可用时退回进程内 port */
|
|
1233
|
+
async extractDocument(filePath, sourceId) {
|
|
1234
|
+
const signal = this.abortSignalFor(sourceId);
|
|
1235
|
+
try {
|
|
1236
|
+
const { extractDocumentInWorker } = await import('./extract-document.js');
|
|
1237
|
+
return await extractDocumentInWorker(filePath, {
|
|
1238
|
+
timeoutMs: this.parseTimeoutMs,
|
|
1239
|
+
maxFileBytes: this.maxFileBytes,
|
|
1240
|
+
signal,
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
catch (err) {
|
|
1244
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1245
|
+
if (msg === 'extract_aborted' || signal.aborted) {
|
|
1246
|
+
throw new Error('aborted');
|
|
1247
|
+
}
|
|
1248
|
+
if (msg.includes('Cannot find module') || msg.includes('ERR_MODULE')) {
|
|
1249
|
+
return this.documentPort.extract({ path: filePath, name: filePath }, { timeoutMs: this.parseTimeoutMs });
|
|
1250
|
+
}
|
|
1251
|
+
throw err;
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
781
1254
|
/**
|
|
782
|
-
* Phase B:嵌入待处理 chunk
|
|
1255
|
+
* Phase B:嵌入待处理 chunk
|
|
1256
|
+
*
|
|
1257
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
1258
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
783
1259
|
*/
|
|
784
|
-
async embedPending(sourceId) {
|
|
785
|
-
if (!this.embedding)
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
1260
|
+
async embedPending(sourceId, maxRounds = 20, jobId) {
|
|
1261
|
+
if (!this.embedding) {
|
|
1262
|
+
if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
|
|
1263
|
+
throw new Error('embedding_provider_missing');
|
|
1264
|
+
}
|
|
789
1265
|
return;
|
|
790
|
-
if (this.embedMinIntervalMs > 0) {
|
|
791
|
-
const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
|
|
792
|
-
if (wait > 0)
|
|
793
|
-
await new Promise((r) => setTimeout(r, wait));
|
|
794
1266
|
}
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1267
|
+
// 磁盘对齐在 reindex/walk 做;embed 任务内不再全量 prune(曾拖慢解析)
|
|
1268
|
+
const lanes = Math.max(1, this.embedConcurrency);
|
|
1269
|
+
const batchSize = Math.max(1, this.embedBatch);
|
|
1270
|
+
for (let round = 0; round < maxRounds; round++) {
|
|
1271
|
+
if (this.isAborted(sourceId))
|
|
1272
|
+
return;
|
|
1273
|
+
const pending = this.index.listChunksMissingEmbedding(sourceId, batchSize * lanes);
|
|
1274
|
+
if (pending.length === 0) {
|
|
1275
|
+
this.refreshCoverage(sourceId);
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
if (jobId)
|
|
1279
|
+
this.heartbeatJob(jobId);
|
|
1280
|
+
// 轮次限速(并行片共享同一闸门)
|
|
1281
|
+
if (this.embedMinIntervalMs > 0) {
|
|
1282
|
+
const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
|
|
1283
|
+
if (wait > 0)
|
|
1284
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
1285
|
+
}
|
|
1286
|
+
const slices = [];
|
|
1287
|
+
for (let i = 0; i < pending.length; i += batchSize) {
|
|
1288
|
+
slices.push(pending.slice(i, i + batchSize));
|
|
1289
|
+
}
|
|
1290
|
+
const settled = await Promise.allSettled(slices.map((slice) => this.embedOneSlice(sourceId, slice)));
|
|
1291
|
+
let ok = 0;
|
|
1292
|
+
let failed = false;
|
|
1293
|
+
for (const s of settled) {
|
|
1294
|
+
if (s.status === 'fulfilled')
|
|
1295
|
+
ok += s.value;
|
|
1296
|
+
else
|
|
1297
|
+
failed = true;
|
|
801
1298
|
}
|
|
802
1299
|
this.lastEmbedAt = Date.now();
|
|
803
1300
|
this.emitProgress({
|
|
804
1301
|
type: 'embed',
|
|
805
1302
|
sourceId,
|
|
806
1303
|
status: 'batch_done',
|
|
807
|
-
detail: `embedded=${pending.length}`,
|
|
808
|
-
});
|
|
809
|
-
}
|
|
810
|
-
catch (err) {
|
|
811
|
-
this.emitProgress({
|
|
812
|
-
type: 'error',
|
|
813
|
-
sourceId,
|
|
814
|
-
status: 'embed_failed',
|
|
815
|
-
detail: err instanceof Error ? err.message : String(err),
|
|
1304
|
+
detail: `embedded=${ok}/${pending.length} lanes=${slices.length}`,
|
|
816
1305
|
});
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
822
|
-
this.kick();
|
|
1306
|
+
if (failed) {
|
|
1307
|
+
// 至少一片失败:交给 done 后的 ensureEmbedJob 续跑,不把整 job 打成 failed
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
823
1310
|
}
|
|
824
1311
|
this.refreshCoverage(sourceId);
|
|
825
1312
|
}
|
|
1313
|
+
/** 单片 embedding;返回成功写入条数 */
|
|
1314
|
+
async embedOneSlice(sourceId, slice) {
|
|
1315
|
+
if (slice.length === 0)
|
|
1316
|
+
return 0;
|
|
1317
|
+
const vectors = await Promise.race([
|
|
1318
|
+
this.embedding.embedBatch(slice.map((p) => p.text)),
|
|
1319
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('embed_batch_timeout')), 120_000)),
|
|
1320
|
+
]);
|
|
1321
|
+
let ok = 0;
|
|
1322
|
+
const batch = [];
|
|
1323
|
+
for (let i = 0; i < slice.length; i++) {
|
|
1324
|
+
const vec = vectors[i];
|
|
1325
|
+
if (vec?.length) {
|
|
1326
|
+
batch.push([slice[i].id, vec]);
|
|
1327
|
+
ok += 1;
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
if (batch.length > 0)
|
|
1331
|
+
this.index.setChunkEmbeddings(batch);
|
|
1332
|
+
return ok;
|
|
1333
|
+
}
|
|
1334
|
+
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
1335
|
+
ensureEmbedJob(sourceId) {
|
|
1336
|
+
if (!this.embedding)
|
|
1337
|
+
return;
|
|
1338
|
+
if (this.isAborted(sourceId))
|
|
1339
|
+
return;
|
|
1340
|
+
if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
|
|
1341
|
+
return;
|
|
1342
|
+
this.enqueue(sourceId, 'embed_source', null, 3);
|
|
1343
|
+
this.kick();
|
|
1344
|
+
}
|
|
1345
|
+
heartbeatJob(jobId) {
|
|
1346
|
+
this.sources.database.raw
|
|
1347
|
+
.prepare(`UPDATE knowledge_jobs SET updated_at = ? WHERE id = ? AND status = 'running'`)
|
|
1348
|
+
.run(Date.now(), jobId);
|
|
1349
|
+
}
|
|
826
1350
|
refreshCoverage(sourceId) {
|
|
827
1351
|
const stats = this.index.sourceStats(sourceId);
|
|
828
1352
|
const source = this.sources.get(sourceId);
|