@octopi-agent/engine 0.56.0 → 0.58.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/config-schema/engine.d.ts +5 -0
  2. package/dist/config-schema/substrate.d.ts +5 -0
  3. package/dist/config-schema/substrate.js +11 -0
  4. package/dist/config.d.ts +9 -0
  5. package/dist/harness/agent/config-bridge.js +3 -0
  6. package/dist/harness/context/constitution/default-agents.md +1 -0
  7. package/dist/harness/context/index.d.ts +2 -2
  8. package/dist/harness/context/index.js +1 -1
  9. package/dist/harness/context/layer-types.d.ts +26 -2
  10. package/dist/harness/context/layer-types.js +59 -11
  11. package/dist/harness/context/layers.d.ts +3 -0
  12. package/dist/harness/context/layers.js +38 -12
  13. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  14. package/dist/harness/context/system-prompt-assembler.js +2 -0
  15. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
  16. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  17. package/dist/harness/index.d.ts +1 -1
  18. package/dist/harness/index.js +1 -1
  19. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  20. package/dist/harness/knowledge/db.d.ts +4 -0
  21. package/dist/harness/knowledge/db.js +58 -3
  22. package/dist/harness/knowledge/describe.d.ts +2 -0
  23. package/dist/harness/knowledge/describe.js +5 -1
  24. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  25. package/dist/harness/knowledge/document-extract-worker.js +25 -0
  26. package/dist/harness/knowledge/extract-document.d.ts +17 -0
  27. package/dist/harness/knowledge/extract-document.js +62 -0
  28. package/dist/harness/knowledge/index-store.d.ts +42 -4
  29. package/dist/harness/knowledge/index-store.js +234 -32
  30. package/dist/harness/knowledge/index.d.ts +2 -0
  31. package/dist/harness/knowledge/index.js +1 -0
  32. package/dist/harness/knowledge/ingest.d.ts +100 -2
  33. package/dist/harness/knowledge/ingest.js +582 -58
  34. package/dist/harness/knowledge/retriever.d.ts +8 -0
  35. package/dist/harness/knowledge/retriever.js +33 -1
  36. package/dist/harness/knowledge/source-store.d.ts +4 -0
  37. package/dist/harness/knowledge/source-store.js +62 -6
  38. package/dist/harness/knowledge/topics.d.ts +13 -0
  39. package/dist/harness/knowledge/topics.js +48 -0
  40. package/dist/harness/memory/index.d.ts +3 -2
  41. package/dist/harness/memory/index.js +3 -1
  42. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  43. package/dist/harness/memory/retrieval-rank.js +60 -0
  44. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  45. package/dist/harness/memory/sqlite/index.js +1 -1
  46. package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
  47. package/dist/harness/memory/sqlite/keyword-search.js +40 -17
  48. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  49. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  50. package/dist/harness/memory/store.js +39 -21
  51. package/dist/harness/memory/types.d.ts +7 -0
  52. package/dist/harness/run/runner.d.ts +1 -0
  53. package/dist/harness/run/runner.js +4 -2
  54. package/dist/harness/session/tasks/index.d.ts +1 -1
  55. package/dist/harness/session/tasks/index.js +1 -1
  56. package/dist/harness/session/tasks/render.d.ts +10 -0
  57. package/dist/harness/session/tasks/render.js +27 -0
  58. package/package.json +2 -2
@@ -24,6 +24,8 @@ export class KnowledgeIngest extends EventEmitter {
24
24
  maxQueueDepth;
25
25
  debounceMs;
26
26
  maxFileBytes;
27
+ /** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
28
+ parseTimeoutMs;
27
29
  embedding;
28
30
  embedBatch;
29
31
  embedMinIntervalMs;
@@ -39,9 +41,17 @@ export class KnowledgeIngest extends EventEmitter {
39
41
  documentPort;
40
42
  lastEmbedAt = 0;
41
43
  pollTimer = null;
42
- running = 0;
44
+ reconcileTimer = null;
45
+ /** 源级中止控制器(手动 abort / 重建前清场) */
46
+ abortBySource = new Map();
47
+ /** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
48
+ onSourceSettled;
49
+ /** 上次 parse 缺口扫描时间 */
50
+ lastParseGapScanAt = new Map();
51
+ runningParse = 0;
43
52
  runningEmbed = 0;
44
53
  draining = false;
54
+ kickAgain = false;
45
55
  /** source 级锁:同一 source 不并行 walk/parse 冲突写 */
46
56
  sourceLocks = new Set();
47
57
  /** walk 发现的文件数(Phase A coverage 分母) */
@@ -54,14 +64,17 @@ export class KnowledgeIngest extends EventEmitter {
54
64
  this.sources = options.sourceStore;
55
65
  this.index = options.indexStore ?? new KnowledgeIndexStore(options.sourceStore.database);
56
66
  this.adapters = options.adapterRegistry ?? new FormatAdapterRegistry();
57
- this.parseConcurrency = options.parseConcurrency ?? 4;
67
+ this.parseConcurrency = options.parseConcurrency ?? 8;
58
68
  this.maxQueueDepth = options.maxQueueDepth ?? 10_000;
59
- this.debounceMs = options.debounceMs ?? 2000;
69
+ // watch 批量落盘时缩短静默窗,新增文件更快入队
70
+ this.debounceMs = options.debounceMs ?? 800;
60
71
  this.maxFileBytes = options.maxFileBytes ?? 5_000_000;
72
+ this.parseTimeoutMs = options.parseTimeoutMs ?? 45_000;
61
73
  this.embedding = options.embeddingProvider ?? null;
62
74
  this.embedBatch = options.embedBatch ?? 32;
63
75
  this.embedMinIntervalMs = options.embedMinIntervalMs ?? 0;
64
- this.embedConcurrency = options.embedConcurrency ?? options.parseConcurrency ?? 2;
76
+ // embed 与 parse 分槽;缺省 4 路并行打 embedding API
77
+ this.embedConcurrency = options.embedConcurrency ?? 4;
65
78
  this.diskWatermarkAlert = options.diskWatermarkAlert ?? false;
66
79
  this.credentials = options.credentials ?? null;
67
80
  this.documentPort = options.documentPort ?? null;
@@ -74,6 +87,17 @@ export class KnowledgeIngest extends EventEmitter {
74
87
  this.pollTickMs = options.pollTickMs ?? 60_000;
75
88
  this.pollMinIntervalMs = options.pollMinIntervalMs ?? 15 * 60_000;
76
89
  this.maxPollPerTick = options.maxPollPerTick ?? 8;
90
+ // 崩溃/重启遗留的 running 会堵住 enqueue 去重并永久显示 indexing
91
+ this.reclaimOrphanRunningJobs();
92
+ }
93
+ /**
94
+ * 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
95
+ * 仅构造时调用——运行中同进程的 running 是合法的。
96
+ */
97
+ reclaimOrphanRunningJobs() {
98
+ this.sources.database.raw
99
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ? WHERE status = 'running'`)
100
+ .run(Date.now());
77
101
  }
78
102
  get indexStore() {
79
103
  return this.index;
@@ -93,6 +117,8 @@ export class KnowledgeIngest extends EventEmitter {
93
117
  * 全量/增量索引某源(Phase A)
94
118
  */
95
119
  async ingestSource(sourceId, opts) {
120
+ // 重建 = supersede:以本次磁盘扫描为准,作废本源未完成任务
121
+ this.supersedeSourceWork(sourceId);
96
122
  const source = this.sources.get(sourceId);
97
123
  if (!source || source.status === 'removed') {
98
124
  throw new Error(`knowledge source not found: ${sourceId}`);
@@ -403,7 +429,7 @@ export class KnowledgeIngest extends EventEmitter {
403
429
  */
404
430
  async idle(timeoutMs = 30_000) {
405
431
  const start = Date.now();
406
- while (this.running > 0 || this.hasQueuedJobs()) {
432
+ while (this.runningParse > 0 || this.runningEmbed > 0 || this.hasQueuedJobs()) {
407
433
  if (Date.now() - start > timeoutMs) {
408
434
  throw new Error('knowledge ingest idle timeout');
409
435
  }
@@ -414,9 +440,344 @@ export class KnowledgeIngest extends EventEmitter {
414
440
  dispose() {
415
441
  this.disposed = true;
416
442
  this.stopPolling();
443
+ this.stopReconciler();
444
+ void this.abortJobs();
417
445
  for (const id of this.watchers.keys())
418
446
  this.stopWatch(id);
419
447
  }
448
+ /**
449
+ * 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
450
+ *
451
+ * @param opts.sourceId - 仅中止该源;省略则全部源
452
+ * @returns 取消统计
453
+ */
454
+ abortJobs(opts) {
455
+ const sid = opts?.sourceId?.trim();
456
+ const ids = sid ? [sid] : [...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)];
457
+ const unique = [...new Set(ids)];
458
+ let cancelledQueued = 0;
459
+ let abortedRunning = 0;
460
+ // 1) 清空 queued → cancelled
461
+ for (const id of unique) {
462
+ const res = this.sources.database.raw
463
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
464
+ WHERE source_id = ? AND status = 'queued'`)
465
+ .run(Date.now(), id);
466
+ cancelledQueued += Number(res.changes ?? 0);
467
+ // 2) 打断在跑(worker terminate / embed 循环退出);保持 aborted 直到下次 reindex
468
+ let ctrl = this.abortBySource.get(id);
469
+ if (!ctrl) {
470
+ ctrl = new AbortController();
471
+ this.abortBySource.set(id, ctrl);
472
+ }
473
+ if (!ctrl.signal.aborted) {
474
+ ctrl.abort();
475
+ abortedRunning += 1;
476
+ }
477
+ }
478
+ // 3) 看门狗不要再把 aborted running 收回 queued
479
+ this.sources.database.raw
480
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
481
+ WHERE status = 'running' AND last_error = 'aborted'`)
482
+ .run(Date.now());
483
+ this.emitProgress({
484
+ type: 'source',
485
+ sourceId: sid ?? '*',
486
+ status: 'aborted',
487
+ detail: `cancelledQueued=${cancelledQueued}`,
488
+ });
489
+ return { cancelledQueued, abortedRunning };
490
+ }
491
+ /**
492
+ * 恢复/继续索引(相对「重建」更轻)
493
+ *
494
+ * - 清除该源中止态
495
+ * - 把 cancelled 任务放回 queued
496
+ * - 若仍缺向量则排队 embed
497
+ * - 不重新 walk 全量;需要全量对齐请 reindex
498
+ *
499
+ * @param opts.sourceId - 仅恢复该源;省略则全部源
500
+ * @returns 恢复统计
501
+ */
502
+ resumeJobs(opts) {
503
+ const sid = opts?.sourceId?.trim();
504
+ const ids = sid
505
+ ? [sid]
506
+ : [...new Set([...this.abortBySource.keys(), ...this.sources.list().map((s) => s.id)])];
507
+ let restoredCancelled = 0;
508
+ let embedQueued = 0;
509
+ for (const id of ids) {
510
+ // 1) 清除中止纪元
511
+ this.beginAbortEpoch(id);
512
+ // 2) cancelled → queued(用户中止时被杀的活)
513
+ const res = this.sources.database.raw
514
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
515
+ WHERE source_id = ? AND status = 'cancelled'`)
516
+ .run(Date.now(), id);
517
+ restoredCancelled += Number(res.changes ?? 0);
518
+ // 3) 仍缺向量则确保 embed 在队
519
+ if (this.embedding) {
520
+ const missing = this.index.listChunksMissingEmbedding(id, 1).length > 0;
521
+ const active = this.countActiveJobs(id);
522
+ if (missing && active.embed === 0) {
523
+ this.ensureEmbedJob(id);
524
+ embedQueued += 1;
525
+ }
526
+ }
527
+ }
528
+ this.kick();
529
+ this.emitProgress({
530
+ type: 'source',
531
+ sourceId: sid ?? '*',
532
+ status: 'resumed',
533
+ detail: `restoredCancelled=${restoredCancelled} embedQueued=${embedQueued}`,
534
+ });
535
+ return { restoredCancelled, embedQueued };
536
+ }
537
+ /** 开启新的中止纪元(abort 后新任务不受旧信号影响) */
538
+ beginAbortEpoch(sourceId) {
539
+ this.abortBySource.set(sourceId, new AbortController());
540
+ }
541
+ /**
542
+ * 重建前作废本源未完成任务(supersede)
543
+ *
544
+ * - queued → cancelled(superseded_by_reindex)
545
+ * - running 打断(worker/循环退出)
546
+ * - 随后 beginAbortEpoch,本轮重建可继续写
547
+ */
548
+ supersedeSourceWork(sourceId) {
549
+ this.sources.database.raw
550
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'superseded_by_reindex', updated_at = ?
551
+ WHERE source_id = ? AND status = 'queued'`)
552
+ .run(Date.now(), sourceId);
553
+ // 打断在跑 parse/embed;再开新纪元,避免 walk 自己被 abort 卡死
554
+ let ctrl = this.abortBySource.get(sourceId);
555
+ if (!ctrl) {
556
+ ctrl = new AbortController();
557
+ this.abortBySource.set(sourceId, ctrl);
558
+ }
559
+ if (!ctrl.signal.aborted) {
560
+ ctrl.abort();
561
+ }
562
+ this.beginAbortEpoch(sourceId);
563
+ this.emitProgress({
564
+ type: 'source',
565
+ sourceId,
566
+ status: 'superseded',
567
+ detail: 'reindex will resync from disk',
568
+ });
569
+ }
570
+ async ensureParseCoverage(sourceId) {
571
+ const source = this.sources.get(sourceId);
572
+ if (!source || source.status === 'removed' || source.status === 'disabled')
573
+ return 0;
574
+ if (source.kind !== 'directory' && source.kind !== 'workspace')
575
+ return 0;
576
+ if (this.isAborted(sourceId))
577
+ return 0;
578
+ // 已有 parse 积压则不重复扫
579
+ if (this.countQueuedKinds(['parse_file', 'walk_source']) > 0)
580
+ return 0;
581
+ let found;
582
+ try {
583
+ found = await this.localFetcher.discover(source);
584
+ }
585
+ catch {
586
+ return 0;
587
+ }
588
+ const indexed = new Set(this.index.listFiles(sourceId).map((f) => f.path));
589
+ let added = 0;
590
+ for (const f of found) {
591
+ if (!indexed.has(f.path)) {
592
+ this.enqueue(sourceId, 'parse_file', f.path, 2, f.path);
593
+ added += 1;
594
+ }
595
+ }
596
+ if (added > 0) {
597
+ this.emitProgress({
598
+ type: 'source',
599
+ sourceId,
600
+ status: 'parse_gap_found',
601
+ detail: `missing_files=${added}`,
602
+ });
603
+ this.kick();
604
+ }
605
+ return added;
606
+ }
607
+ /**
608
+ * 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
609
+ *
610
+ * 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
611
+ * embed 前调用,避免给已移出文件的残 chunks 上向量。
612
+ *
613
+ * @param sourceId - 知识源 id
614
+ * @returns 删除的文件数
615
+ */
616
+ async pruneMissingOnDisk(sourceId) {
617
+ const source = this.sources.get(sourceId);
618
+ if (!source || source.kind === 'url' || source.kind === 'connector')
619
+ return 0;
620
+ let removed = 0;
621
+ if (source.kind === 'directory' || source.kind === 'workspace') {
622
+ try {
623
+ const found = await this.localFetcher.discover(source);
624
+ const keep = new Set(found.map((f) => f.path));
625
+ // walk 成功:即使空目录也可清空(与逐文件 stat 语义一致)
626
+ removed = this.index.pruneMissing(sourceId, keep, { allowEmptyKeep: true });
627
+ }
628
+ catch {
629
+ removed = await this.pruneByStatScan(sourceId);
630
+ }
631
+ }
632
+ else {
633
+ removed = await this.pruneByStatScan(sourceId);
634
+ }
635
+ if (removed > 0) {
636
+ this.emitProgress({
637
+ type: 'source',
638
+ sourceId,
639
+ status: 'pruned_missing',
640
+ detail: `removed_files=${removed}`,
641
+ });
642
+ this.refreshCoverage(sourceId);
643
+ }
644
+ return removed;
645
+ }
646
+ /** 逐文件 stat(file 源 / walk 失败回退);并行限流 */
647
+ async pruneByStatScan(sourceId, concurrency = 32) {
648
+ const files = this.index.listFiles(sourceId);
649
+ let removed = 0;
650
+ let i = 0;
651
+ const worker = async () => {
652
+ for (;;) {
653
+ if (this.isAborted(sourceId))
654
+ return;
655
+ const idx = i++;
656
+ if (idx >= files.length)
657
+ return;
658
+ const f = files[idx];
659
+ let exists = true;
660
+ try {
661
+ const st = await stat(f.path);
662
+ exists = st.isFile();
663
+ }
664
+ catch {
665
+ exists = false;
666
+ }
667
+ if (!exists) {
668
+ this.index.removePathTree(sourceId, f.path);
669
+ removed += 1;
670
+ }
671
+ }
672
+ };
673
+ await Promise.all(Array.from({ length: Math.min(concurrency, files.length || 1) }, () => worker()));
674
+ return removed;
675
+ }
676
+ abortSignalFor(sourceId) {
677
+ let ac = this.abortBySource.get(sourceId);
678
+ if (!ac) {
679
+ ac = new AbortController();
680
+ this.abortBySource.set(sourceId, ac);
681
+ }
682
+ return ac.signal;
683
+ }
684
+ isAborted(sourceId) {
685
+ return this.abortSignalFor(sourceId).aborted;
686
+ }
687
+ /**
688
+ * 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
689
+ *
690
+ * 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
691
+ * 1. 回收超时 running(崩溃/挂死孤儿)
692
+ * 2. 有缺向量且无 active embed → 自动排队
693
+ * 3. 无队列、无缺口 → 刷 coverage/status
694
+ */
695
+ startReconciler(intervalMs = 15_000) {
696
+ if (this.reconcileTimer || this.disposed)
697
+ return;
698
+ this.reconcileTimer = setInterval(() => {
699
+ void this.reconcileJobs().catch((err) => {
700
+ this.emitProgress({
701
+ type: 'error',
702
+ sourceId: '*',
703
+ status: 'reconcile_failed',
704
+ detail: err instanceof Error ? err.message : String(err),
705
+ });
706
+ });
707
+ }, intervalMs);
708
+ this.reconcileTimer.unref?.();
709
+ }
710
+ stopReconciler() {
711
+ if (this.reconcileTimer) {
712
+ clearInterval(this.reconcileTimer);
713
+ this.reconcileTimer = null;
714
+ }
715
+ }
716
+ /** 对账入口(也可手工调用) */
717
+ async reconcileJobs() {
718
+ if (this.disposed)
719
+ return;
720
+ this.reclaimStaleRunningJobs(5 * 60_000);
721
+ if (!this.embedding) {
722
+ for (const s of this.sources.list()) {
723
+ if (s.status === 'removed' || s.status === 'disabled')
724
+ continue;
725
+ this.refreshCoverage(s.id);
726
+ }
727
+ return;
728
+ }
729
+ for (const s of this.sources.list()) {
730
+ if (s.status === 'removed' || s.status === 'disabled')
731
+ continue;
732
+ if (this.isAborted(s.id))
733
+ continue;
734
+ // watch 漏事件时补齐 parse:磁盘有、索引无 → 入队(解析优先于 embed)
735
+ const lastGap = this.lastParseGapScanAt.get(s.id) ?? 0;
736
+ if (Date.now() - lastGap > 60_000) {
737
+ this.lastParseGapScanAt.set(s.id, Date.now());
738
+ await this.ensureParseCoverage(s.id);
739
+ }
740
+ const missing = this.index.listChunksMissingEmbedding(s.id, 1).length > 0;
741
+ const active = this.countActiveJobs(s.id);
742
+ if (missing && active.embed === 0) {
743
+ this.ensureEmbedJob(s.id);
744
+ }
745
+ else if (!missing && active.total === 0) {
746
+ this.refreshCoverage(s.id);
747
+ // 稳定态收尾(auto-describe 等);回调不得抛出打断对账
748
+ try {
749
+ this.onSourceSettled?.(s.id);
750
+ }
751
+ catch {
752
+ /* ignore settle hook errors */
753
+ }
754
+ }
755
+ }
756
+ this.kick();
757
+ }
758
+ /** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
759
+ reclaimStaleRunningJobs(staleMs) {
760
+ const cutoff = Date.now() - staleMs;
761
+ // 先放弃重试过多的
762
+ this.sources.database.raw
763
+ .prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
764
+ WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
765
+ .run(Date.now(), cutoff);
766
+ this.sources.database.raw
767
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
768
+ WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
769
+ .run(Date.now(), cutoff);
770
+ }
771
+ countActiveJobs(sourceId) {
772
+ const row = this.sources.database.raw
773
+ .prepare(`SELECT
774
+ COUNT(*) AS total,
775
+ SUM(CASE WHEN kind = 'embed_source' THEN 1 ELSE 0 END) AS embed
776
+ FROM knowledge_jobs
777
+ WHERE source_id = ? AND status IN ('queued', 'running')`)
778
+ .get(sourceId);
779
+ return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
780
+ }
420
781
  /**
421
782
  * 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
422
783
  */
@@ -495,6 +856,8 @@ export class KnowledgeIngest extends EventEmitter {
495
856
  return (row?.n ?? 0) > 0;
496
857
  }
497
858
  enqueue(sourceId, kind, path, priority, _detail) {
859
+ if (this.isAborted(sourceId))
860
+ return;
498
861
  // 同 source+kind 去重(queued + running,避免并发重复 embed)
499
862
  const activeStatuses = ['queued', 'running'];
500
863
  if (path != null) {
@@ -536,30 +899,57 @@ export class KnowledgeIngest extends EventEmitter {
536
899
  .run(`kj_${randomUUID().slice(0, 12)}`, sourceId, kind, path, priority, now, now);
537
900
  }
538
901
  kick() {
539
- if (this.disposed || this.draining)
902
+ if (this.disposed)
540
903
  return;
904
+ if (this.draining) {
905
+ // 等待中的 drain 会再 kick;标记一次避免丢信号
906
+ this.kickAgain = true;
907
+ return;
908
+ }
541
909
  this.draining = true;
542
910
  queueMicrotask(() => {
543
911
  this.draining = false;
544
912
  this.drain();
913
+ if (this.kickAgain) {
914
+ this.kickAgain = false;
915
+ this.kick();
916
+ }
545
917
  });
546
918
  }
547
919
  drain() {
548
- while (this.running < this.parseConcurrency) {
549
- const job = this.claimJob();
550
- if (!job)
920
+ for (;;) {
921
+ const allowParse = this.runningParse < this.parseConcurrency;
922
+ const allowEmbed = this.runningEmbed < this.embedConcurrency;
923
+ if (!allowParse && !allowEmbed)
551
924
  return;
552
- const isEmbed = job.kind === 'embed_source';
553
- if (isEmbed && this.runningEmbed >= this.embedConcurrency) {
554
- this.sources.database.raw
555
- .prepare(`UPDATE knowledge_jobs SET status = 'queued' WHERE id = ? AND status = 'running'`)
556
- .run(job.id);
557
- // embed 满:本轮不再认领,避免空转把同一 job 抢来抢去
925
+ const parsePending = this.countQueuedKinds([
926
+ 'parse_file',
927
+ 'walk_source',
928
+ 'drop_file',
929
+ 'fetch_doc',
930
+ ]);
931
+ const kinds = [];
932
+ if (allowParse)
933
+ kinds.push('parse_file', 'drop_file', 'walk_source', 'fetch_doc');
934
+ // 业务优先级:解析/分块做完即可关键词搜;embedding 仅在无 parse 积压时占槽
935
+ if (allowEmbed && parsePending === 0)
936
+ kinds.push('embed_source');
937
+ if (kinds.length === 0) {
938
+ if (!allowParse && allowEmbed && parsePending === 0)
939
+ return;
940
+ // parse 槽满但仍有 parse 积压 → 等 parse,不抢 embed
941
+ if (parsePending > 0 && !allowParse)
942
+ return;
558
943
  return;
559
944
  }
560
- this.running += 1;
945
+ const job = this.claimJob(kinds);
946
+ if (!job)
947
+ return;
948
+ const isEmbed = job.kind === 'embed_source';
561
949
  if (isEmbed)
562
950
  this.runningEmbed += 1;
951
+ else
952
+ this.runningParse += 1;
563
953
  void this.runJob(job)
564
954
  .catch((err) => {
565
955
  this.emitProgress({
@@ -571,20 +961,33 @@ export class KnowledgeIngest extends EventEmitter {
571
961
  });
572
962
  })
573
963
  .finally(() => {
574
- this.running -= 1;
575
964
  if (isEmbed)
576
965
  this.runningEmbed -= 1;
966
+ else
967
+ this.runningParse -= 1;
577
968
  this.kick();
578
969
  });
579
970
  }
580
971
  }
581
- claimJob() {
972
+ countQueuedKinds(kinds) {
973
+ if (kinds.length === 0)
974
+ return 0;
975
+ const placeholders = kinds.map(() => '?').join(',');
976
+ const row = this.sources.database.raw
977
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`)
978
+ .get(...kinds);
979
+ return row?.n ?? 0;
980
+ }
981
+ claimJob(kinds) {
982
+ if (kinds.length === 0)
983
+ return null;
984
+ const placeholders = kinds.map(() => '?').join(',');
582
985
  const row = this.sources.database.raw
583
986
  .prepare(`SELECT * FROM knowledge_jobs
584
- WHERE status = 'queued'
987
+ WHERE status = 'queued' AND kind IN (${placeholders})
585
988
  ORDER BY priority ASC, created_at ASC
586
989
  LIMIT 1`)
587
- .get();
990
+ .get(...kinds);
588
991
  if (!row)
589
992
  return null;
590
993
  const res = this.sources.database.raw
@@ -597,9 +1000,20 @@ export class KnowledgeIngest extends EventEmitter {
597
1000
  }
598
1001
  async runJob(job) {
599
1002
  const now = Date.now();
1003
+ if (this.isAborted(job.source_id)) {
1004
+ this.sources.database.raw
1005
+ .prepare(`UPDATE knowledge_jobs SET status = 'cancelled', last_error = 'aborted', updated_at = ?
1006
+ WHERE id = ? AND status = 'running'`)
1007
+ .run(Date.now(), job.id);
1008
+ return;
1009
+ }
600
1010
  try {
601
1011
  if (job.kind === 'parse_file' && job.path) {
602
- await this.parseOne(job.source_id, job.path);
1012
+ // 单文件解析硬超时:大 xlsx/pdf 同步抽取会堵死事件循环
1013
+ await this.withTimeout(this.parseOne(job.source_id, job.path), this.parseTimeoutMs, `parse_timeout:${job.path}`);
1014
+ if (this.isAborted(job.source_id)) {
1015
+ throw new Error('aborted');
1016
+ }
603
1017
  this.refreshCoverage(job.source_id);
604
1018
  if (this.embedding) {
605
1019
  this.enqueue(job.source_id, 'embed_source', null, 3);
@@ -607,7 +1021,7 @@ export class KnowledgeIngest extends EventEmitter {
607
1021
  }
608
1022
  }
609
1023
  else if (job.kind === 'fetch_doc' && job.path) {
610
- await this.fetchDocJob(job.source_id, job.path);
1024
+ await this.withTimeout(this.fetchDocJob(job.source_id, job.path), this.parseTimeoutMs, `fetch_timeout:${job.path}`);
611
1025
  this.refreshCoverage(job.source_id);
612
1026
  if (this.embedding) {
613
1027
  this.enqueue(job.source_id, 'embed_source', null, 3);
@@ -615,14 +1029,23 @@ export class KnowledgeIngest extends EventEmitter {
615
1029
  }
616
1030
  }
617
1031
  else if (job.kind === 'drop_file' && job.path) {
618
- this.index.removeFile(job.source_id, job.path);
1032
+ // 目录移出/删除:连同子路径一并清(否则只剩空目录节点)
1033
+ this.index.removePathTree(job.source_id, job.path);
619
1034
  this.refreshCoverage(job.source_id);
620
1035
  }
621
1036
  else if (job.kind === 'walk_source') {
622
1037
  await this.ingestSource(job.source_id);
623
1038
  }
624
1039
  else if (job.kind === 'embed_source') {
625
- await this.embedPending(job.source_id);
1040
+ if (!this.embedding) {
1041
+ const stillMissing = this.index.listChunksMissingEmbedding(job.source_id, 1).length > 0;
1042
+ if (stillMissing) {
1043
+ // 禁止空跑标 done:迁移后缺向量却显示 queue=0
1044
+ throw new Error('embedding_provider_missing');
1045
+ }
1046
+ return;
1047
+ }
1048
+ await this.embedPending(job.source_id, 50, job.id);
626
1049
  }
627
1050
  this.sources.database.raw
628
1051
  .prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
@@ -633,17 +1056,44 @@ export class KnowledgeIngest extends EventEmitter {
633
1056
  path: job.path ?? undefined,
634
1057
  status: 'done',
635
1058
  });
1059
+ // 标 done 后再续跑:否则 enqueue 去重会把「自己」当成 active 而丢掉下一棒
1060
+ if (job.kind === 'embed_source' && !this.isAborted(job.source_id)) {
1061
+ this.ensureEmbedJob(job.source_id);
1062
+ this.refreshCoverage(job.source_id);
1063
+ }
636
1064
  }
637
1065
  catch (err) {
638
1066
  const message = err instanceof Error ? err.message : String(err);
1067
+ const aborted = message === 'aborted' || this.isAborted(job.source_id);
1068
+ if (job.kind === 'parse_file' && job.path && !aborted) {
1069
+ this.index.markFileError(job.source_id, job.path, message);
1070
+ }
639
1071
  this.sources.database.raw
640
1072
  .prepare(`UPDATE knowledge_jobs
641
- SET status = 'failed', attempts = attempts + 1, last_error = ?, updated_at = ?
1073
+ SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
642
1074
  WHERE id = ?`)
643
- .run(message, now, job.id);
1075
+ .run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, now, job.id);
1076
+ if (job.kind === 'embed_source' && !aborted) {
1077
+ this.ensureEmbedJob(job.source_id);
1078
+ }
644
1079
  throw err;
645
1080
  }
646
1081
  }
1082
+ async withTimeout(p, ms, label) {
1083
+ let timer;
1084
+ try {
1085
+ return await Promise.race([
1086
+ p,
1087
+ new Promise((_, reject) => {
1088
+ timer = setTimeout(() => reject(new Error(label)), ms);
1089
+ }),
1090
+ ]);
1091
+ }
1092
+ finally {
1093
+ if (timer)
1094
+ clearTimeout(timer);
1095
+ }
1096
+ }
647
1097
  /**
648
1098
  * 外源单文档 job:按逻辑键 fetch + 索引
649
1099
  */
@@ -750,7 +1200,8 @@ export class KnowledgeIngest extends EventEmitter {
750
1200
  return true;
751
1201
  }
752
1202
  try {
753
- const result = await port.extract({ path: filePath, name: filePath });
1203
+ // worker 抽取:同步 xlsx/pdf 不堵主循环;超时/中止可 terminate
1204
+ const result = await this.extractDocument(filePath, sourceId);
754
1205
  const markdown = result.markdown?.trim();
755
1206
  if (!markdown) {
756
1207
  this.index.markFileSkipped(sourceId, filePath, 'empty_content');
@@ -778,51 +1229,124 @@ export class KnowledgeIngest extends EventEmitter {
778
1229
  return false;
779
1230
  }
780
1231
  }
1232
+ /** 优先 worker 抽取;worker 不可用时退回进程内 port */
1233
+ async extractDocument(filePath, sourceId) {
1234
+ const signal = this.abortSignalFor(sourceId);
1235
+ try {
1236
+ const { extractDocumentInWorker } = await import('./extract-document.js');
1237
+ return await extractDocumentInWorker(filePath, {
1238
+ timeoutMs: this.parseTimeoutMs,
1239
+ maxFileBytes: this.maxFileBytes,
1240
+ signal,
1241
+ });
1242
+ }
1243
+ catch (err) {
1244
+ const msg = err instanceof Error ? err.message : String(err);
1245
+ if (msg === 'extract_aborted' || signal.aborted) {
1246
+ throw new Error('aborted');
1247
+ }
1248
+ if (msg.includes('Cannot find module') || msg.includes('ERR_MODULE')) {
1249
+ return this.documentPort.extract({ path: filePath, name: filePath }, { timeoutMs: this.parseTimeoutMs });
1250
+ }
1251
+ throw err;
1252
+ }
1253
+ }
781
1254
  /**
782
- * Phase B:嵌入待处理 chunk(批 + 限速;失败不丢,下次再补)
1255
+ * Phase B:嵌入待处理 chunk
1256
+ *
1257
+ * 单 job 内按 embedConcurrency **并行**多批调用 API;
1258
+ * 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
783
1259
  */
784
- async embedPending(sourceId) {
785
- if (!this.embedding)
786
- return;
787
- const pending = this.index.listChunksMissingEmbedding(sourceId, this.embedBatch);
788
- if (pending.length === 0)
1260
+ async embedPending(sourceId, maxRounds = 20, jobId) {
1261
+ if (!this.embedding) {
1262
+ if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
1263
+ throw new Error('embedding_provider_missing');
1264
+ }
789
1265
  return;
790
- if (this.embedMinIntervalMs > 0) {
791
- const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
792
- if (wait > 0)
793
- await new Promise((r) => setTimeout(r, wait));
794
1266
  }
795
- try {
796
- const vectors = await this.embedding.embedBatch(pending.map((p) => p.text));
797
- for (let i = 0; i < pending.length; i++) {
798
- const vec = vectors[i];
799
- if (vec?.length)
800
- this.index.setChunkEmbedding(pending[i].id, vec);
1267
+ // 磁盘对齐在 reindex/walk 做;embed 任务内不再全量 prune(曾拖慢解析)
1268
+ const lanes = Math.max(1, this.embedConcurrency);
1269
+ const batchSize = Math.max(1, this.embedBatch);
1270
+ for (let round = 0; round < maxRounds; round++) {
1271
+ if (this.isAborted(sourceId))
1272
+ return;
1273
+ const pending = this.index.listChunksMissingEmbedding(sourceId, batchSize * lanes);
1274
+ if (pending.length === 0) {
1275
+ this.refreshCoverage(sourceId);
1276
+ return;
1277
+ }
1278
+ if (jobId)
1279
+ this.heartbeatJob(jobId);
1280
+ // 轮次限速(并行片共享同一闸门)
1281
+ if (this.embedMinIntervalMs > 0) {
1282
+ const wait = this.lastEmbedAt + this.embedMinIntervalMs - Date.now();
1283
+ if (wait > 0)
1284
+ await new Promise((r) => setTimeout(r, wait));
1285
+ }
1286
+ const slices = [];
1287
+ for (let i = 0; i < pending.length; i += batchSize) {
1288
+ slices.push(pending.slice(i, i + batchSize));
1289
+ }
1290
+ const settled = await Promise.allSettled(slices.map((slice) => this.embedOneSlice(sourceId, slice)));
1291
+ let ok = 0;
1292
+ let failed = false;
1293
+ for (const s of settled) {
1294
+ if (s.status === 'fulfilled')
1295
+ ok += s.value;
1296
+ else
1297
+ failed = true;
801
1298
  }
802
1299
  this.lastEmbedAt = Date.now();
803
1300
  this.emitProgress({
804
1301
  type: 'embed',
805
1302
  sourceId,
806
1303
  status: 'batch_done',
807
- detail: `embedded=${pending.length}`,
808
- });
809
- }
810
- catch (err) {
811
- this.emitProgress({
812
- type: 'error',
813
- sourceId,
814
- status: 'embed_failed',
815
- detail: err instanceof Error ? err.message : String(err),
1304
+ detail: `embedded=${ok}/${pending.length} lanes=${slices.length}`,
816
1305
  });
817
- return;
818
- }
819
- // 仍有 pending 则继续排队
820
- if (this.index.listChunksMissingEmbedding(sourceId, 1).length > 0) {
821
- this.enqueue(sourceId, 'embed_source', null, 3);
822
- this.kick();
1306
+ if (failed) {
1307
+ // 至少一片失败:交给 done 后的 ensureEmbedJob 续跑,不把整 job 打成 failed
1308
+ return;
1309
+ }
823
1310
  }
824
1311
  this.refreshCoverage(sourceId);
825
1312
  }
1313
+ /** 单片 embedding;返回成功写入条数 */
1314
+ async embedOneSlice(sourceId, slice) {
1315
+ if (slice.length === 0)
1316
+ return 0;
1317
+ const vectors = await Promise.race([
1318
+ this.embedding.embedBatch(slice.map((p) => p.text)),
1319
+ new Promise((_, reject) => setTimeout(() => reject(new Error('embed_batch_timeout')), 120_000)),
1320
+ ]);
1321
+ let ok = 0;
1322
+ const batch = [];
1323
+ for (let i = 0; i < slice.length; i++) {
1324
+ const vec = vectors[i];
1325
+ if (vec?.length) {
1326
+ batch.push([slice[i].id, vec]);
1327
+ ok += 1;
1328
+ }
1329
+ }
1330
+ if (batch.length > 0)
1331
+ this.index.setChunkEmbeddings(batch);
1332
+ return ok;
1333
+ }
1334
+ /** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
1335
+ ensureEmbedJob(sourceId) {
1336
+ if (!this.embedding)
1337
+ return;
1338
+ if (this.isAborted(sourceId))
1339
+ return;
1340
+ if (this.index.listChunksMissingEmbedding(sourceId, 1).length === 0)
1341
+ return;
1342
+ this.enqueue(sourceId, 'embed_source', null, 3);
1343
+ this.kick();
1344
+ }
1345
+ heartbeatJob(jobId) {
1346
+ this.sources.database.raw
1347
+ .prepare(`UPDATE knowledge_jobs SET updated_at = ? WHERE id = ? AND status = 'running'`)
1348
+ .run(Date.now(), jobId);
1349
+ }
826
1350
  refreshCoverage(sourceId) {
827
1351
  const stats = this.index.sourceStats(sourceId);
828
1352
  const source = this.sources.get(sourceId);