@octopi-agent/engine 0.58.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +38 -1
  4. package/dist/config-schema/substrate.d.ts +37 -0
  5. package/dist/config-schema/substrate.js +47 -0
  6. package/dist/config.d.ts +30 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  23. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  24. package/dist/harness/capabilities/document/factory.js +37 -0
  25. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  37. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  39. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  40. package/dist/harness/capabilities/document/worker-port.js +78 -0
  41. package/dist/harness/context/capabilities/index.d.ts +2 -2
  42. package/dist/harness/context/capabilities/index.js +2 -2
  43. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  44. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  45. package/dist/harness/index.d.ts +4 -2
  46. package/dist/harness/index.js +2 -1
  47. package/dist/harness/knowledge/adapters.d.ts +8 -5
  48. package/dist/harness/knowledge/adapters.js +53 -13
  49. package/dist/harness/knowledge/client.d.ts +197 -0
  50. package/dist/harness/knowledge/client.js +237 -0
  51. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  52. package/dist/harness/knowledge/connectors.d.ts +3 -3
  53. package/dist/harness/knowledge/connectors.js +14 -8
  54. package/dist/harness/knowledge/db.d.ts +40 -5
  55. package/dist/harness/knowledge/db.js +441 -113
  56. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  57. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  58. package/dist/harness/knowledge/embed-runner.js +171 -0
  59. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  60. package/dist/harness/knowledge/engine-thread.js +302 -0
  61. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  62. package/dist/harness/knowledge/extract-document.js +32 -8
  63. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  64. package/dist/harness/knowledge/fetchers.js +52 -16
  65. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  66. package/dist/harness/knowledge/file-identity.js +172 -0
  67. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  68. package/dist/harness/knowledge/file-index-store.js +132 -0
  69. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  70. package/dist/harness/knowledge/file-limits.js +175 -0
  71. package/dist/harness/knowledge/fts.d.ts +60 -0
  72. package/dist/harness/knowledge/fts.js +218 -0
  73. package/dist/harness/knowledge/grounding.d.ts +3 -2
  74. package/dist/harness/knowledge/http-app.d.ts +50 -0
  75. package/dist/harness/knowledge/http-app.js +652 -0
  76. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  77. package/dist/harness/knowledge/http-bridge.js +17 -0
  78. package/dist/harness/knowledge/index-store.d.ts +158 -93
  79. package/dist/harness/knowledge/index-store.js +982 -441
  80. package/dist/harness/knowledge/index.d.ts +45 -5
  81. package/dist/harness/knowledge/index.js +23 -2
  82. package/dist/harness/knowledge/ingest.d.ts +262 -87
  83. package/dist/harness/knowledge/ingest.js +1072 -480
  84. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  85. package/dist/harness/knowledge/job-control-state.js +44 -0
  86. package/dist/harness/knowledge/job-control.d.ts +35 -0
  87. package/dist/harness/knowledge/job-control.js +86 -0
  88. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  89. package/dist/harness/knowledge/job-queue.js +173 -0
  90. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  91. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  92. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  93. package/dist/harness/knowledge/local-stack.js +63 -0
  94. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  95. package/dist/harness/knowledge/membership-store.js +198 -0
  96. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  97. package/dist/harness/knowledge/network-guard.js +130 -85
  98. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  99. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  100. package/dist/harness/knowledge/query-service.d.ts +272 -0
  101. package/dist/harness/knowledge/query-service.js +326 -0
  102. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  103. package/dist/harness/knowledge/query-worker-client.js +207 -0
  104. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  105. package/dist/harness/knowledge/query-worker.js +186 -0
  106. package/dist/harness/knowledge/read-http.d.ts +35 -0
  107. package/dist/harness/knowledge/read-http.js +373 -0
  108. package/dist/harness/knowledge/retriever.d.ts +22 -5
  109. package/dist/harness/knowledge/retriever.js +66 -35
  110. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  111. package/dist/harness/knowledge/secret-scan.js +25 -2
  112. package/dist/harness/knowledge/serve.d.ts +48 -0
  113. package/dist/harness/knowledge/serve.js +336 -0
  114. package/dist/harness/knowledge/source-store.d.ts +80 -17
  115. package/dist/harness/knowledge/source-store.js +160 -89
  116. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  117. package/dist/harness/knowledge/start-service-process.js +158 -0
  118. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  119. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  120. package/dist/harness/knowledge/types.d.ts +1 -1
  121. package/dist/harness/knowledge/types.js +1 -1
  122. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  123. package/dist/harness/knowledge/vector-ann.js +68 -0
  124. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  125. package/dist/harness/knowledge/writer-lock.js +71 -0
  126. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  127. package/dist/harness/knowledge/writer-service.js +355 -0
  128. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  129. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  130. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  131. package/dist/harness/knowledge/writer-worker.js +189 -0
  132. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  133. package/dist/harness/memory/sqlite/embedding.js +109 -16
  134. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  135. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  136. package/dist/harness/session/attachments/service.d.ts +1 -1
  137. package/dist/harness/session/attachments/service.js +1 -1
  138. package/dist/index.d.ts +4 -2
  139. package/dist/index.js +2 -1
  140. package/dist/worker-path.d.ts +8 -0
  141. package/dist/worker-path.js +15 -0
  142. package/package.json +2 -2
@@ -0,0 +1,29 @@
1
+ /**
2
+ * 源级 job 控制状态 — 纯 SQL,只读连接可跑。
3
+ *
4
+ * 与 KnowledgeIngest.jobControlState 同口径;禁止依赖 ingest 内存态
5
+ * (中止纪元以 knowledge_source_control 为权威)。
6
+ */
7
+ import type { KnowledgeDatabase } from './db.js';
8
+ import type { KnowledgeIndexStore } from './index-store.js';
9
+ export interface KnowledgeJobControlState {
10
+ aborted: boolean;
11
+ jobsQueued: number;
12
+ jobsRunning: number;
13
+ jobsCancelled: number;
14
+ embedMissing: boolean;
15
+ canAbort: boolean;
16
+ canResume: boolean;
17
+ }
18
+ /**
19
+ * 计算源的中止/继续控制面。
20
+ *
21
+ * @param db - knowledge 连接(只读亦可)
22
+ * @param index - 缺向量探测
23
+ * @param sourceId - 源 id
24
+ * @param opts.embeddingEnabled - 是否配置了 embedding(未配置则 embedMissing=false)
25
+ * @returns 控制按钮状态
26
+ */
27
+ export declare function readJobControlState(db: KnowledgeDatabase, index: KnowledgeIndexStore, sourceId: string, opts?: {
28
+ embeddingEnabled?: boolean;
29
+ }): KnowledgeJobControlState;
@@ -0,0 +1,44 @@
1
+ /**
2
+ * 源级 job 控制状态 — 纯 SQL,只读连接可跑。
3
+ *
4
+ * 与 KnowledgeIngest.jobControlState 同口径;禁止依赖 ingest 内存态
5
+ * (中止纪元以 knowledge_source_control 为权威)。
6
+ */
7
+ /**
8
+ * 计算源的中止/继续控制面。
9
+ *
10
+ * @param db - knowledge 连接(只读亦可)
11
+ * @param index - 缺向量探测
12
+ * @param sourceId - 源 id
13
+ * @param opts.embeddingEnabled - 是否配置了 embedding(未配置则 embedMissing=false)
14
+ * @returns 控制按钮状态
15
+ */
16
+ export function readJobControlState(db, index, sourceId, opts) {
17
+ const row = db.raw
18
+ .prepare(`SELECT
19
+ SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS q,
20
+ SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS r,
21
+ SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS c
22
+ FROM knowledge_jobs WHERE source_id = ?`)
23
+ .get(sourceId);
24
+ const jobsQueued = Number(row?.q ?? 0);
25
+ const jobsRunning = Number(row?.r ?? 0);
26
+ const jobsCancelled = Number(row?.c ?? 0);
27
+ const abortedRow = db.raw
28
+ .prepare('SELECT aborted FROM knowledge_source_control WHERE source_id = ?')
29
+ .get(sourceId);
30
+ const aborted = Number(abortedRow?.aborted ?? 0) === 1;
31
+ const embedMissing = opts?.embeddingEnabled
32
+ ? index.hasChunksMissingEmbedding(sourceId)
33
+ : false;
34
+ const active = jobsQueued + jobsRunning > 0;
35
+ return {
36
+ aborted,
37
+ jobsQueued,
38
+ jobsRunning,
39
+ jobsCancelled,
40
+ embedMissing,
41
+ canAbort: active && !aborted,
42
+ canResume: aborted || jobsCancelled > 0 || (embedMissing && !active),
43
+ };
44
+ }
@@ -0,0 +1,35 @@
1
+ /**
2
+ * KnowledgeJobControl — 源级中止/继续纪元
3
+ *
4
+ * 中止态必须跨进程存活:DB 是权威,内存 AbortController 只是本进程信号。
5
+ * 调度前读库;`aborted` 则不派发新任务。
6
+ */
7
+ import type { KnowledgeDatabase } from './db.js';
8
+ export interface AbortResumeStats {
9
+ cancelledQueued: number;
10
+ abortedRunning: number;
11
+ runningJobs: number;
12
+ restoredCancelled: number;
13
+ embedQueued: number;
14
+ }
15
+ export declare class KnowledgeJobControl {
16
+ private readonly db;
17
+ /** 本进程信号:abort 后 worker/embed 循环应立刻退出 */
18
+ private abortBySource;
19
+ constructor(db: KnowledgeDatabase);
20
+ /** 是否处于中止态(DB 权威;进程重启后仍生效)。连接已关时视为停止调度 */
21
+ isAborted(sourceId: string): boolean;
22
+ /** 本进程 AbortSignal(无记录时惰性创建未中止的纪元) */
23
+ signalFor(sourceId: string): AbortSignal;
24
+ /** 开启新的未中止纪元(resume / supersede 后用) */
25
+ beginEpoch(sourceId: string): void;
26
+ /**
27
+ * 中止:DB 落 aborted + 打断本进程信号。
28
+ * 不在此清 jobs(调用方负责 queued/running 收尾)。
29
+ */
30
+ markAborted(sourceId: string): boolean;
31
+ /** 列出全部处于中止态的源(启动恢复 / 全局 abort 用) */
32
+ listAbortedSourceIds(): string[];
33
+ /** 调度闸门:中止源不得 claim 新任务 */
34
+ allowsDispatch(sourceId: string): boolean;
35
+ }
@@ -0,0 +1,86 @@
1
+ /**
2
+ * KnowledgeJobControl — 源级中止/继续纪元
3
+ *
4
+ * 中止态必须跨进程存活:DB 是权威,内存 AbortController 只是本进程信号。
5
+ * 调度前读库;`aborted` 则不派发新任务。
6
+ */
7
+ export class KnowledgeJobControl {
8
+ db;
9
+ /** 本进程信号:abort 后 worker/embed 循环应立刻退出 */
10
+ abortBySource = new Map();
11
+ constructor(db) {
12
+ this.db = db;
13
+ this.db.raw.exec(`
14
+ CREATE TABLE IF NOT EXISTS knowledge_source_control (
15
+ source_id TEXT PRIMARY KEY,
16
+ aborted INTEGER NOT NULL DEFAULT 0,
17
+ aborted_at INTEGER,
18
+ updated_at INTEGER NOT NULL
19
+ );
20
+ `);
21
+ }
22
+ /** 是否处于中止态(DB 权威;进程重启后仍生效)。连接已关时视为停止调度 */
23
+ isAborted(sourceId) {
24
+ try {
25
+ const row = this.db.raw
26
+ .prepare('SELECT aborted FROM knowledge_source_control WHERE source_id = ?')
27
+ .get(sourceId);
28
+ return Number(row?.aborted ?? 0) === 1;
29
+ }
30
+ catch {
31
+ return true;
32
+ }
33
+ }
34
+ /** 本进程 AbortSignal(无记录时惰性创建未中止的纪元) */
35
+ signalFor(sourceId) {
36
+ let ac = this.abortBySource.get(sourceId);
37
+ if (!ac) {
38
+ ac = new AbortController();
39
+ this.abortBySource.set(sourceId, ac);
40
+ // 库里是 aborted 而内存还没有 → 补上信号,保证在跑任务能被打断
41
+ if (this.isAborted(sourceId) && !ac.signal.aborted)
42
+ ac.abort();
43
+ }
44
+ return ac.signal;
45
+ }
46
+ /** 开启新的未中止纪元(resume / supersede 后用) */
47
+ beginEpoch(sourceId) {
48
+ this.abortBySource.set(sourceId, new AbortController());
49
+ this.db.raw
50
+ .prepare(`INSERT INTO knowledge_source_control (source_id, aborted, aborted_at, updated_at)
51
+ VALUES (?, 0, NULL, ?)
52
+ ON CONFLICT(source_id) DO UPDATE SET aborted = 0, aborted_at = NULL, updated_at = excluded.updated_at`)
53
+ .run(sourceId, Date.now());
54
+ }
55
+ /**
56
+ * 中止:DB 落 aborted + 打断本进程信号。
57
+ * 不在此清 jobs(调用方负责 queued/running 收尾)。
58
+ */
59
+ markAborted(sourceId) {
60
+ const already = this.isAborted(sourceId);
61
+ this.db.raw
62
+ .prepare(`INSERT INTO knowledge_source_control (source_id, aborted, aborted_at, updated_at)
63
+ VALUES (?, 1, ?, ?)
64
+ ON CONFLICT(source_id) DO UPDATE SET aborted = 1, aborted_at = excluded.aborted_at, updated_at = excluded.updated_at`)
65
+ .run(sourceId, Date.now(), Date.now());
66
+ let ctrl = this.abortBySource.get(sourceId);
67
+ if (!ctrl) {
68
+ ctrl = new AbortController();
69
+ this.abortBySource.set(sourceId, ctrl);
70
+ }
71
+ if (!ctrl.signal.aborted)
72
+ ctrl.abort();
73
+ return !already;
74
+ }
75
+ /** 列出全部处于中止态的源(启动恢复 / 全局 abort 用) */
76
+ listAbortedSourceIds() {
77
+ const rows = this.db.raw
78
+ .prepare('SELECT source_id FROM knowledge_source_control WHERE aborted = 1')
79
+ .all();
80
+ return rows.map((r) => r.source_id);
81
+ }
82
+ /** 调度闸门:中止源不得 claim 新任务 */
83
+ allowsDispatch(sourceId) {
84
+ return !this.isAborted(sourceId);
85
+ }
86
+ }
@@ -0,0 +1,62 @@
1
+ /**
2
+ * JobQueue — knowledge_jobs 表原语(enqueue / claim / 计数 / 回收 / 清理)
3
+ *
4
+ * 从 KnowledgeIngest 拆出;调度编排(drain/runJob)仍在 ingest。
5
+ */
6
+ import type { KnowledgeDatabase } from './db.js';
7
+ export type IngestJobKind = 'parse_file' | 'walk_source' | 'drop_file' | 'embed_source' | 'fetch_doc';
8
+ export interface JobRow {
9
+ id: string;
10
+ source_id: string;
11
+ file_id: string | null;
12
+ kind: string;
13
+ path: string | null;
14
+ priority: number;
15
+ status: string;
16
+ attempts: number;
17
+ last_error: string | null;
18
+ created_at: number;
19
+ updated_at: number;
20
+ }
21
+ export interface JobQueueDeps {
22
+ db: KnowledgeDatabase;
23
+ maxQueueDepth: number;
24
+ isAborted: (sourceId: string) => boolean;
25
+ onBackpressure?: (sourceId: string, queued: number) => void;
26
+ }
27
+ export declare class JobQueue {
28
+ private readonly deps;
29
+ /** 深度探测节流:watch 突发 800 文件时禁止每次 enqueue 都 COUNT(*) */
30
+ private depthSampledAt;
31
+ private depthSample;
32
+ constructor(deps: JobQueueDeps);
33
+ private sampleDepth;
34
+ hasQueuedJobs(): boolean;
35
+ countQueuedKinds(kinds: IngestJobKind[], sourceId?: string): number;
36
+ countActiveJobs(sourceId: string): {
37
+ total: number;
38
+ embed: number;
39
+ };
40
+ hasActivePathJob(sourceId: string, path: string): boolean;
41
+ /**
42
+ * 入队(同 source+kind+path 去重;有 file_id 时按 (file_id,kind) 去重;中止源拒绝)
43
+ *
44
+ * @returns 是否真正插入
45
+ */
46
+ enqueue(sourceId: string, kind: IngestJobKind, path: string | null, priority: number, fileId?: string | null): boolean;
47
+ /**
48
+ * 原子认领:queued → running;中止源不派发。
49
+ *
50
+ * @param kinds - 允许的 job 类型
51
+ * @param accept - 槽位/类型闸门;返回 false 则跳过该条继续找下一条(不永久占队)
52
+ */
53
+ claimJob(kinds: IngestJobKind[], accept?: (job: JobRow) => boolean): JobRow | null;
54
+ heartbeat(jobId: string): void;
55
+ markDone(jobId: string): void;
56
+ markFailedOrCancelled(jobId: string, aborted: boolean, message: string): void;
57
+ /** 启动回收孤儿 running → queued */
58
+ reclaimOrphanRunning(): void;
59
+ /** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed */
60
+ reclaimStaleRunning(staleMs: number): void;
61
+ cleanupTerminal(retainMs?: number): void;
62
+ }
@@ -0,0 +1,173 @@
1
+ /**
2
+ * JobQueue — knowledge_jobs 表原语(enqueue / claim / 计数 / 回收 / 清理)
3
+ *
4
+ * 从 KnowledgeIngest 拆出;调度编排(drain/runJob)仍在 ingest。
5
+ */
6
+ import { randomUUID } from 'node:crypto';
7
+ export class JobQueue {
8
+ deps;
9
+ /** 深度探测节流:watch 突发 800 文件时禁止每次 enqueue 都 COUNT(*) */
10
+ depthSampledAt = 0;
11
+ depthSample = 0;
12
+ constructor(deps) {
13
+ this.deps = deps;
14
+ }
15
+ sampleDepth() {
16
+ const now = Date.now();
17
+ if (now - this.depthSampledAt < 250)
18
+ return this.depthSample;
19
+ this.depthSampledAt = now;
20
+ const row = this.deps.db.raw
21
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
22
+ .get();
23
+ this.depthSample = row?.n ?? 0;
24
+ return this.depthSample;
25
+ }
26
+ hasQueuedJobs() {
27
+ return this.sampleDepth() > 0;
28
+ }
29
+ countQueuedKinds(kinds, sourceId) {
30
+ if (kinds.length === 0)
31
+ return 0;
32
+ const placeholders = kinds.map(() => '?').join(',');
33
+ const sql = sourceId
34
+ ? `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND source_id = ? AND kind IN (${placeholders})`
35
+ : `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`;
36
+ const row = this.deps.db.raw
37
+ .prepare(sql)
38
+ .get(...(sourceId ? [sourceId, ...kinds] : kinds));
39
+ return row?.n ?? 0;
40
+ }
41
+ countActiveJobs(sourceId) {
42
+ const row = this.deps.db.raw
43
+ .prepare(`SELECT
44
+ COUNT(*) AS total,
45
+ SUM(CASE WHEN kind = 'embed_source' THEN 1 ELSE 0 END) AS embed
46
+ FROM knowledge_jobs
47
+ WHERE source_id = ? AND status IN ('queued', 'running')`)
48
+ .get(sourceId);
49
+ return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
50
+ }
51
+ hasActivePathJob(sourceId, path) {
52
+ const row = this.deps.db.raw
53
+ .prepare(`SELECT id FROM knowledge_jobs
54
+ WHERE source_id = ? AND path = ? AND kind IN ('parse_file', 'fetch_doc') AND status IN ('queued','running')
55
+ LIMIT 1`)
56
+ .get(sourceId, path);
57
+ return Boolean(row?.id);
58
+ }
59
+ /**
60
+ * 入队(同 source+kind+path 去重;有 file_id 时按 (file_id,kind) 去重;中止源拒绝)
61
+ *
62
+ * @returns 是否真正插入
63
+ */
64
+ enqueue(sourceId, kind, path, priority, fileId) {
65
+ if (this.deps.isAborted(sourceId))
66
+ return false;
67
+ const activeStatuses = ['queued', 'running'];
68
+ const placeholders = activeStatuses.map(() => '?').join(',');
69
+ // 优先 (file_id, kind):共享 File 不双 parse
70
+ if (fileId) {
71
+ const byFile = this.deps.db.raw
72
+ .prepare(`SELECT id FROM knowledge_jobs
73
+ WHERE file_id = ? AND kind = ? AND status IN (${placeholders}) LIMIT 1`)
74
+ .get(fileId, kind, ...activeStatuses);
75
+ if (byFile?.id)
76
+ return false;
77
+ }
78
+ const dup = (path != null
79
+ ? this.deps.db.raw
80
+ .prepare(`SELECT id FROM knowledge_jobs
81
+ WHERE source_id = ? AND kind = ? AND path = ? AND status IN (${placeholders})`)
82
+ .get(sourceId, kind, path, ...activeStatuses)
83
+ : this.deps.db.raw
84
+ .prepare(`SELECT id FROM knowledge_jobs
85
+ WHERE source_id = ? AND kind = ? AND path IS NULL AND status IN (${placeholders})`)
86
+ .get(sourceId, kind, ...activeStatuses));
87
+ if (dup?.id)
88
+ return false;
89
+ const now = Date.now();
90
+ const depth = this.sampleDepth() + 1;
91
+ if (depth >= this.deps.maxQueueDepth) {
92
+ this.deps.onBackpressure?.(sourceId, depth);
93
+ }
94
+ this.deps.db.raw
95
+ .prepare(`INSERT INTO knowledge_jobs (id, source_id, file_id, kind, path, priority, status, attempts, created_at, updated_at)
96
+ VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?)`)
97
+ .run(`kj_${randomUUID().slice(0, 12)}`, sourceId, fileId ?? null, kind, path, priority, now, now);
98
+ return true;
99
+ }
100
+ /**
101
+ * 原子认领:queued → running;中止源不派发。
102
+ *
103
+ * @param kinds - 允许的 job 类型
104
+ * @param accept - 槽位/类型闸门;返回 false 则跳过该条继续找下一条(不永久占队)
105
+ */
106
+ claimJob(kinds, accept) {
107
+ if (kinds.length === 0)
108
+ return null;
109
+ const placeholders = kinds.map(() => '?').join(',');
110
+ const rows = this.deps.db.raw
111
+ .prepare(`SELECT * FROM knowledge_jobs
112
+ WHERE status = 'queued' AND kind IN (${placeholders})
113
+ AND source_id NOT IN (SELECT source_id FROM knowledge_source_control WHERE aborted = 1)
114
+ ORDER BY priority ASC, created_at ASC
115
+ LIMIT 32`)
116
+ .all(...kinds);
117
+ for (const row of rows) {
118
+ if (accept && !accept(row))
119
+ continue;
120
+ const res = this.deps.db.raw
121
+ .prepare(`UPDATE knowledge_jobs SET status = 'running', updated_at = ?
122
+ WHERE id = ? AND status = 'queued'`)
123
+ .run(Date.now(), row.id);
124
+ if (Number(res.changes ?? 0) === 0)
125
+ continue;
126
+ return row;
127
+ }
128
+ return null;
129
+ }
130
+ heartbeat(jobId) {
131
+ this.deps.db.raw
132
+ .prepare(`UPDATE knowledge_jobs SET updated_at = ? WHERE id = ? AND status = 'running'`)
133
+ .run(Date.now(), jobId);
134
+ }
135
+ markDone(jobId) {
136
+ this.deps.db.raw
137
+ .prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
138
+ .run(Date.now(), jobId);
139
+ }
140
+ markFailedOrCancelled(jobId, aborted, message) {
141
+ this.deps.db.raw
142
+ .prepare(`UPDATE knowledge_jobs
143
+ SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
144
+ WHERE id = ?`)
145
+ .run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, Date.now(), jobId);
146
+ }
147
+ /** 启动回收孤儿 running → queued */
148
+ reclaimOrphanRunning() {
149
+ this.deps.db.raw
150
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
151
+ WHERE status = 'running'`)
152
+ .run(Date.now());
153
+ }
154
+ /** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed */
155
+ reclaimStaleRunning(staleMs) {
156
+ const cutoff = Date.now() - staleMs;
157
+ this.deps.db.raw
158
+ .prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
159
+ WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
160
+ .run(Date.now(), cutoff);
161
+ this.deps.db.raw
162
+ .prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
163
+ WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
164
+ .run(Date.now(), cutoff);
165
+ }
166
+ cleanupTerminal(retainMs = 24 * 60 * 60_000) {
167
+ const cutoff = Date.now() - retainMs;
168
+ this.deps.db.raw
169
+ .prepare(`DELETE FROM knowledge_jobs
170
+ WHERE status IN ('done', 'failed', 'cancelled') AND updated_at < ?`)
171
+ .run(cutoff);
172
+ }
173
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,100 @@
1
+ /**
2
+ * Knowledge Service 独立子进程入口
3
+ *
4
+ * Gateway manageLocal 通过 fork 本文件拉起 Service:知识索引再重、再不稳,
5
+ * 也不得与 Gateway 共进程/共命运。本进程是 knowledge.db 唯一写者。
6
+ *
7
+ * 启动协议:argv[2] = boot JSON 路径;就绪后 process.send({ type: 'ready', port })。
8
+ * IPC 是 best-effort 通知:channel 断开时 HTTP 仍服务,不得因 send 崩进程。
9
+ */
10
+ import { readFile, unlink } from 'node:fs/promises';
11
+ import { startKnowledgeService } from './serve.js';
12
+ import { acquireKnowledgeWriterLock } from './writer-lock.js';
13
+ function sendToParent(msg) {
14
+ if (typeof process.send !== 'function')
15
+ return;
16
+ try {
17
+ process.send(msg, (err) => {
18
+ if (err) {
19
+ // 父进程已退出 / 通道已断:HTTP 仍可服务
20
+ }
21
+ });
22
+ }
23
+ catch {
24
+ // 同步 send 失败同样不致命
25
+ }
26
+ }
27
+ async function main() {
28
+ const bootPath = process.argv[2];
29
+ if (!bootPath) {
30
+ throw new Error('knowledge-serve-child: missing boot file path');
31
+ }
32
+ const raw = await readFile(bootPath, 'utf8');
33
+ const boot = JSON.parse(raw.replace(/^/, ''));
34
+ try {
35
+ await unlink(bootPath);
36
+ }
37
+ catch {
38
+ /* parent may already have cleaned up */
39
+ }
40
+ // 写者锁必须由真正打开 knowledge.db 的本进程持有
41
+ const lock = await acquireKnowledgeWriterLock(boot.dbPath);
42
+ const handle = await startKnowledgeService({
43
+ dbPath: boot.dbPath,
44
+ ...(boot.host != null ? { host: boot.host } : {}),
45
+ ...(boot.port != null ? { port: boot.port } : {}),
46
+ tokens: boot.tokens,
47
+ autoRegisterPrincipals: true,
48
+ documentConfig: boot.documentConfig ?? null,
49
+ embeddingModels: boot.embeddingModels ?? null,
50
+ embed: boot.embed ?? null,
51
+ ...(boot.sqliteVecExtensionPath
52
+ ? { sqliteVecExtensionPath: boot.sqliteVecExtensionPath }
53
+ : {}),
54
+ });
55
+ let closing = false;
56
+ const shutdown = async (code) => {
57
+ if (closing)
58
+ return;
59
+ closing = true;
60
+ try {
61
+ await handle.close();
62
+ }
63
+ catch {
64
+ /* best-effort */
65
+ }
66
+ try {
67
+ lock.release();
68
+ }
69
+ catch {
70
+ /* best-effort */
71
+ }
72
+ process.exit(code);
73
+ };
74
+ process.on('message', (msg) => {
75
+ if (msg?.type === 'close')
76
+ void shutdown(0);
77
+ });
78
+ process.on('SIGTERM', () => void shutdown(0));
79
+ process.on('SIGINT', () => void shutdown(0));
80
+ // process.send 失败以 'error' 事件冒出;不监听会 Unhandled 打崩本进程
81
+ process.on('error', () => {
82
+ /* IPC channel noise; service keeps listening */
83
+ });
84
+ if (typeof process.send === 'function') {
85
+ sendToParent({ type: 'ready', port: handle.port, pid: process.pid });
86
+ }
87
+ else {
88
+ console.log(`[Knowledge] serve listening port=${handle.port} pid=${process.pid}`);
89
+ }
90
+ }
91
+ main().catch((err) => {
92
+ const message = err instanceof Error ? err.message : String(err);
93
+ if (typeof process.send === 'function') {
94
+ sendToParent({ type: 'error', error: message });
95
+ }
96
+ else {
97
+ console.error(`[Knowledge] serve failed: ${message}`);
98
+ }
99
+ process.exit(1);
100
+ });
@@ -0,0 +1,27 @@
1
+ /**
2
+ * 测试 / :memory: 用:同进程 Write + Query 装配(非生产路径)。
3
+ */
4
+ import { KnowledgeDatabase } from './db.js';
5
+ import { LocalKnowledgeQueryService } from './query-service.js';
6
+ import { LocalKnowledgeWriteService } from './writer-service.js';
7
+ import { KnowledgeHttpApp, type KnowledgeServiceToken } from './http-app.js';
8
+ export interface LocalKnowledgeStack {
9
+ db: KnowledgeDatabase;
10
+ write: LocalKnowledgeWriteService;
11
+ query: LocalKnowledgeQueryService;
12
+ app: KnowledgeHttpApp;
13
+ dispose(): Promise<void>;
14
+ }
15
+ /**
16
+ * 同进程装配 HttpApp(单元测试)。生产走 engine-thread + Writer Worker。
17
+ *
18
+ * @param opts - db 路径 / token / autoRegister
19
+ * @returns 可关闭的 stack
20
+ */
21
+ export declare function createLocalKnowledgeStack(opts: {
22
+ dbPath?: string;
23
+ tokens: KnowledgeServiceToken[];
24
+ autoRegisterPrincipals?: boolean;
25
+ embeddingProvider?: import('../memory/sqlite/embedding.js').EmbeddingProvider | null;
26
+ testEmbeddingStub?: boolean;
27
+ }): Promise<LocalKnowledgeStack>;
@@ -0,0 +1,63 @@
1
+ /**
2
+ * 测试 / :memory: 用:同进程 Write + Query 装配(非生产路径)。
3
+ */
4
+ import { KnowledgeDatabase } from './db.js';
5
+ import { KnowledgeRetriever } from './retriever.js';
6
+ import { LocalKnowledgeQueryService } from './query-service.js';
7
+ import { LocalKnowledgeWriteService } from './writer-service.js';
8
+ import { createKnowledgeHttpApp, } from './http-app.js';
9
+ /**
10
+ * 同进程装配 HttpApp(单元测试)。生产走 engine-thread + Writer Worker。
11
+ *
12
+ * @param opts - db 路径 / token / autoRegister
13
+ * @returns 可关闭的 stack
14
+ */
15
+ export async function createLocalKnowledgeStack(opts) {
16
+ const db = await KnowledgeDatabase.create({ dbPath: opts.dbPath ?? ':memory:' });
17
+ const embeddingProvider = opts.testEmbeddingStub === true
18
+ ? {
19
+ name: 'test-stub',
20
+ dimensions: 4,
21
+ async embed() {
22
+ return [0.1, 0.2, 0.3, 0.4];
23
+ },
24
+ async embedBatch(texts) {
25
+ return texts.map(() => [0.1, 0.2, 0.3, 0.4]);
26
+ },
27
+ }
28
+ : (opts.embeddingProvider ?? null);
29
+ const write = new LocalKnowledgeWriteService({
30
+ db,
31
+ embeddingProvider: embeddingProvider,
32
+ });
33
+ const sources = write.sources;
34
+ const index = write.index;
35
+ const retriever = new KnowledgeRetriever({
36
+ sourceStore: sources,
37
+ indexStore: index,
38
+ embeddingProvider: embeddingProvider,
39
+ });
40
+ const query = new LocalKnowledgeQueryService({
41
+ db,
42
+ sources,
43
+ index,
44
+ retriever,
45
+ embeddingEnabled: Boolean(embeddingProvider),
46
+ });
47
+ const app = createKnowledgeHttpApp({
48
+ write,
49
+ query,
50
+ tokens: opts.tokens,
51
+ autoRegisterPrincipals: opts.autoRegisterPrincipals ?? true,
52
+ });
53
+ return {
54
+ db,
55
+ write,
56
+ query,
57
+ app,
58
+ dispose: async () => {
59
+ await app.dispose();
60
+ db.close();
61
+ },
62
+ };
63
+ }