@octopi-agent/engine 0.58.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +38 -1
- package/dist/config-schema/substrate.d.ts +37 -0
- package/dist/config-schema/substrate.js +47 -0
- package/dist/config.d.ts +30 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/index.d.ts +4 -2
- package/dist/harness/index.js +2 -1
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +80 -17
- package/dist/harness/knowledge/source-store.js +160 -89
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 源级 job 控制状态 — 纯 SQL,只读连接可跑。
|
|
3
|
+
*
|
|
4
|
+
* 与 KnowledgeIngest.jobControlState 同口径;禁止依赖 ingest 内存态
|
|
5
|
+
* (中止纪元以 knowledge_source_control 为权威)。
|
|
6
|
+
*/
|
|
7
|
+
import type { KnowledgeDatabase } from './db.js';
|
|
8
|
+
import type { KnowledgeIndexStore } from './index-store.js';
|
|
9
|
+
export interface KnowledgeJobControlState {
|
|
10
|
+
aborted: boolean;
|
|
11
|
+
jobsQueued: number;
|
|
12
|
+
jobsRunning: number;
|
|
13
|
+
jobsCancelled: number;
|
|
14
|
+
embedMissing: boolean;
|
|
15
|
+
canAbort: boolean;
|
|
16
|
+
canResume: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* 计算源的中止/继续控制面。
|
|
20
|
+
*
|
|
21
|
+
* @param db - knowledge 连接(只读亦可)
|
|
22
|
+
* @param index - 缺向量探测
|
|
23
|
+
* @param sourceId - 源 id
|
|
24
|
+
* @param opts.embeddingEnabled - 是否配置了 embedding(未配置则 embedMissing=false)
|
|
25
|
+
* @returns 控制按钮状态
|
|
26
|
+
*/
|
|
27
|
+
export declare function readJobControlState(db: KnowledgeDatabase, index: KnowledgeIndexStore, sourceId: string, opts?: {
|
|
28
|
+
embeddingEnabled?: boolean;
|
|
29
|
+
}): KnowledgeJobControlState;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 源级 job 控制状态 — 纯 SQL,只读连接可跑。
|
|
3
|
+
*
|
|
4
|
+
* 与 KnowledgeIngest.jobControlState 同口径;禁止依赖 ingest 内存态
|
|
5
|
+
* (中止纪元以 knowledge_source_control 为权威)。
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* 计算源的中止/继续控制面。
|
|
9
|
+
*
|
|
10
|
+
* @param db - knowledge 连接(只读亦可)
|
|
11
|
+
* @param index - 缺向量探测
|
|
12
|
+
* @param sourceId - 源 id
|
|
13
|
+
* @param opts.embeddingEnabled - 是否配置了 embedding(未配置则 embedMissing=false)
|
|
14
|
+
* @returns 控制按钮状态
|
|
15
|
+
*/
|
|
16
|
+
export function readJobControlState(db, index, sourceId, opts) {
|
|
17
|
+
const row = db.raw
|
|
18
|
+
.prepare(`SELECT
|
|
19
|
+
SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END) AS q,
|
|
20
|
+
SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) AS r,
|
|
21
|
+
SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS c
|
|
22
|
+
FROM knowledge_jobs WHERE source_id = ?`)
|
|
23
|
+
.get(sourceId);
|
|
24
|
+
const jobsQueued = Number(row?.q ?? 0);
|
|
25
|
+
const jobsRunning = Number(row?.r ?? 0);
|
|
26
|
+
const jobsCancelled = Number(row?.c ?? 0);
|
|
27
|
+
const abortedRow = db.raw
|
|
28
|
+
.prepare('SELECT aborted FROM knowledge_source_control WHERE source_id = ?')
|
|
29
|
+
.get(sourceId);
|
|
30
|
+
const aborted = Number(abortedRow?.aborted ?? 0) === 1;
|
|
31
|
+
const embedMissing = opts?.embeddingEnabled
|
|
32
|
+
? index.hasChunksMissingEmbedding(sourceId)
|
|
33
|
+
: false;
|
|
34
|
+
const active = jobsQueued + jobsRunning > 0;
|
|
35
|
+
return {
|
|
36
|
+
aborted,
|
|
37
|
+
jobsQueued,
|
|
38
|
+
jobsRunning,
|
|
39
|
+
jobsCancelled,
|
|
40
|
+
embedMissing,
|
|
41
|
+
canAbort: active && !aborted,
|
|
42
|
+
canResume: aborted || jobsCancelled > 0 || (embedMissing && !active),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeJobControl — 源级中止/继续纪元
|
|
3
|
+
*
|
|
4
|
+
* 中止态必须跨进程存活:DB 是权威,内存 AbortController 只是本进程信号。
|
|
5
|
+
* 调度前读库;`aborted` 则不派发新任务。
|
|
6
|
+
*/
|
|
7
|
+
import type { KnowledgeDatabase } from './db.js';
|
|
8
|
+
export interface AbortResumeStats {
|
|
9
|
+
cancelledQueued: number;
|
|
10
|
+
abortedRunning: number;
|
|
11
|
+
runningJobs: number;
|
|
12
|
+
restoredCancelled: number;
|
|
13
|
+
embedQueued: number;
|
|
14
|
+
}
|
|
15
|
+
export declare class KnowledgeJobControl {
|
|
16
|
+
private readonly db;
|
|
17
|
+
/** 本进程信号:abort 后 worker/embed 循环应立刻退出 */
|
|
18
|
+
private abortBySource;
|
|
19
|
+
constructor(db: KnowledgeDatabase);
|
|
20
|
+
/** 是否处于中止态(DB 权威;进程重启后仍生效)。连接已关时视为停止调度 */
|
|
21
|
+
isAborted(sourceId: string): boolean;
|
|
22
|
+
/** 本进程 AbortSignal(无记录时惰性创建未中止的纪元) */
|
|
23
|
+
signalFor(sourceId: string): AbortSignal;
|
|
24
|
+
/** 开启新的未中止纪元(resume / supersede 后用) */
|
|
25
|
+
beginEpoch(sourceId: string): void;
|
|
26
|
+
/**
|
|
27
|
+
* 中止:DB 落 aborted + 打断本进程信号。
|
|
28
|
+
* 不在此清 jobs(调用方负责 queued/running 收尾)。
|
|
29
|
+
*/
|
|
30
|
+
markAborted(sourceId: string): boolean;
|
|
31
|
+
/** 列出全部处于中止态的源(启动恢复 / 全局 abort 用) */
|
|
32
|
+
listAbortedSourceIds(): string[];
|
|
33
|
+
/** 调度闸门:中止源不得 claim 新任务 */
|
|
34
|
+
allowsDispatch(sourceId: string): boolean;
|
|
35
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KnowledgeJobControl — 源级中止/继续纪元
|
|
3
|
+
*
|
|
4
|
+
* 中止态必须跨进程存活:DB 是权威,内存 AbortController 只是本进程信号。
|
|
5
|
+
* 调度前读库;`aborted` 则不派发新任务。
|
|
6
|
+
*/
|
|
7
|
+
export class KnowledgeJobControl {
|
|
8
|
+
db;
|
|
9
|
+
/** 本进程信号:abort 后 worker/embed 循环应立刻退出 */
|
|
10
|
+
abortBySource = new Map();
|
|
11
|
+
constructor(db) {
|
|
12
|
+
this.db = db;
|
|
13
|
+
this.db.raw.exec(`
|
|
14
|
+
CREATE TABLE IF NOT EXISTS knowledge_source_control (
|
|
15
|
+
source_id TEXT PRIMARY KEY,
|
|
16
|
+
aborted INTEGER NOT NULL DEFAULT 0,
|
|
17
|
+
aborted_at INTEGER,
|
|
18
|
+
updated_at INTEGER NOT NULL
|
|
19
|
+
);
|
|
20
|
+
`);
|
|
21
|
+
}
|
|
22
|
+
/** 是否处于中止态(DB 权威;进程重启后仍生效)。连接已关时视为停止调度 */
|
|
23
|
+
isAborted(sourceId) {
|
|
24
|
+
try {
|
|
25
|
+
const row = this.db.raw
|
|
26
|
+
.prepare('SELECT aborted FROM knowledge_source_control WHERE source_id = ?')
|
|
27
|
+
.get(sourceId);
|
|
28
|
+
return Number(row?.aborted ?? 0) === 1;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** 本进程 AbortSignal(无记录时惰性创建未中止的纪元) */
|
|
35
|
+
signalFor(sourceId) {
|
|
36
|
+
let ac = this.abortBySource.get(sourceId);
|
|
37
|
+
if (!ac) {
|
|
38
|
+
ac = new AbortController();
|
|
39
|
+
this.abortBySource.set(sourceId, ac);
|
|
40
|
+
// 库里是 aborted 而内存还没有 → 补上信号,保证在跑任务能被打断
|
|
41
|
+
if (this.isAborted(sourceId) && !ac.signal.aborted)
|
|
42
|
+
ac.abort();
|
|
43
|
+
}
|
|
44
|
+
return ac.signal;
|
|
45
|
+
}
|
|
46
|
+
/** 开启新的未中止纪元(resume / supersede 后用) */
|
|
47
|
+
beginEpoch(sourceId) {
|
|
48
|
+
this.abortBySource.set(sourceId, new AbortController());
|
|
49
|
+
this.db.raw
|
|
50
|
+
.prepare(`INSERT INTO knowledge_source_control (source_id, aborted, aborted_at, updated_at)
|
|
51
|
+
VALUES (?, 0, NULL, ?)
|
|
52
|
+
ON CONFLICT(source_id) DO UPDATE SET aborted = 0, aborted_at = NULL, updated_at = excluded.updated_at`)
|
|
53
|
+
.run(sourceId, Date.now());
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* 中止:DB 落 aborted + 打断本进程信号。
|
|
57
|
+
* 不在此清 jobs(调用方负责 queued/running 收尾)。
|
|
58
|
+
*/
|
|
59
|
+
markAborted(sourceId) {
|
|
60
|
+
const already = this.isAborted(sourceId);
|
|
61
|
+
this.db.raw
|
|
62
|
+
.prepare(`INSERT INTO knowledge_source_control (source_id, aborted, aborted_at, updated_at)
|
|
63
|
+
VALUES (?, 1, ?, ?)
|
|
64
|
+
ON CONFLICT(source_id) DO UPDATE SET aborted = 1, aborted_at = excluded.aborted_at, updated_at = excluded.updated_at`)
|
|
65
|
+
.run(sourceId, Date.now(), Date.now());
|
|
66
|
+
let ctrl = this.abortBySource.get(sourceId);
|
|
67
|
+
if (!ctrl) {
|
|
68
|
+
ctrl = new AbortController();
|
|
69
|
+
this.abortBySource.set(sourceId, ctrl);
|
|
70
|
+
}
|
|
71
|
+
if (!ctrl.signal.aborted)
|
|
72
|
+
ctrl.abort();
|
|
73
|
+
return !already;
|
|
74
|
+
}
|
|
75
|
+
/** 列出全部处于中止态的源(启动恢复 / 全局 abort 用) */
|
|
76
|
+
listAbortedSourceIds() {
|
|
77
|
+
const rows = this.db.raw
|
|
78
|
+
.prepare('SELECT source_id FROM knowledge_source_control WHERE aborted = 1')
|
|
79
|
+
.all();
|
|
80
|
+
return rows.map((r) => r.source_id);
|
|
81
|
+
}
|
|
82
|
+
/** 调度闸门:中止源不得 claim 新任务 */
|
|
83
|
+
allowsDispatch(sourceId) {
|
|
84
|
+
return !this.isAborted(sourceId);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JobQueue — knowledge_jobs 表原语(enqueue / claim / 计数 / 回收 / 清理)
|
|
3
|
+
*
|
|
4
|
+
* 从 KnowledgeIngest 拆出;调度编排(drain/runJob)仍在 ingest。
|
|
5
|
+
*/
|
|
6
|
+
import type { KnowledgeDatabase } from './db.js';
|
|
7
|
+
export type IngestJobKind = 'parse_file' | 'walk_source' | 'drop_file' | 'embed_source' | 'fetch_doc';
|
|
8
|
+
export interface JobRow {
|
|
9
|
+
id: string;
|
|
10
|
+
source_id: string;
|
|
11
|
+
file_id: string | null;
|
|
12
|
+
kind: string;
|
|
13
|
+
path: string | null;
|
|
14
|
+
priority: number;
|
|
15
|
+
status: string;
|
|
16
|
+
attempts: number;
|
|
17
|
+
last_error: string | null;
|
|
18
|
+
created_at: number;
|
|
19
|
+
updated_at: number;
|
|
20
|
+
}
|
|
21
|
+
export interface JobQueueDeps {
|
|
22
|
+
db: KnowledgeDatabase;
|
|
23
|
+
maxQueueDepth: number;
|
|
24
|
+
isAborted: (sourceId: string) => boolean;
|
|
25
|
+
onBackpressure?: (sourceId: string, queued: number) => void;
|
|
26
|
+
}
|
|
27
|
+
export declare class JobQueue {
|
|
28
|
+
private readonly deps;
|
|
29
|
+
/** 深度探测节流:watch 突发 800 文件时禁止每次 enqueue 都 COUNT(*) */
|
|
30
|
+
private depthSampledAt;
|
|
31
|
+
private depthSample;
|
|
32
|
+
constructor(deps: JobQueueDeps);
|
|
33
|
+
private sampleDepth;
|
|
34
|
+
hasQueuedJobs(): boolean;
|
|
35
|
+
countQueuedKinds(kinds: IngestJobKind[], sourceId?: string): number;
|
|
36
|
+
countActiveJobs(sourceId: string): {
|
|
37
|
+
total: number;
|
|
38
|
+
embed: number;
|
|
39
|
+
};
|
|
40
|
+
hasActivePathJob(sourceId: string, path: string): boolean;
|
|
41
|
+
/**
|
|
42
|
+
* 入队(同 source+kind+path 去重;有 file_id 时按 (file_id,kind) 去重;中止源拒绝)
|
|
43
|
+
*
|
|
44
|
+
* @returns 是否真正插入
|
|
45
|
+
*/
|
|
46
|
+
enqueue(sourceId: string, kind: IngestJobKind, path: string | null, priority: number, fileId?: string | null): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* 原子认领:queued → running;中止源不派发。
|
|
49
|
+
*
|
|
50
|
+
* @param kinds - 允许的 job 类型
|
|
51
|
+
* @param accept - 槽位/类型闸门;返回 false 则跳过该条继续找下一条(不永久占队)
|
|
52
|
+
*/
|
|
53
|
+
claimJob(kinds: IngestJobKind[], accept?: (job: JobRow) => boolean): JobRow | null;
|
|
54
|
+
heartbeat(jobId: string): void;
|
|
55
|
+
markDone(jobId: string): void;
|
|
56
|
+
markFailedOrCancelled(jobId: string, aborted: boolean, message: string): void;
|
|
57
|
+
/** 启动回收孤儿 running → queued */
|
|
58
|
+
reclaimOrphanRunning(): void;
|
|
59
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed */
|
|
60
|
+
reclaimStaleRunning(staleMs: number): void;
|
|
61
|
+
cleanupTerminal(retainMs?: number): void;
|
|
62
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JobQueue — knowledge_jobs 表原语(enqueue / claim / 计数 / 回收 / 清理)
|
|
3
|
+
*
|
|
4
|
+
* 从 KnowledgeIngest 拆出;调度编排(drain/runJob)仍在 ingest。
|
|
5
|
+
*/
|
|
6
|
+
import { randomUUID } from 'node:crypto';
|
|
7
|
+
export class JobQueue {
|
|
8
|
+
deps;
|
|
9
|
+
/** 深度探测节流:watch 突发 800 文件时禁止每次 enqueue 都 COUNT(*) */
|
|
10
|
+
depthSampledAt = 0;
|
|
11
|
+
depthSample = 0;
|
|
12
|
+
constructor(deps) {
|
|
13
|
+
this.deps = deps;
|
|
14
|
+
}
|
|
15
|
+
sampleDepth() {
|
|
16
|
+
const now = Date.now();
|
|
17
|
+
if (now - this.depthSampledAt < 250)
|
|
18
|
+
return this.depthSample;
|
|
19
|
+
this.depthSampledAt = now;
|
|
20
|
+
const row = this.deps.db.raw
|
|
21
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued'`)
|
|
22
|
+
.get();
|
|
23
|
+
this.depthSample = row?.n ?? 0;
|
|
24
|
+
return this.depthSample;
|
|
25
|
+
}
|
|
26
|
+
hasQueuedJobs() {
|
|
27
|
+
return this.sampleDepth() > 0;
|
|
28
|
+
}
|
|
29
|
+
countQueuedKinds(kinds, sourceId) {
|
|
30
|
+
if (kinds.length === 0)
|
|
31
|
+
return 0;
|
|
32
|
+
const placeholders = kinds.map(() => '?').join(',');
|
|
33
|
+
const sql = sourceId
|
|
34
|
+
? `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND source_id = ? AND kind IN (${placeholders})`
|
|
35
|
+
: `SELECT COUNT(*) AS n FROM knowledge_jobs WHERE status = 'queued' AND kind IN (${placeholders})`;
|
|
36
|
+
const row = this.deps.db.raw
|
|
37
|
+
.prepare(sql)
|
|
38
|
+
.get(...(sourceId ? [sourceId, ...kinds] : kinds));
|
|
39
|
+
return row?.n ?? 0;
|
|
40
|
+
}
|
|
41
|
+
countActiveJobs(sourceId) {
|
|
42
|
+
const row = this.deps.db.raw
|
|
43
|
+
.prepare(`SELECT
|
|
44
|
+
COUNT(*) AS total,
|
|
45
|
+
SUM(CASE WHEN kind = 'embed_source' THEN 1 ELSE 0 END) AS embed
|
|
46
|
+
FROM knowledge_jobs
|
|
47
|
+
WHERE source_id = ? AND status IN ('queued', 'running')`)
|
|
48
|
+
.get(sourceId);
|
|
49
|
+
return { total: row?.total ?? 0, embed: row?.embed ?? 0 };
|
|
50
|
+
}
|
|
51
|
+
hasActivePathJob(sourceId, path) {
|
|
52
|
+
const row = this.deps.db.raw
|
|
53
|
+
.prepare(`SELECT id FROM knowledge_jobs
|
|
54
|
+
WHERE source_id = ? AND path = ? AND kind IN ('parse_file', 'fetch_doc') AND status IN ('queued','running')
|
|
55
|
+
LIMIT 1`)
|
|
56
|
+
.get(sourceId, path);
|
|
57
|
+
return Boolean(row?.id);
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* 入队(同 source+kind+path 去重;有 file_id 时按 (file_id,kind) 去重;中止源拒绝)
|
|
61
|
+
*
|
|
62
|
+
* @returns 是否真正插入
|
|
63
|
+
*/
|
|
64
|
+
enqueue(sourceId, kind, path, priority, fileId) {
|
|
65
|
+
if (this.deps.isAborted(sourceId))
|
|
66
|
+
return false;
|
|
67
|
+
const activeStatuses = ['queued', 'running'];
|
|
68
|
+
const placeholders = activeStatuses.map(() => '?').join(',');
|
|
69
|
+
// 优先 (file_id, kind):共享 File 不双 parse
|
|
70
|
+
if (fileId) {
|
|
71
|
+
const byFile = this.deps.db.raw
|
|
72
|
+
.prepare(`SELECT id FROM knowledge_jobs
|
|
73
|
+
WHERE file_id = ? AND kind = ? AND status IN (${placeholders}) LIMIT 1`)
|
|
74
|
+
.get(fileId, kind, ...activeStatuses);
|
|
75
|
+
if (byFile?.id)
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
const dup = (path != null
|
|
79
|
+
? this.deps.db.raw
|
|
80
|
+
.prepare(`SELECT id FROM knowledge_jobs
|
|
81
|
+
WHERE source_id = ? AND kind = ? AND path = ? AND status IN (${placeholders})`)
|
|
82
|
+
.get(sourceId, kind, path, ...activeStatuses)
|
|
83
|
+
: this.deps.db.raw
|
|
84
|
+
.prepare(`SELECT id FROM knowledge_jobs
|
|
85
|
+
WHERE source_id = ? AND kind = ? AND path IS NULL AND status IN (${placeholders})`)
|
|
86
|
+
.get(sourceId, kind, ...activeStatuses));
|
|
87
|
+
if (dup?.id)
|
|
88
|
+
return false;
|
|
89
|
+
const now = Date.now();
|
|
90
|
+
const depth = this.sampleDepth() + 1;
|
|
91
|
+
if (depth >= this.deps.maxQueueDepth) {
|
|
92
|
+
this.deps.onBackpressure?.(sourceId, depth);
|
|
93
|
+
}
|
|
94
|
+
this.deps.db.raw
|
|
95
|
+
.prepare(`INSERT INTO knowledge_jobs (id, source_id, file_id, kind, path, priority, status, attempts, created_at, updated_at)
|
|
96
|
+
VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, ?, ?)`)
|
|
97
|
+
.run(`kj_${randomUUID().slice(0, 12)}`, sourceId, fileId ?? null, kind, path, priority, now, now);
|
|
98
|
+
return true;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* 原子认领:queued → running;中止源不派发。
|
|
102
|
+
*
|
|
103
|
+
* @param kinds - 允许的 job 类型
|
|
104
|
+
* @param accept - 槽位/类型闸门;返回 false 则跳过该条继续找下一条(不永久占队)
|
|
105
|
+
*/
|
|
106
|
+
claimJob(kinds, accept) {
|
|
107
|
+
if (kinds.length === 0)
|
|
108
|
+
return null;
|
|
109
|
+
const placeholders = kinds.map(() => '?').join(',');
|
|
110
|
+
const rows = this.deps.db.raw
|
|
111
|
+
.prepare(`SELECT * FROM knowledge_jobs
|
|
112
|
+
WHERE status = 'queued' AND kind IN (${placeholders})
|
|
113
|
+
AND source_id NOT IN (SELECT source_id FROM knowledge_source_control WHERE aborted = 1)
|
|
114
|
+
ORDER BY priority ASC, created_at ASC
|
|
115
|
+
LIMIT 32`)
|
|
116
|
+
.all(...kinds);
|
|
117
|
+
for (const row of rows) {
|
|
118
|
+
if (accept && !accept(row))
|
|
119
|
+
continue;
|
|
120
|
+
const res = this.deps.db.raw
|
|
121
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'running', updated_at = ?
|
|
122
|
+
WHERE id = ? AND status = 'queued'`)
|
|
123
|
+
.run(Date.now(), row.id);
|
|
124
|
+
if (Number(res.changes ?? 0) === 0)
|
|
125
|
+
continue;
|
|
126
|
+
return row;
|
|
127
|
+
}
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
heartbeat(jobId) {
|
|
131
|
+
this.deps.db.raw
|
|
132
|
+
.prepare(`UPDATE knowledge_jobs SET updated_at = ? WHERE id = ? AND status = 'running'`)
|
|
133
|
+
.run(Date.now(), jobId);
|
|
134
|
+
}
|
|
135
|
+
markDone(jobId) {
|
|
136
|
+
this.deps.db.raw
|
|
137
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'done', updated_at = ? WHERE id = ?`)
|
|
138
|
+
.run(Date.now(), jobId);
|
|
139
|
+
}
|
|
140
|
+
markFailedOrCancelled(jobId, aborted, message) {
|
|
141
|
+
this.deps.db.raw
|
|
142
|
+
.prepare(`UPDATE knowledge_jobs
|
|
143
|
+
SET status = ?, attempts = attempts + 1, last_error = ?, updated_at = ?
|
|
144
|
+
WHERE id = ?`)
|
|
145
|
+
.run(aborted ? 'cancelled' : 'failed', aborted ? 'aborted' : message, Date.now(), jobId);
|
|
146
|
+
}
|
|
147
|
+
/** 启动回收孤儿 running → queued */
|
|
148
|
+
reclaimOrphanRunning() {
|
|
149
|
+
this.deps.db.raw
|
|
150
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', updated_at = ?
|
|
151
|
+
WHERE status = 'running'`)
|
|
152
|
+
.run(Date.now());
|
|
153
|
+
}
|
|
154
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed */
|
|
155
|
+
reclaimStaleRunning(staleMs) {
|
|
156
|
+
const cutoff = Date.now() - staleMs;
|
|
157
|
+
this.deps.db.raw
|
|
158
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'failed', last_error = COALESCE(last_error, 'stale_give_up'), updated_at = ?
|
|
159
|
+
WHERE status = 'running' AND updated_at < ? AND attempts >= 2`)
|
|
160
|
+
.run(Date.now(), cutoff);
|
|
161
|
+
this.deps.db.raw
|
|
162
|
+
.prepare(`UPDATE knowledge_jobs SET status = 'queued', attempts = attempts + 1, updated_at = ?
|
|
163
|
+
WHERE status = 'running' AND updated_at < ? AND attempts < 2`)
|
|
164
|
+
.run(Date.now(), cutoff);
|
|
165
|
+
}
|
|
166
|
+
cleanupTerminal(retainMs = 24 * 60 * 60_000) {
|
|
167
|
+
const cutoff = Date.now() - retainMs;
|
|
168
|
+
this.deps.db.raw
|
|
169
|
+
.prepare(`DELETE FROM knowledge_jobs
|
|
170
|
+
WHERE status IN ('done', 'failed', 'cancelled') AND updated_at < ?`)
|
|
171
|
+
.run(cutoff);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Knowledge Service 独立子进程入口
|
|
3
|
+
*
|
|
4
|
+
* Gateway manageLocal 通过 fork 本文件拉起 Service:知识索引再重、再不稳,
|
|
5
|
+
* 也不得与 Gateway 共进程/共命运。本进程是 knowledge.db 唯一写者。
|
|
6
|
+
*
|
|
7
|
+
* 启动协议:argv[2] = boot JSON 路径;就绪后 process.send({ type: 'ready', port })。
|
|
8
|
+
* IPC 是 best-effort 通知:channel 断开时 HTTP 仍服务,不得因 send 崩进程。
|
|
9
|
+
*/
|
|
10
|
+
import { readFile, unlink } from 'node:fs/promises';
|
|
11
|
+
import { startKnowledgeService } from './serve.js';
|
|
12
|
+
import { acquireKnowledgeWriterLock } from './writer-lock.js';
|
|
13
|
+
function sendToParent(msg) {
|
|
14
|
+
if (typeof process.send !== 'function')
|
|
15
|
+
return;
|
|
16
|
+
try {
|
|
17
|
+
process.send(msg, (err) => {
|
|
18
|
+
if (err) {
|
|
19
|
+
// 父进程已退出 / 通道已断:HTTP 仍可服务
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
// 同步 send 失败同样不致命
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async function main() {
|
|
28
|
+
const bootPath = process.argv[2];
|
|
29
|
+
if (!bootPath) {
|
|
30
|
+
throw new Error('knowledge-serve-child: missing boot file path');
|
|
31
|
+
}
|
|
32
|
+
const raw = await readFile(bootPath, 'utf8');
|
|
33
|
+
const boot = JSON.parse(raw.replace(/^/, ''));
|
|
34
|
+
try {
|
|
35
|
+
await unlink(bootPath);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
/* parent may already have cleaned up */
|
|
39
|
+
}
|
|
40
|
+
// 写者锁必须由真正打开 knowledge.db 的本进程持有
|
|
41
|
+
const lock = await acquireKnowledgeWriterLock(boot.dbPath);
|
|
42
|
+
const handle = await startKnowledgeService({
|
|
43
|
+
dbPath: boot.dbPath,
|
|
44
|
+
...(boot.host != null ? { host: boot.host } : {}),
|
|
45
|
+
...(boot.port != null ? { port: boot.port } : {}),
|
|
46
|
+
tokens: boot.tokens,
|
|
47
|
+
autoRegisterPrincipals: true,
|
|
48
|
+
documentConfig: boot.documentConfig ?? null,
|
|
49
|
+
embeddingModels: boot.embeddingModels ?? null,
|
|
50
|
+
embed: boot.embed ?? null,
|
|
51
|
+
...(boot.sqliteVecExtensionPath
|
|
52
|
+
? { sqliteVecExtensionPath: boot.sqliteVecExtensionPath }
|
|
53
|
+
: {}),
|
|
54
|
+
});
|
|
55
|
+
let closing = false;
|
|
56
|
+
const shutdown = async (code) => {
|
|
57
|
+
if (closing)
|
|
58
|
+
return;
|
|
59
|
+
closing = true;
|
|
60
|
+
try {
|
|
61
|
+
await handle.close();
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
/* best-effort */
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
lock.release();
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
/* best-effort */
|
|
71
|
+
}
|
|
72
|
+
process.exit(code);
|
|
73
|
+
};
|
|
74
|
+
process.on('message', (msg) => {
|
|
75
|
+
if (msg?.type === 'close')
|
|
76
|
+
void shutdown(0);
|
|
77
|
+
});
|
|
78
|
+
process.on('SIGTERM', () => void shutdown(0));
|
|
79
|
+
process.on('SIGINT', () => void shutdown(0));
|
|
80
|
+
// process.send 失败以 'error' 事件冒出;不监听会 Unhandled 打崩本进程
|
|
81
|
+
process.on('error', () => {
|
|
82
|
+
/* IPC channel noise; service keeps listening */
|
|
83
|
+
});
|
|
84
|
+
if (typeof process.send === 'function') {
|
|
85
|
+
sendToParent({ type: 'ready', port: handle.port, pid: process.pid });
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
console.log(`[Knowledge] serve listening port=${handle.port} pid=${process.pid}`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
main().catch((err) => {
|
|
92
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
93
|
+
if (typeof process.send === 'function') {
|
|
94
|
+
sendToParent({ type: 'error', error: message });
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
console.error(`[Knowledge] serve failed: ${message}`);
|
|
98
|
+
}
|
|
99
|
+
process.exit(1);
|
|
100
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 测试 / :memory: 用:同进程 Write + Query 装配(非生产路径)。
|
|
3
|
+
*/
|
|
4
|
+
import { KnowledgeDatabase } from './db.js';
|
|
5
|
+
import { LocalKnowledgeQueryService } from './query-service.js';
|
|
6
|
+
import { LocalKnowledgeWriteService } from './writer-service.js';
|
|
7
|
+
import { KnowledgeHttpApp, type KnowledgeServiceToken } from './http-app.js';
|
|
8
|
+
export interface LocalKnowledgeStack {
|
|
9
|
+
db: KnowledgeDatabase;
|
|
10
|
+
write: LocalKnowledgeWriteService;
|
|
11
|
+
query: LocalKnowledgeQueryService;
|
|
12
|
+
app: KnowledgeHttpApp;
|
|
13
|
+
dispose(): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* 同进程装配 HttpApp(单元测试)。生产走 engine-thread + Writer Worker。
|
|
17
|
+
*
|
|
18
|
+
* @param opts - db 路径 / token / autoRegister
|
|
19
|
+
* @returns 可关闭的 stack
|
|
20
|
+
*/
|
|
21
|
+
export declare function createLocalKnowledgeStack(opts: {
|
|
22
|
+
dbPath?: string;
|
|
23
|
+
tokens: KnowledgeServiceToken[];
|
|
24
|
+
autoRegisterPrincipals?: boolean;
|
|
25
|
+
embeddingProvider?: import('../memory/sqlite/embedding.js').EmbeddingProvider | null;
|
|
26
|
+
testEmbeddingStub?: boolean;
|
|
27
|
+
}): Promise<LocalKnowledgeStack>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 测试 / :memory: 用:同进程 Write + Query 装配(非生产路径)。
|
|
3
|
+
*/
|
|
4
|
+
import { KnowledgeDatabase } from './db.js';
|
|
5
|
+
import { KnowledgeRetriever } from './retriever.js';
|
|
6
|
+
import { LocalKnowledgeQueryService } from './query-service.js';
|
|
7
|
+
import { LocalKnowledgeWriteService } from './writer-service.js';
|
|
8
|
+
import { createKnowledgeHttpApp, } from './http-app.js';
|
|
9
|
+
/**
|
|
10
|
+
* 同进程装配 HttpApp(单元测试)。生产走 engine-thread + Writer Worker。
|
|
11
|
+
*
|
|
12
|
+
* @param opts - db 路径 / token / autoRegister
|
|
13
|
+
* @returns 可关闭的 stack
|
|
14
|
+
*/
|
|
15
|
+
export async function createLocalKnowledgeStack(opts) {
|
|
16
|
+
const db = await KnowledgeDatabase.create({ dbPath: opts.dbPath ?? ':memory:' });
|
|
17
|
+
const embeddingProvider = opts.testEmbeddingStub === true
|
|
18
|
+
? {
|
|
19
|
+
name: 'test-stub',
|
|
20
|
+
dimensions: 4,
|
|
21
|
+
async embed() {
|
|
22
|
+
return [0.1, 0.2, 0.3, 0.4];
|
|
23
|
+
},
|
|
24
|
+
async embedBatch(texts) {
|
|
25
|
+
return texts.map(() => [0.1, 0.2, 0.3, 0.4]);
|
|
26
|
+
},
|
|
27
|
+
}
|
|
28
|
+
: (opts.embeddingProvider ?? null);
|
|
29
|
+
const write = new LocalKnowledgeWriteService({
|
|
30
|
+
db,
|
|
31
|
+
embeddingProvider: embeddingProvider,
|
|
32
|
+
});
|
|
33
|
+
const sources = write.sources;
|
|
34
|
+
const index = write.index;
|
|
35
|
+
const retriever = new KnowledgeRetriever({
|
|
36
|
+
sourceStore: sources,
|
|
37
|
+
indexStore: index,
|
|
38
|
+
embeddingProvider: embeddingProvider,
|
|
39
|
+
});
|
|
40
|
+
const query = new LocalKnowledgeQueryService({
|
|
41
|
+
db,
|
|
42
|
+
sources,
|
|
43
|
+
index,
|
|
44
|
+
retriever,
|
|
45
|
+
embeddingEnabled: Boolean(embeddingProvider),
|
|
46
|
+
});
|
|
47
|
+
const app = createKnowledgeHttpApp({
|
|
48
|
+
write,
|
|
49
|
+
query,
|
|
50
|
+
tokens: opts.tokens,
|
|
51
|
+
autoRegisterPrincipals: opts.autoRegisterPrincipals ?? true,
|
|
52
|
+
});
|
|
53
|
+
return {
|
|
54
|
+
db,
|
|
55
|
+
write,
|
|
56
|
+
query,
|
|
57
|
+
app,
|
|
58
|
+
dispose: async () => {
|
|
59
|
+
await app.dispose();
|
|
60
|
+
db.close();
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|