@octopi-agent/engine 0.58.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +38 -1
  4. package/dist/config-schema/substrate.d.ts +37 -0
  5. package/dist/config-schema/substrate.js +47 -0
  6. package/dist/config.d.ts +30 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  23. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  24. package/dist/harness/capabilities/document/factory.js +37 -0
  25. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  37. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  39. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  40. package/dist/harness/capabilities/document/worker-port.js +78 -0
  41. package/dist/harness/context/capabilities/index.d.ts +2 -2
  42. package/dist/harness/context/capabilities/index.js +2 -2
  43. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  44. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  45. package/dist/harness/index.d.ts +4 -2
  46. package/dist/harness/index.js +2 -1
  47. package/dist/harness/knowledge/adapters.d.ts +8 -5
  48. package/dist/harness/knowledge/adapters.js +53 -13
  49. package/dist/harness/knowledge/client.d.ts +197 -0
  50. package/dist/harness/knowledge/client.js +237 -0
  51. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  52. package/dist/harness/knowledge/connectors.d.ts +3 -3
  53. package/dist/harness/knowledge/connectors.js +14 -8
  54. package/dist/harness/knowledge/db.d.ts +40 -5
  55. package/dist/harness/knowledge/db.js +441 -113
  56. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  57. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  58. package/dist/harness/knowledge/embed-runner.js +171 -0
  59. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  60. package/dist/harness/knowledge/engine-thread.js +302 -0
  61. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  62. package/dist/harness/knowledge/extract-document.js +32 -8
  63. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  64. package/dist/harness/knowledge/fetchers.js +52 -16
  65. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  66. package/dist/harness/knowledge/file-identity.js +172 -0
  67. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  68. package/dist/harness/knowledge/file-index-store.js +132 -0
  69. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  70. package/dist/harness/knowledge/file-limits.js +175 -0
  71. package/dist/harness/knowledge/fts.d.ts +60 -0
  72. package/dist/harness/knowledge/fts.js +218 -0
  73. package/dist/harness/knowledge/grounding.d.ts +3 -2
  74. package/dist/harness/knowledge/http-app.d.ts +50 -0
  75. package/dist/harness/knowledge/http-app.js +652 -0
  76. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  77. package/dist/harness/knowledge/http-bridge.js +17 -0
  78. package/dist/harness/knowledge/index-store.d.ts +158 -93
  79. package/dist/harness/knowledge/index-store.js +982 -441
  80. package/dist/harness/knowledge/index.d.ts +45 -5
  81. package/dist/harness/knowledge/index.js +23 -2
  82. package/dist/harness/knowledge/ingest.d.ts +262 -87
  83. package/dist/harness/knowledge/ingest.js +1072 -480
  84. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  85. package/dist/harness/knowledge/job-control-state.js +44 -0
  86. package/dist/harness/knowledge/job-control.d.ts +35 -0
  87. package/dist/harness/knowledge/job-control.js +86 -0
  88. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  89. package/dist/harness/knowledge/job-queue.js +173 -0
  90. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  91. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  92. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  93. package/dist/harness/knowledge/local-stack.js +63 -0
  94. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  95. package/dist/harness/knowledge/membership-store.js +198 -0
  96. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  97. package/dist/harness/knowledge/network-guard.js +130 -85
  98. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  99. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  100. package/dist/harness/knowledge/query-service.d.ts +272 -0
  101. package/dist/harness/knowledge/query-service.js +326 -0
  102. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  103. package/dist/harness/knowledge/query-worker-client.js +207 -0
  104. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  105. package/dist/harness/knowledge/query-worker.js +186 -0
  106. package/dist/harness/knowledge/read-http.d.ts +35 -0
  107. package/dist/harness/knowledge/read-http.js +373 -0
  108. package/dist/harness/knowledge/retriever.d.ts +22 -5
  109. package/dist/harness/knowledge/retriever.js +66 -35
  110. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  111. package/dist/harness/knowledge/secret-scan.js +25 -2
  112. package/dist/harness/knowledge/serve.d.ts +48 -0
  113. package/dist/harness/knowledge/serve.js +336 -0
  114. package/dist/harness/knowledge/source-store.d.ts +80 -17
  115. package/dist/harness/knowledge/source-store.js +160 -89
  116. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  117. package/dist/harness/knowledge/start-service-process.js +158 -0
  118. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  119. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  120. package/dist/harness/knowledge/types.d.ts +1 -1
  121. package/dist/harness/knowledge/types.js +1 -1
  122. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  123. package/dist/harness/knowledge/vector-ann.js +68 -0
  124. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  125. package/dist/harness/knowledge/writer-lock.js +71 -0
  126. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  127. package/dist/harness/knowledge/writer-service.js +355 -0
  128. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  129. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  130. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  131. package/dist/harness/knowledge/writer-worker.js +189 -0
  132. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  133. package/dist/harness/memory/sqlite/embedding.js +109 -16
  134. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  135. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  136. package/dist/harness/session/attachments/service.d.ts +1 -1
  137. package/dist/harness/session/attachments/service.js +1 -1
  138. package/dist/index.d.ts +4 -2
  139. package/dist/index.js +2 -1
  140. package/dist/worker-path.d.ts +8 -0
  141. package/dist/worker-path.js +15 -0
  142. package/package.json +2 -2
@@ -0,0 +1,132 @@
1
+ /**
2
+ * FileIndexStore — File 本位索引投影(chunks 挂 file_id;path 在 Membership)
3
+ */
4
+ import { randomUUID } from 'node:crypto';
5
+ import { KnowledgeFts } from './fts.js';
6
+ export class FileIndexStore {
7
+ db;
8
+ fts;
9
+ constructor(db) {
10
+ this.db = db;
11
+ this.fts = new KnowledgeFts(db);
12
+ }
13
+ /**
14
+ * 覆盖写入该 File 的全部 chunk(原地更新 / 首次解析)。
15
+ * 不触碰 memberships。
16
+ */
17
+ upsertFileContent(input) {
18
+ const now = Date.now();
19
+ this.db.raw.exec('BEGIN');
20
+ try {
21
+ const oldIds = this.db.raw
22
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
23
+ .all(input.fileId).map((r) => r.id);
24
+ this.dropChunks(oldIds);
25
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(input.fileId);
26
+ this.db.raw
27
+ .prepare(`UPDATE knowledge_files
28
+ SET content_hash = ?, size = ?, mtime = ?, adapter_id = ?,
29
+ status = 'indexed', error = NULL, chunk_count = ?, indexed_at = ?
30
+ WHERE id = ?`)
31
+ .run(input.contentHash ?? null, input.size, input.mtime, input.adapterId ?? null, input.chunks.length, now, input.fileId);
32
+ const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, ordinal, text, start_line, end_line)
33
+ VALUES (?, ?, ?, ?, ?, ?)`);
34
+ const ftsRows = [];
35
+ for (const c of input.chunks) {
36
+ const cid = `kc_${randomUUID().slice(0, 12)}`;
37
+ insertChunk.run(cid, input.fileId, c.ordinal, c.text, c.startLine, c.endLine);
38
+ ftsRows.push({ id: cid, text: c.text, path: '' });
39
+ }
40
+ this.fts.upsertMany(ftsRows);
41
+ this.db.raw.exec('COMMIT');
42
+ return { fileId: input.fileId, chunkCount: input.chunks.length, indexedAt: now };
43
+ }
44
+ catch (err) {
45
+ try {
46
+ this.db.raw.exec('ROLLBACK');
47
+ }
48
+ catch {
49
+ // 以原异常为准
50
+ }
51
+ throw err;
52
+ }
53
+ }
54
+ markFileStatus(fileId, status, detail) {
55
+ this.db.raw
56
+ .prepare(`UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?`)
57
+ .run(status, detail ?? null, Date.now(), fileId);
58
+ }
59
+ /** purge File + chunks + embeddings + FTS(membership 已清) */
60
+ purgeFile(fileId) {
61
+ const oldIds = this.db.raw
62
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
63
+ .all(fileId).map((r) => r.id);
64
+ this.dropChunks(oldIds);
65
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
66
+ this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
67
+ }
68
+ dropChunks(chunkIds) {
69
+ if (chunkIds.length === 0)
70
+ return;
71
+ const ph = chunkIds.map(() => '?').join(',');
72
+ this.db.raw
73
+ .prepare(`DELETE FROM knowledge_chunk_embeddings WHERE chunk_id IN (${ph})`)
74
+ .run(...chunkIds);
75
+ this.deleteKnowledgeVec(chunkIds);
76
+ this.fts.removeMany(chunkIds);
77
+ }
78
+ deleteKnowledgeVec(chunkIds) {
79
+ if (!this.db.sqliteVecEnabled || chunkIds.length === 0)
80
+ return;
81
+ try {
82
+ this.db.raw
83
+ .prepare(`DELETE FROM knowledge_vec_chunks WHERE rowid IN
84
+ (SELECT rowid FROM knowledge_chunks WHERE id IN (${chunkIds.map(() => '?').join(',')}))`)
85
+ .run(...chunkIds);
86
+ }
87
+ catch {
88
+ // 无 vec 表时忽略
89
+ }
90
+ }
91
+ /**
92
+ * 按可见 source 集合做关键词候选(chunk 去重;path 取 membership)。
93
+ */
94
+ searchKeywordVisible(query, visibleSourceIds, limit) {
95
+ if (visibleSourceIds.length === 0 || limit <= 0)
96
+ return [];
97
+ const srcPh = visibleSourceIds.map(() => '?').join(',');
98
+ // LIKE 预筛 + FTS 可选;此处先 LIKE(FTS 路径可后接)
99
+ const like = `%${query.replace(/[%_\\]/g, (c) => `\\${c}`)}%`;
100
+ const rows = this.db.raw
101
+ .prepare(`SELECT c.id AS chunk_id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
102
+ m.source_id AS source_id, m.logical_path AS logical_path
103
+ FROM knowledge_chunks c
104
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
105
+ WHERE m.source_id IN (${srcPh})
106
+ AND c.text LIKE ? ESCAPE '\\'
107
+ LIMIT ?`)
108
+ .all(...visibleSourceIds, like, limit * 4);
109
+ const byChunk = new Map();
110
+ for (const r of rows) {
111
+ let hit = byChunk.get(r.chunk_id);
112
+ if (!hit) {
113
+ if (byChunk.size >= limit)
114
+ continue;
115
+ hit = {
116
+ chunkId: r.chunk_id,
117
+ fileId: r.file_id,
118
+ sourceIds: [],
119
+ path: r.logical_path,
120
+ ordinal: r.ordinal,
121
+ text: r.text,
122
+ startLine: r.start_line,
123
+ endLine: r.end_line,
124
+ };
125
+ byChunk.set(r.chunk_id, hit);
126
+ }
127
+ if (!hit.sourceIds.includes(r.source_id))
128
+ hit.sourceIds.push(r.source_id);
129
+ }
130
+ return [...byChunk.values()];
131
+ }
132
+ }
@@ -0,0 +1,86 @@
1
+ /**
2
+ * Knowledge 索引文件限额 — 按格式分级上限 + 部分抽取
3
+ *
4
+ * 业务原则:大文件优先 **部分入索引** 而非整文件 skip;
5
+ * 压缩包等容器不抽;媒体走「无适配器」默认跳过(后续可加适配器)。
6
+ *
7
+ * @module
8
+ */
9
+ export type KnowledgeFileKind = 'text' | 'pdf' | 'officeDoc' | 'sheet' | 'other';
10
+ export type OversizePolicy = 'partial' | 'skip';
11
+ export interface KnowledgeFileLimits {
12
+ /** 超过格式上限时:partial=截断入索引(默认);skip=整文件跳过 */
13
+ oversize: OversizePolicy;
14
+ /** 未归类 / 兜底单文件上限 */
15
+ maxFileBytes: number;
16
+ /** 绝对内存闸门(partial 也拒绝);防止 worker 被超大文件打爆 */
17
+ hardMaxFileBytes: number;
18
+ maxBytes: {
19
+ text: number;
20
+ pdf: number;
21
+ officeDoc: number;
22
+ sheet: number;
23
+ };
24
+ partial: {
25
+ maxSheets: number;
26
+ maxRowsPerSheet: number;
27
+ maxPdfPages: number;
28
+ maxTextChars: number;
29
+ };
30
+ /** parse 基超时 */
31
+ parseTimeoutMs: number;
32
+ /** 每 MB 追加超时 */
33
+ parseTimeoutPerMBMs: number;
34
+ maxParseTimeoutMs: number;
35
+ }
36
+ export type KnowledgeFileLimitsInput = {
37
+ oversize?: OversizePolicy;
38
+ maxFileBytes?: number;
39
+ hardMaxFileBytes?: number;
40
+ maxBytes?: Partial<KnowledgeFileLimits['maxBytes']>;
41
+ partial?: Partial<KnowledgeFileLimits['partial']>;
42
+ parseTimeoutMs?: number;
43
+ parseTimeoutPerMBMs?: number;
44
+ maxParseTimeoutMs?: number;
45
+ };
46
+ /**
47
+ * 推荐缺省(可按机器资源在 knowledge.index.files 覆盖)。
48
+ *
49
+ * - pptx/docx 体积多为图片,文本抽取可放宽到 200MB
50
+ * - xlsx 全量 SheetJS 进内存,50MB 起步;超限走 partial
51
+ * - pdf 50MB;text 20MB
52
+ * - hardMax 256MB:partial 尝试的绝对上限
53
+ */
54
+ export declare const DEFAULT_KNOWLEDGE_FILE_LIMITS: KnowledgeFileLimits;
55
+ /** 按扩展名归类(媒体/压缩包等归 other → 无适配器时跳过) */
56
+ export declare function classifyFileKind(filePath: string): KnowledgeFileKind;
57
+ export declare function resolveKnowledgeFileLimits(input?: KnowledgeFileLimitsInput): KnowledgeFileLimits;
58
+ /** 该格式的单文件上限 */
59
+ export declare function formatMaxBytes(limits: KnowledgeFileLimits, kind: KnowledgeFileKind): number;
60
+ export type SizeDecision = {
61
+ action: 'ok';
62
+ } | {
63
+ action: 'partial';
64
+ reason: 'oversize_soft';
65
+ } | {
66
+ action: 'skip';
67
+ reason: 'oversize_hard' | 'oversize_skip';
68
+ };
69
+ /**
70
+ * 按尺寸裁决:ok / partial / skip
71
+ *
72
+ * @param size - 文件字节数
73
+ * @param kind - 格式类别
74
+ * @param limits - 已解析限额
75
+ */
76
+ export declare function decideBySize(size: number, kind: KnowledgeFileKind, limits: KnowledgeFileLimits): SizeDecision;
77
+ /** parse 超时随体积放大(封顶 maxParseTimeoutMs) */
78
+ export declare function parseTimeoutForSize(size: number, limits: KnowledgeFileLimits): number;
79
+ /**
80
+ * skip 是否值得再解析(配置调宽 / 新策略后重试)。
81
+ *
82
+ * - `ignored_path` / `no_adapter`:压缩包与媒体等,无适配器前不重试
83
+ * - `oversize*`:可重试;`oversize_hard` 仅当 size 已落入当前 hardMax(或 size 未知)
84
+ * - `empty_content`:可重试
85
+ */
86
+ export declare function isRetryableSkipReason(reason: string | null | undefined, size: number, limits: KnowledgeFileLimits): boolean;
@@ -0,0 +1,175 @@
1
+ /**
2
+ * Knowledge 索引文件限额 — 按格式分级上限 + 部分抽取
3
+ *
4
+ * 业务原则:大文件优先 **部分入索引** 而非整文件 skip;
5
+ * 压缩包等容器不抽;媒体走「无适配器」默认跳过(后续可加适配器)。
6
+ *
7
+ * @module
8
+ */
9
+ import { extname } from 'node:path';
10
+ /**
11
+ * 推荐缺省(可按机器资源在 knowledge.index.files 覆盖)。
12
+ *
13
+ * - pptx/docx 体积多为图片,文本抽取可放宽到 200MB
14
+ * - xlsx 全量 SheetJS 进内存,50MB 起步;超限走 partial
15
+ * - pdf 50MB;text 20MB
16
+ * - hardMax 256MB:partial 尝试的绝对上限
17
+ */
18
+ export const DEFAULT_KNOWLEDGE_FILE_LIMITS = {
19
+ oversize: 'partial',
20
+ maxFileBytes: 50_000_000,
21
+ hardMaxFileBytes: 256 * 1024 * 1024,
22
+ maxBytes: {
23
+ text: 20_000_000,
24
+ pdf: 50_000_000,
25
+ officeDoc: 200_000_000,
26
+ sheet: 50_000_000,
27
+ },
28
+ partial: {
29
+ maxSheets: 20,
30
+ maxRowsPerSheet: 5_000,
31
+ maxPdfPages: 200,
32
+ maxTextChars: 2_000_000,
33
+ },
34
+ parseTimeoutMs: 45_000,
35
+ parseTimeoutPerMBMs: 5_000,
36
+ maxParseTimeoutMs: 300_000,
37
+ };
38
+ const TEXT_EXT = new Set([
39
+ '.txt',
40
+ '.md',
41
+ '.markdown',
42
+ '.mdx',
43
+ '.csv',
44
+ '.json',
45
+ '.jsonl',
46
+ '.yml',
47
+ '.yaml',
48
+ '.toml',
49
+ '.ini',
50
+ '.log',
51
+ '.xml',
52
+ '.html',
53
+ '.htm',
54
+ '.ts',
55
+ '.tsx',
56
+ '.js',
57
+ '.jsx',
58
+ '.mjs',
59
+ '.cjs',
60
+ '.py',
61
+ '.go',
62
+ '.rs',
63
+ '.java',
64
+ '.kt',
65
+ '.sql',
66
+ '.sh',
67
+ '.ps1',
68
+ '.css',
69
+ '.scss',
70
+ ]);
71
+ const OFFICE_DOC_EXT = new Set([
72
+ '.docx',
73
+ '.pptx',
74
+ '.doc',
75
+ '.ppt',
76
+ '.odt',
77
+ '.odp',
78
+ '.rtf',
79
+ '.xmind',
80
+ '.epub',
81
+ '.tex',
82
+ ]);
83
+ const SHEET_EXT = new Set(['.xlsx', '.xls', '.ods', '.xlsm']);
84
+ /** 按扩展名归类(媒体/压缩包等归 other → 无适配器时跳过) */
85
+ export function classifyFileKind(filePath) {
86
+ const ext = extname(filePath).toLowerCase();
87
+ if (TEXT_EXT.has(ext))
88
+ return 'text';
89
+ if (ext === '.pdf')
90
+ return 'pdf';
91
+ if (OFFICE_DOC_EXT.has(ext))
92
+ return 'officeDoc';
93
+ if (SHEET_EXT.has(ext))
94
+ return 'sheet';
95
+ return 'other';
96
+ }
97
+ export function resolveKnowledgeFileLimits(input) {
98
+ const d = DEFAULT_KNOWLEDGE_FILE_LIMITS;
99
+ return {
100
+ oversize: input?.oversize ?? d.oversize,
101
+ maxFileBytes: input?.maxFileBytes ?? d.maxFileBytes,
102
+ hardMaxFileBytes: input?.hardMaxFileBytes ?? d.hardMaxFileBytes,
103
+ maxBytes: {
104
+ text: input?.maxBytes?.text ?? d.maxBytes.text,
105
+ pdf: input?.maxBytes?.pdf ?? d.maxBytes.pdf,
106
+ officeDoc: input?.maxBytes?.officeDoc ?? d.maxBytes.officeDoc,
107
+ sheet: input?.maxBytes?.sheet ?? d.maxBytes.sheet,
108
+ },
109
+ partial: {
110
+ maxSheets: input?.partial?.maxSheets ?? d.partial.maxSheets,
111
+ maxRowsPerSheet: input?.partial?.maxRowsPerSheet ?? d.partial.maxRowsPerSheet,
112
+ maxPdfPages: input?.partial?.maxPdfPages ?? d.partial.maxPdfPages,
113
+ maxTextChars: input?.partial?.maxTextChars ?? d.partial.maxTextChars,
114
+ },
115
+ parseTimeoutMs: input?.parseTimeoutMs ?? d.parseTimeoutMs,
116
+ parseTimeoutPerMBMs: input?.parseTimeoutPerMBMs ?? d.parseTimeoutPerMBMs,
117
+ maxParseTimeoutMs: input?.maxParseTimeoutMs ?? d.maxParseTimeoutMs,
118
+ };
119
+ }
120
+ /** 该格式的单文件上限 */
121
+ export function formatMaxBytes(limits, kind) {
122
+ if (kind === 'text')
123
+ return limits.maxBytes.text;
124
+ if (kind === 'pdf')
125
+ return limits.maxBytes.pdf;
126
+ if (kind === 'officeDoc')
127
+ return limits.maxBytes.officeDoc;
128
+ if (kind === 'sheet')
129
+ return limits.maxBytes.sheet;
130
+ return limits.maxFileBytes;
131
+ }
132
+ /**
133
+ * 按尺寸裁决:ok / partial / skip
134
+ *
135
+ * @param size - 文件字节数
136
+ * @param kind - 格式类别
137
+ * @param limits - 已解析限额
138
+ */
139
+ export function decideBySize(size, kind, limits) {
140
+ if (size > limits.hardMaxFileBytes) {
141
+ return { action: 'skip', reason: 'oversize_hard' };
142
+ }
143
+ const soft = formatMaxBytes(limits, kind);
144
+ if (size <= soft)
145
+ return { action: 'ok' };
146
+ if (limits.oversize === 'skip')
147
+ return { action: 'skip', reason: 'oversize_skip' };
148
+ return { action: 'partial', reason: 'oversize_soft' };
149
+ }
150
+ /** parse 超时随体积放大(封顶 maxParseTimeoutMs) */
151
+ export function parseTimeoutForSize(size, limits) {
152
+ const mb = Math.max(0, size / (1024 * 1024));
153
+ const t = limits.parseTimeoutMs + mb * limits.parseTimeoutPerMBMs;
154
+ return Math.min(limits.maxParseTimeoutMs, Math.ceil(t));
155
+ }
156
+ /**
157
+ * skip 是否值得再解析(配置调宽 / 新策略后重试)。
158
+ *
159
+ * - `ignored_path` / `no_adapter`:压缩包与媒体等,无适配器前不重试
160
+ * - `oversize*`:可重试;`oversize_hard` 仅当 size 已落入当前 hardMax(或 size 未知)
161
+ * - `empty_content`:可重试
162
+ */
163
+ export function isRetryableSkipReason(reason, size, limits) {
164
+ if (!reason)
165
+ return false;
166
+ if (reason === 'empty_content')
167
+ return true;
168
+ if (!reason.startsWith('oversize'))
169
+ return false;
170
+ if (reason === 'oversize_hard') {
171
+ // size=0 多为历史脏数据(未记录体积),允许重试一次以写入真实 size
172
+ return size <= 0 || size <= limits.hardMaxFileBytes;
173
+ }
174
+ return true;
175
+ }
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Knowledge FTS5 — 关键词倒排(可重建投影)
3
+ *
4
+ * 中文不靠默认分词:索引/查询都写入「词 + CJK 二元组」token 流(对齐 Memory)。
5
+ * FTS 表非权威;upsert/remove 同步维护,可整库 rebuild。
6
+ */
7
+ import type { KnowledgeDatabase } from './db.js';
8
+ /** 路径 + 正文 → 空格分隔 token 流(FTS unicode61 再切一次也无害) */
9
+ export declare function buildFtsTokens(text: string, path?: string): string;
10
+ /** 查询 → FTS MATCH 表达式(OR) */
11
+ export declare function buildFtsQuery(query: string): string;
12
+ export declare class KnowledgeFts {
13
+ private readonly db;
14
+ private enabled;
15
+ constructor(db: KnowledgeDatabase);
16
+ get available(): boolean;
17
+ /** 单条 upsert(先删后插,兼容 contentless 语义) */
18
+ upsert(chunkId: string, text: string, path?: string): void;
19
+ /**
20
+ * 批量 upsert。**不自开事务**:调用方(upsertFile 等)已在外层事务内,
21
+ * 嵌套 BEGIN 会炸;rebuild 时由调用方包事务。
22
+ *
23
+ * `toks` 已预计算时直接写入,避免在写事务里跑 CJK 滑窗。
24
+ */
25
+ upsertMany(rows: Array<{
26
+ id: string;
27
+ text: string;
28
+ path?: string;
29
+ toks?: string;
30
+ }>): void;
31
+ remove(chunkId: string): void;
32
+ removeMany(chunkIds: string[]): void;
33
+ /** 源级清空(卸载 / rebuild)— 经 Membership */
34
+ clearSource(sourceId: string): void;
35
+ /**
36
+ * MATCH 检索,返回 chunkId(已按可见 Membership 过滤)
37
+ *
38
+ * @returns 命中 chunk id 列表;FTS 不可用或 query 空时返回 null(调用方退 LIKE)
39
+ */
40
+ search(query: string, sourceIds: string[], limit: number): string[] | null;
41
+ /**
42
+ * 全量重建(Index 非权威;可整库重跑)
43
+ *
44
+ * **同步**版本:仅测试/小库。大库请用 {@link rebuildFromChunksAsync}。
45
+ *
46
+ * @returns 写入条数
47
+ */
48
+ rebuildFromChunks(batch?: number): number;
49
+ /**
50
+ * 分批 + 让出事件循环的重建。禁止在构造/请求同步路径整库跑(会堵死 Gateway)。
51
+ *
52
+ * @param opts.batch - 每批条数
53
+ * @param opts.onProgress - 进度回调
54
+ * @returns 写入条数
55
+ */
56
+ rebuildFromChunksAsync(opts?: {
57
+ batch?: number;
58
+ onProgress?: (done: number) => void;
59
+ }): Promise<number>;
60
+ }
@@ -0,0 +1,218 @@
1
+ /**
2
+ * Knowledge FTS5 — 关键词倒排(可重建投影)
3
+ *
4
+ * 中文不靠默认分词:索引/查询都写入「词 + CJK 二元组」token 流(对齐 Memory)。
5
+ * FTS 表非权威;upsert/remove 同步维护,可整库 rebuild。
6
+ */
7
+ import { tokenizeKeywordQuery } from '../memory/sqlite/keyword-search.js';
8
+ const TOKEN_SPLIT = /[^\p{L}\p{N}_]+/u;
9
+ const CJK_RE = /[㐀-䶿一-鿿豈-﫿぀-ヿ]/;
10
+ /** 超长 token(base64/uuid/hash/压缩 JS 标识)不进倒排,避免 FTS 膨胀 */
11
+ const MAX_TOKEN_CHARS = 32;
12
+ /** 单 chunk token 流上限,防止病态正文写爆 FTS */
13
+ const MAX_TOKENS_PER_CHUNK = 4000;
14
+ /** 路径 + 正文 → 空格分隔 token 流(FTS unicode61 再切一次也无害) */
15
+ export function buildFtsTokens(text, path) {
16
+ const parts = [];
17
+ const push = (s) => {
18
+ if (!s || s.length > MAX_TOKEN_CHARS)
19
+ return;
20
+ if (parts.length >= MAX_TOKENS_PER_CHUNK)
21
+ return;
22
+ parts.push(s.toLowerCase());
23
+ };
24
+ const add = (raw) => {
25
+ for (const p of raw.split(TOKEN_SPLIT)) {
26
+ if (!p)
27
+ continue;
28
+ if (p.length > MAX_TOKEN_CHARS)
29
+ continue;
30
+ push(p);
31
+ if (CJK_RE.test(p[0] ?? '')) {
32
+ // CJK 二元组:两字中文词在 unicode61 下也是单 token,滑窗保证子串可搜
33
+ for (let i = 0; i + 1 < p.length; i++) {
34
+ if (parts.length >= MAX_TOKENS_PER_CHUNK)
35
+ return;
36
+ push(p.slice(i, i + 2));
37
+ }
38
+ }
39
+ }
40
+ };
41
+ add(path ?? '');
42
+ add(text ?? '');
43
+ return parts.join(' ');
44
+ }
45
+ /** 查询 → FTS MATCH 表达式(OR) */
46
+ export function buildFtsQuery(query) {
47
+ const tokens = tokenizeKeywordQuery(query);
48
+ if (tokens.length === 0)
49
+ return '';
50
+ const quoted = tokens
51
+ .filter((t) => t.length >= 2)
52
+ .map((t) => `"${t.replace(/"/g, '""')}"`);
53
+ return quoted.join(' OR ');
54
+ }
55
+ export class KnowledgeFts {
56
+ db;
57
+ enabled = false;
58
+ constructor(db) {
59
+ this.db = db;
60
+ try {
61
+ this.db.raw.exec(`
62
+ CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_chunks_fts USING fts5(
63
+ chunk_id UNINDEXED,
64
+ toks,
65
+ tokenize='unicode61'
66
+ );
67
+ `);
68
+ this.enabled = true;
69
+ }
70
+ catch {
71
+ // FTS5 未编译进 sqlite:退回 LIKE 路径
72
+ this.enabled = false;
73
+ }
74
+ }
75
+ get available() {
76
+ return this.enabled;
77
+ }
78
+ /** 单条 upsert(先删后插,兼容 contentless 语义) */
79
+ upsert(chunkId, text, path) {
80
+ if (!this.enabled)
81
+ return;
82
+ this.db.raw
83
+ .prepare('DELETE FROM knowledge_chunks_fts WHERE chunk_id = ?')
84
+ .run(chunkId);
85
+ this.db.raw
86
+ .prepare('INSERT INTO knowledge_chunks_fts (chunk_id, toks) VALUES (?, ?)')
87
+ .run(chunkId, buildFtsTokens(text, path));
88
+ }
89
+ /**
90
+ * 批量 upsert。**不自开事务**:调用方(upsertFile 等)已在外层事务内,
91
+ * 嵌套 BEGIN 会炸;rebuild 时由调用方包事务。
92
+ *
93
+ * `toks` 已预计算时直接写入,避免在写事务里跑 CJK 滑窗。
94
+ */
95
+ upsertMany(rows) {
96
+ if (!this.enabled || rows.length === 0)
97
+ return;
98
+ const del = this.db.raw.prepare('DELETE FROM knowledge_chunks_fts WHERE chunk_id = ?');
99
+ const ins = this.db.raw.prepare('INSERT INTO knowledge_chunks_fts (chunk_id, toks) VALUES (?, ?)');
100
+ for (const r of rows) {
101
+ del.run(r.id);
102
+ ins.run(r.id, r.toks ?? buildFtsTokens(r.text, r.path));
103
+ }
104
+ }
105
+ remove(chunkId) {
106
+ if (!this.enabled)
107
+ return;
108
+ this.db.raw
109
+ .prepare('DELETE FROM knowledge_chunks_fts WHERE chunk_id = ?')
110
+ .run(chunkId);
111
+ }
112
+ removeMany(chunkIds) {
113
+ if (!this.enabled || chunkIds.length === 0)
114
+ return;
115
+ // 批量 IN 删除:单条 DELETE 循环在数千 chunk 时会把事件循环堵成秒级
116
+ const del = this.db.raw.prepare(`DELETE FROM knowledge_chunks_fts WHERE chunk_id IN (${chunkIds.map(() => '?').join(',')})`);
117
+ for (let i = 0; i < chunkIds.length; i += 400) {
118
+ del.run(...chunkIds.slice(i, i + 400));
119
+ }
120
+ }
121
+ /** 源级清空(卸载 / rebuild)— 经 Membership */
122
+ clearSource(sourceId) {
123
+ if (!this.enabled)
124
+ return;
125
+ this.db.raw
126
+ .prepare(`DELETE FROM knowledge_chunks_fts
127
+ WHERE chunk_id IN (
128
+ SELECT c.id FROM knowledge_chunks c
129
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
130
+ WHERE m.source_id = ?
131
+ )`)
132
+ .run(sourceId);
133
+ }
134
+ /**
135
+ * MATCH 检索,返回 chunkId(已按可见 Membership 过滤)
136
+ *
137
+ * @returns 命中 chunk id 列表;FTS 不可用或 query 空时返回 null(调用方退 LIKE)
138
+ */
139
+ search(query, sourceIds, limit) {
140
+ if (!this.enabled)
141
+ return null;
142
+ const match = buildFtsQuery(query);
143
+ if (!match || sourceIds.length === 0)
144
+ return null;
145
+ const placeholders = sourceIds.map(() => '?').join(',');
146
+ try {
147
+ const rows = this.db.raw
148
+ .prepare(`SELECT f.chunk_id AS id
149
+ FROM knowledge_chunks_fts f
150
+ JOIN knowledge_chunks c ON c.id = f.chunk_id
151
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
152
+ WHERE knowledge_chunks_fts MATCH ?
153
+ AND m.source_id IN (${placeholders})
154
+ LIMIT ?`)
155
+ .all(match, ...sourceIds, limit);
156
+ return rows.map((r) => r.id);
157
+ }
158
+ catch {
159
+ return null;
160
+ }
161
+ }
162
+ /**
163
+ * 全量重建(Index 非权威;可整库重跑)
164
+ *
165
+ * **同步**版本:仅测试/小库。大库请用 {@link rebuildFromChunksAsync}。
166
+ *
167
+ * @returns 写入条数
168
+ */
169
+ rebuildFromChunks(batch = 500) {
170
+ if (!this.enabled)
171
+ return 0;
172
+ this.db.raw.exec('DELETE FROM knowledge_chunks_fts');
173
+ let cursor = '';
174
+ let n = 0;
175
+ for (;;) {
176
+ const rows = this.db.raw
177
+ .prepare(`SELECT id, text FROM knowledge_chunks
178
+ WHERE id > ? ORDER BY id LIMIT ?`)
179
+ .all(cursor, batch);
180
+ if (rows.length === 0)
181
+ break;
182
+ this.upsertMany(rows.map((r) => ({ id: r.id, text: r.text })));
183
+ n += rows.length;
184
+ cursor = rows[rows.length - 1].id;
185
+ }
186
+ return n;
187
+ }
188
+ /**
189
+ * 分批 + 让出事件循环的重建。禁止在构造/请求同步路径整库跑(会堵死 Gateway)。
190
+ *
191
+ * @param opts.batch - 每批条数
192
+ * @param opts.onProgress - 进度回调
193
+ * @returns 写入条数
194
+ */
195
+ async rebuildFromChunksAsync(opts) {
196
+ if (!this.enabled)
197
+ return 0;
198
+ // 小批量 + 频繁让出:CJK 滑窗分词是主线程 CPU,批太大仍会饿死 /health
199
+ const batch = Math.max(20, opts?.batch ?? 50);
200
+ this.db.raw.exec('DELETE FROM knowledge_chunks_fts');
201
+ let cursor = '';
202
+ let n = 0;
203
+ for (;;) {
204
+ const rows = this.db.raw
205
+ .prepare(`SELECT id, text FROM knowledge_chunks
206
+ WHERE id > ? ORDER BY id LIMIT ?`)
207
+ .all(cursor, batch);
208
+ if (rows.length === 0)
209
+ break;
210
+ this.upsertMany(rows.map((r) => ({ id: r.id, text: r.text })));
211
+ n += rows.length;
212
+ cursor = rows[rows.length - 1].id;
213
+ opts?.onProgress?.(n);
214
+ await new Promise((r) => setImmediate(r));
215
+ }
216
+ return n;
217
+ }
218
+ }
@@ -6,7 +6,7 @@
6
6
  import type { Message } from '@octopi-agent/core/types.js';
7
7
  import type { ChunkHit, KnowledgeIndexStore } from './index-store.js';
8
8
  import type { KnowledgeHitLog } from './hit-log.js';
9
- import type { KnowledgeRetriever } from './retriever.js';
9
+ import type { AutoGroundPort } from './retriever.js';
10
10
  /** 消息 metadata.source 标记 */
11
11
  export declare const KNOWLEDGE_GROUNDING_SOURCE: "knowledgeGrounding";
12
12
  export interface GroundingPack {
@@ -21,7 +21,8 @@ export interface GroundingPack {
21
21
  text?: string;
22
22
  }
23
23
  export interface GroundingAssemblerOptions {
24
- retriever: KnowledgeRetriever;
24
+ /** 自动召回端口:本进程 KnowledgeRetriever 或 ClientKnowledgeGrounding */
25
+ retriever: AutoGroundPort;
25
26
  indexStore?: KnowledgeIndexStore;
26
27
  /** 固定预算 token(无 messagesBudget 时用;默认 1200) */
27
28
  budgetTokens?: number;