@octopi-agent/engine 0.58.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +38 -1
  4. package/dist/config-schema/substrate.d.ts +37 -0
  5. package/dist/config-schema/substrate.js +47 -0
  6. package/dist/config.d.ts +30 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  23. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  24. package/dist/harness/capabilities/document/factory.js +37 -0
  25. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  37. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  39. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  40. package/dist/harness/capabilities/document/worker-port.js +78 -0
  41. package/dist/harness/context/capabilities/index.d.ts +2 -2
  42. package/dist/harness/context/capabilities/index.js +2 -2
  43. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  44. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  45. package/dist/harness/index.d.ts +4 -2
  46. package/dist/harness/index.js +2 -1
  47. package/dist/harness/knowledge/adapters.d.ts +8 -5
  48. package/dist/harness/knowledge/adapters.js +53 -13
  49. package/dist/harness/knowledge/client.d.ts +197 -0
  50. package/dist/harness/knowledge/client.js +237 -0
  51. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  52. package/dist/harness/knowledge/connectors.d.ts +3 -3
  53. package/dist/harness/knowledge/connectors.js +14 -8
  54. package/dist/harness/knowledge/db.d.ts +40 -5
  55. package/dist/harness/knowledge/db.js +441 -113
  56. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  57. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  58. package/dist/harness/knowledge/embed-runner.js +171 -0
  59. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  60. package/dist/harness/knowledge/engine-thread.js +302 -0
  61. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  62. package/dist/harness/knowledge/extract-document.js +32 -8
  63. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  64. package/dist/harness/knowledge/fetchers.js +52 -16
  65. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  66. package/dist/harness/knowledge/file-identity.js +172 -0
  67. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  68. package/dist/harness/knowledge/file-index-store.js +132 -0
  69. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  70. package/dist/harness/knowledge/file-limits.js +175 -0
  71. package/dist/harness/knowledge/fts.d.ts +60 -0
  72. package/dist/harness/knowledge/fts.js +218 -0
  73. package/dist/harness/knowledge/grounding.d.ts +3 -2
  74. package/dist/harness/knowledge/http-app.d.ts +50 -0
  75. package/dist/harness/knowledge/http-app.js +652 -0
  76. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  77. package/dist/harness/knowledge/http-bridge.js +17 -0
  78. package/dist/harness/knowledge/index-store.d.ts +158 -93
  79. package/dist/harness/knowledge/index-store.js +982 -441
  80. package/dist/harness/knowledge/index.d.ts +45 -5
  81. package/dist/harness/knowledge/index.js +23 -2
  82. package/dist/harness/knowledge/ingest.d.ts +262 -87
  83. package/dist/harness/knowledge/ingest.js +1072 -480
  84. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  85. package/dist/harness/knowledge/job-control-state.js +44 -0
  86. package/dist/harness/knowledge/job-control.d.ts +35 -0
  87. package/dist/harness/knowledge/job-control.js +86 -0
  88. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  89. package/dist/harness/knowledge/job-queue.js +173 -0
  90. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  91. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  92. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  93. package/dist/harness/knowledge/local-stack.js +63 -0
  94. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  95. package/dist/harness/knowledge/membership-store.js +198 -0
  96. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  97. package/dist/harness/knowledge/network-guard.js +130 -85
  98. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  99. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  100. package/dist/harness/knowledge/query-service.d.ts +272 -0
  101. package/dist/harness/knowledge/query-service.js +326 -0
  102. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  103. package/dist/harness/knowledge/query-worker-client.js +207 -0
  104. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  105. package/dist/harness/knowledge/query-worker.js +186 -0
  106. package/dist/harness/knowledge/read-http.d.ts +35 -0
  107. package/dist/harness/knowledge/read-http.js +373 -0
  108. package/dist/harness/knowledge/retriever.d.ts +22 -5
  109. package/dist/harness/knowledge/retriever.js +66 -35
  110. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  111. package/dist/harness/knowledge/secret-scan.js +25 -2
  112. package/dist/harness/knowledge/serve.d.ts +48 -0
  113. package/dist/harness/knowledge/serve.js +336 -0
  114. package/dist/harness/knowledge/source-store.d.ts +80 -17
  115. package/dist/harness/knowledge/source-store.js +160 -89
  116. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  117. package/dist/harness/knowledge/start-service-process.js +158 -0
  118. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  119. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  120. package/dist/harness/knowledge/types.d.ts +1 -1
  121. package/dist/harness/knowledge/types.js +1 -1
  122. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  123. package/dist/harness/knowledge/vector-ann.js +68 -0
  124. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  125. package/dist/harness/knowledge/writer-lock.js +71 -0
  126. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  127. package/dist/harness/knowledge/writer-service.js +355 -0
  128. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  129. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  130. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  131. package/dist/harness/knowledge/writer-worker.js +189 -0
  132. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  133. package/dist/harness/memory/sqlite/embedding.js +109 -16
  134. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  135. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  136. package/dist/harness/session/attachments/service.d.ts +1 -1
  137. package/dist/harness/session/attachments/service.js +1 -1
  138. package/dist/index.d.ts +4 -2
  139. package/dist/index.js +2 -1
  140. package/dist/worker-path.d.ts +8 -0
  141. package/dist/worker-path.js +15 -0
  142. package/package.json +2 -2
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * KnowledgeDatabase — OCTOPI_HOME/knowledge/knowledge.db
3
3
  *
4
- * 全 scope Source 权威 + Project 挂载 + Global 屏蔽。
5
- * 不存 chunk/vector(Index 可重建投影,P2+)。
4
+ * Source 权威 + Project 挂载 + Global 屏蔽 + **Index 投影**
5
+ * (files/chunks/embeddings/FTS/jobs/control)。Index 非权威,可整库 rebuild。
6
6
  */
7
7
  import { mkdir } from 'node:fs/promises';
8
8
  import { dirname } from 'node:path';
@@ -31,22 +31,30 @@ export class KnowledgeDatabase {
31
31
  if (dbPath !== ':memory:') {
32
32
  await mkdir(dirname(dbPath), { recursive: true });
33
33
  }
34
+ const vecOpt = options?.sqliteVec;
35
+ const readOnly = options?.readOnly === true;
36
+ const skipMigrate = readOnly || options?.skipMigrate === true;
34
37
  const db = new DatabaseSyncCtor(dbPath, {
35
38
  // 索引写入与管理面读并发时需要更长 busy 窗口
36
39
  timeout: options?.busyTimeoutMs ?? 15_000,
37
40
  enableForeignKeyConstraints: false,
38
41
  // sqlite-vec 扩展(可选);构造后无法补开
39
- allowExtension: options?.sqliteVec !== false,
42
+ allowExtension: vecOpt !== false,
43
+ ...(readOnly ? { readOnly: true } : {}),
40
44
  });
41
- if (options?.wal !== false) {
45
+ // 只读连接不改 journal_mode(WAL 由写者建立)
46
+ if (!readOnly && options?.wal !== false) {
42
47
  db.exec('PRAGMA journal_mode = WAL');
43
48
  }
44
49
  const kdb = new KnowledgeDatabase(db);
45
- kdb.createTables();
46
- if (options?.sqliteVec !== false) {
50
+ if (!skipMigrate) {
51
+ kdb.createTables();
52
+ }
53
+ if (vecOpt !== false) {
47
54
  try {
48
55
  const { tryLoadSqliteVec } = await import('../memory/sqlite/sqlite-vec.js');
49
- kdb._sqliteVec = await tryLoadSqliteVec(db);
56
+ const extensionPath = typeof vecOpt === 'object' ? vecOpt.extensionPath : undefined;
57
+ kdb._sqliteVec = await tryLoadSqliteVec(db, extensionPath);
50
58
  }
51
59
  catch {
52
60
  kdb._sqliteVec = false;
@@ -55,14 +63,75 @@ export class KnowledgeDatabase {
55
63
  return kdb;
56
64
  }
57
65
  createTables() {
66
+ // 旧库先升列/重建,再 CREATE INDEX(否则索引引用新列会炸)
67
+ this.upgradeLegacySchema();
68
+ // v2.1 File 本位(arch/knowledge-service-http.md):无兼容迁移,一次建对。
58
69
  this.db.exec(`
70
+ CREATE TABLE IF NOT EXISTS knowledge_principals (
71
+ tenant_id TEXT NOT NULL,
72
+ gateway_id TEXT NOT NULL,
73
+ local_agent_id TEXT NOT NULL,
74
+ display_name TEXT,
75
+ status TEXT NOT NULL DEFAULT 'active',
76
+ last_seen_at INTEGER,
77
+ created_at INTEGER NOT NULL,
78
+ updated_at INTEGER NOT NULL,
79
+ PRIMARY KEY (tenant_id, gateway_id, local_agent_id)
80
+ );
81
+
82
+ CREATE TABLE IF NOT EXISTS knowledge_projects (
83
+ tenant_id TEXT NOT NULL DEFAULT 'default',
84
+ project_key TEXT NOT NULL,
85
+ display_name TEXT,
86
+ registered_by TEXT NOT NULL DEFAULT 'default',
87
+ visibility TEXT NOT NULL DEFAULT 'private',
88
+ created_at INTEGER NOT NULL,
89
+ updated_at INTEGER NOT NULL,
90
+ PRIMARY KEY (tenant_id, project_key)
91
+ );
92
+
93
+ CREATE TABLE IF NOT EXISTS knowledge_project_agents (
94
+ tenant_id TEXT NOT NULL DEFAULT 'default',
95
+ gateway_id TEXT NOT NULL DEFAULT 'default',
96
+ local_agent_id TEXT NOT NULL,
97
+ project_key TEXT NOT NULL,
98
+ created_at INTEGER NOT NULL,
99
+ PRIMARY KEY (tenant_id, gateway_id, local_agent_id, project_key)
100
+ );
101
+
102
+ CREATE TABLE IF NOT EXISTS knowledge_agent_hidden (
103
+ tenant_id TEXT NOT NULL DEFAULT 'default',
104
+ gateway_id TEXT NOT NULL DEFAULT 'default',
105
+ local_agent_id TEXT NOT NULL,
106
+ source_id TEXT NOT NULL,
107
+ created_at INTEGER NOT NULL,
108
+ PRIMARY KEY (tenant_id, gateway_id, local_agent_id, source_id)
109
+ );
110
+
111
+ CREATE TABLE IF NOT EXISTS knowledge_session_visibility (
112
+ tenant_id TEXT NOT NULL DEFAULT 'default',
113
+ gateway_id TEXT NOT NULL DEFAULT 'default',
114
+ local_session_id TEXT NOT NULL,
115
+ target_type TEXT NOT NULL CHECK (target_type IN ('project', 'source')),
116
+ target_id TEXT NOT NULL,
117
+ op TEXT NOT NULL CHECK (op IN ('include', 'exclude')),
118
+ created_at INTEGER NOT NULL,
119
+ PRIMARY KEY (tenant_id, gateway_id, local_session_id, target_type, target_id)
120
+ );
121
+
59
122
  CREATE TABLE IF NOT EXISTS knowledge_sources (
60
123
  id TEXT PRIMARY KEY,
124
+ tenant_id TEXT NOT NULL DEFAULT 'default',
125
+ registered_by TEXT NOT NULL DEFAULT 'default',
126
+ visibility TEXT NOT NULL DEFAULT 'private',
61
127
  kind TEXT NOT NULL,
62
128
  location TEXT NOT NULL,
63
129
  scope_level TEXT NOT NULL,
64
130
  scope_key TEXT NOT NULL,
65
131
  sync_json TEXT NOT NULL DEFAULT '{}',
132
+ network_json TEXT,
133
+ discover_json TEXT,
134
+ auth_ref TEXT,
66
135
  status TEXT NOT NULL DEFAULT 'pending',
67
136
  coverage REAL,
68
137
  errors_json TEXT NOT NULL DEFAULT '[]',
@@ -71,90 +140,68 @@ export class KnowledgeDatabase {
71
140
  generated_description TEXT,
72
141
  catalog_priority INTEGER,
73
142
  hidden_from_catalog INTEGER NOT NULL DEFAULT 0,
143
+ last_polled_at INTEGER,
74
144
  created_at INTEGER NOT NULL,
75
145
  updated_at INTEGER NOT NULL
76
146
  );
77
147
  CREATE INDEX IF NOT EXISTS idx_knowledge_sources_scope
78
148
  ON knowledge_sources(scope_level, scope_key);
79
- CREATE INDEX IF NOT EXISTS idx_knowledge_sources_status
80
- ON knowledge_sources(status);
81
-
82
- -- Project 显式挂载(默认拒绝):project_key + agent_id
83
- CREATE TABLE IF NOT EXISTS knowledge_projects (
84
- project_key TEXT PRIMARY KEY,
85
- display_name TEXT,
86
- created_at INTEGER NOT NULL,
87
- updated_at INTEGER NOT NULL
88
- );
89
-
90
- CREATE TABLE IF NOT EXISTS knowledge_project_agents (
91
- project_key TEXT NOT NULL,
92
- agent_id TEXT NOT NULL,
93
- created_at INTEGER NOT NULL,
94
- PRIMARY KEY (project_key, agent_id)
95
- );
96
-
97
- -- Global 源对 agent 的屏蔽
98
- CREATE TABLE IF NOT EXISTS knowledge_agent_hidden (
99
- agent_id TEXT NOT NULL,
100
- source_id TEXT NOT NULL,
101
- created_at INTEGER NOT NULL,
102
- PRIMARY KEY (agent_id, source_id)
103
- );
104
-
105
- -- 会话可见视图 overlay(资产归属不变;只改本场 effective view)
106
- CREATE TABLE IF NOT EXISTS knowledge_session_visibility (
107
- session_id TEXT NOT NULL,
108
- target_type TEXT NOT NULL CHECK (target_type IN ('project', 'source')),
109
- target_id TEXT NOT NULL,
110
- op TEXT NOT NULL CHECK (op IN ('include', 'exclude')),
111
- created_at INTEGER NOT NULL,
112
- PRIMARY KEY (session_id, target_type, target_id)
113
- );
149
+ CREATE INDEX IF NOT EXISTS idx_knowledge_sources_tenant
150
+ ON knowledge_sources(tenant_id);
114
151
 
115
- -- ── Index 投影(可重建;P2 Phase A)──
152
+ -- 物理 File:identity 稳定,size/mtime 为版本
116
153
  CREATE TABLE IF NOT EXISTS knowledge_files (
117
154
  id TEXT PRIMARY KEY,
118
- source_id TEXT NOT NULL,
119
- path TEXT NOT NULL,
120
- content_hash TEXT NOT NULL,
155
+ tenant_id TEXT NOT NULL DEFAULT 'default',
156
+ identity_key TEXT NOT NULL,
121
157
  size INTEGER NOT NULL DEFAULT 0,
122
158
  mtime INTEGER NOT NULL DEFAULT 0,
159
+ content_hash TEXT,
123
160
  adapter_id TEXT,
124
161
  status TEXT NOT NULL DEFAULT 'indexed',
125
162
  error TEXT,
126
163
  chunk_count INTEGER NOT NULL DEFAULT 0,
127
164
  indexed_at INTEGER NOT NULL,
128
- UNIQUE (source_id, path)
165
+ external_url TEXT,
166
+ etag TEXT,
167
+ last_modified TEXT,
168
+ UNIQUE (tenant_id, identity_key)
169
+ );
170
+
171
+ -- 逻辑认领:path 只活在 Membership
172
+ CREATE TABLE IF NOT EXISTS knowledge_memberships (
173
+ source_id TEXT NOT NULL,
174
+ file_id TEXT NOT NULL,
175
+ logical_path TEXT NOT NULL,
176
+ created_at INTEGER NOT NULL,
177
+ PRIMARY KEY (source_id, logical_path)
129
178
  );
130
- CREATE INDEX IF NOT EXISTS idx_knowledge_files_source ON knowledge_files(source_id);
179
+ CREATE INDEX IF NOT EXISTS idx_knowledge_memberships_file
180
+ ON knowledge_memberships(file_id);
131
181
 
132
182
  CREATE TABLE IF NOT EXISTS knowledge_chunks (
133
183
  id TEXT PRIMARY KEY,
134
184
  file_id TEXT NOT NULL,
135
- source_id TEXT NOT NULL,
136
- path TEXT NOT NULL,
137
185
  ordinal INTEGER NOT NULL,
138
186
  text TEXT NOT NULL,
139
187
  start_line INTEGER NOT NULL DEFAULT 1,
140
188
  end_line INTEGER NOT NULL DEFAULT 1
141
189
  );
142
- CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source ON knowledge_chunks(source_id);
143
190
  CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_file ON knowledge_chunks(file_id);
144
191
 
145
- -- Phase B 向量(Float32 BLOB;可选 sqlite-vec 加速)
146
192
  CREATE TABLE IF NOT EXISTS knowledge_chunk_embeddings (
147
193
  chunk_id TEXT PRIMARY KEY,
148
194
  dimensions INTEGER NOT NULL,
149
195
  embedding BLOB NOT NULL,
196
+ bucket INTEGER NOT NULL DEFAULT -1,
150
197
  created_at INTEGER NOT NULL
151
198
  );
152
199
 
153
- -- 使用痕迹(P5;提升候选计量,非权威)
154
200
  CREATE TABLE IF NOT EXISTS knowledge_hits (
155
201
  id TEXT PRIMARY KEY,
156
- source_id TEXT NOT NULL,
202
+ file_id TEXT,
157
203
  chunk_id TEXT,
204
+ source_id TEXT,
158
205
  path TEXT NOT NULL,
159
206
  agent_id TEXT,
160
207
  session_id TEXT,
@@ -162,14 +209,12 @@ export class KnowledgeDatabase {
162
209
  mode TEXT NOT NULL DEFAULT 'inject',
163
210
  created_at INTEGER NOT NULL
164
211
  );
165
- CREATE INDEX IF NOT EXISTS idx_knowledge_hits_path ON knowledge_hits(path);
166
- CREATE INDEX IF NOT EXISTS idx_knowledge_hits_source ON knowledge_hits(source_id);
167
212
  CREATE INDEX IF NOT EXISTS idx_knowledge_hits_created ON knowledge_hits(created_at);
168
213
 
169
- -- ingest 任务队列(背压排队,不丢任务)
170
214
  CREATE TABLE IF NOT EXISTS knowledge_jobs (
171
215
  id TEXT PRIMARY KEY,
172
- source_id TEXT NOT NULL,
216
+ file_id TEXT,
217
+ source_id TEXT,
173
218
  kind TEXT NOT NULL,
174
219
  path TEXT,
175
220
  priority INTEGER NOT NULL DEFAULT 2,
@@ -179,79 +224,358 @@ export class KnowledgeDatabase {
179
224
  created_at INTEGER NOT NULL,
180
225
  updated_at INTEGER NOT NULL
181
226
  );
227
+ CREATE TABLE IF NOT EXISTS knowledge_source_control (
228
+ source_id TEXT PRIMARY KEY,
229
+ aborted INTEGER NOT NULL DEFAULT 0,
230
+ aborted_at INTEGER,
231
+ updated_at INTEGER NOT NULL
232
+ );
233
+
182
234
  CREATE INDEX IF NOT EXISTS idx_knowledge_jobs_status
183
235
  ON knowledge_jobs(status, priority, created_at);
184
- CREATE INDEX IF NOT EXISTS idx_knowledge_jobs_source ON knowledge_jobs(source_id);
185
-
186
- -- path 删除/查找:无此索引时 5 万+ chunks 全表扫会堵死事件循环
187
- CREATE INDEX IF NOT EXISTS idx_knowledge_files_source_path
188
- ON knowledge_files(source_id, path);
189
- CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source_path
190
- ON knowledge_chunks(source_id, path);
236
+ CREATE INDEX IF NOT EXISTS idx_knowledge_jobs_file ON knowledge_jobs(file_id);
237
+ CREATE INDEX IF NOT EXISTS idx_knowledge_jobs_source_status
238
+ ON knowledge_jobs(source_id, status);
191
239
  `);
192
240
  this.migrate();
193
241
  }
194
- /** 幂等迁移:外源 ingest / 凭证引用列 */
195
- migrate() {
196
- // 旧 JSON 向量表 → Float32 BLOB(测试库可重建;不读旧 JSON)
197
- const embCols = this.db
242
+ /**
243
+ * 旧库升级:补缺失列 + 重建 File identity 投影。
244
+ * 必须在 CREATE INDEX 之前跑,否则旧表上建新列索引会 `no such column`。
245
+ */
246
+ upgradeLegacySchema() {
247
+ // 1) 各表补列(幂等)
248
+ const alterIfMissing = (table, cols) => {
249
+ const existing = this.db.prepare(`PRAGMA table_info(${table})`).all();
250
+ const names = new Set(existing.map((c) => c.name));
251
+ for (const col of cols) {
252
+ if (!names.has(col.name)) {
253
+ try {
254
+ this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${col.ddl}`);
255
+ }
256
+ catch {
257
+ // 表可能尚不存在(createTables 随后会建)
258
+ }
259
+ }
260
+ }
261
+ };
262
+ alterIfMissing('knowledge_sources', [
263
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
264
+ { name: 'registered_by', ddl: `registered_by TEXT NOT NULL DEFAULT 'default'` },
265
+ { name: 'visibility', ddl: `visibility TEXT NOT NULL DEFAULT 'private'` },
266
+ { name: 'network_json', ddl: `network_json TEXT` },
267
+ { name: 'discover_json', ddl: `discover_json TEXT` },
268
+ { name: 'auth_ref', ddl: `auth_ref TEXT` },
269
+ { name: 'last_polled_at', ddl: `last_polled_at INTEGER` },
270
+ ]);
271
+ alterIfMissing('knowledge_projects', [
272
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
273
+ { name: 'registered_by', ddl: `registered_by TEXT NOT NULL DEFAULT 'default'` },
274
+ { name: 'visibility', ddl: `visibility TEXT NOT NULL DEFAULT 'private'` },
275
+ ]);
276
+ alterIfMissing('knowledge_project_agents', [
277
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
278
+ { name: 'gateway_id', ddl: `gateway_id TEXT NOT NULL DEFAULT 'default'` },
279
+ { name: 'local_agent_id', ddl: `local_agent_id TEXT NOT NULL DEFAULT ''` },
280
+ ]);
281
+ alterIfMissing('knowledge_agent_hidden', [
282
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
283
+ { name: 'gateway_id', ddl: `gateway_id TEXT NOT NULL DEFAULT 'default'` },
284
+ { name: 'local_agent_id', ddl: `local_agent_id TEXT NOT NULL DEFAULT ''` },
285
+ ]);
286
+ alterIfMissing('knowledge_session_visibility', [
287
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
288
+ { name: 'gateway_id', ddl: `gateway_id TEXT NOT NULL DEFAULT 'default'` },
289
+ { name: 'local_session_id', ddl: `local_session_id TEXT NOT NULL DEFAULT ''` },
290
+ ]);
291
+ alterIfMissing('knowledge_jobs', [
292
+ { name: 'file_id', ddl: `file_id TEXT` },
293
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
294
+ ]);
295
+ alterIfMissing('knowledge_hits', [
296
+ { name: 'file_id', ddl: `file_id TEXT` },
297
+ { name: 'tenant_id', ddl: `tenant_id TEXT NOT NULL DEFAULT 'default'` },
298
+ ]);
299
+ // 2) 旧 files/chunks(source_id,path)→ File identity 重建
300
+ this.migrateLegacyFileIdentitySchema();
301
+ // 2b) 旧 projects/project_agents 等主键 → 复合键重建
302
+ this.rebuildLegacyProjectTables();
303
+ // 3) 旧 project_agents/hidden/session:local_agent_id 空则从 agent_id 回填
304
+ this.backfillLegacyAgentColumns();
305
+ this.ensureEmbeddingSchema();
306
+ }
307
+ /**
308
+ * 旧 projects PK(project_key) / project_agents PK(project_key,agent_id) → 复合键
309
+ * (含 tenant/gateway),否则 ON CONFLICT(tenant_id, project_key) 会炸。
310
+ */
311
+ rebuildLegacyProjectTables() {
312
+ const tableSql = (name) => {
313
+ try {
314
+ const row = this.db
315
+ .prepare(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?`)
316
+ .get(name);
317
+ return row?.sql ?? null;
318
+ }
319
+ catch {
320
+ return null;
321
+ }
322
+ };
323
+ // knowledge_projects:无 tenant_id 在 PK 中则重建
324
+ const projSql = tableSql('knowledge_projects');
325
+ if (projSql && !/PRIMARY KEY\s*\(\s*tenant_id/i.test(projSql)) {
326
+ this.db.exec('BEGIN');
327
+ try {
328
+ this.db.exec(`
329
+ CREATE TABLE knowledge_projects_v2 (
330
+ tenant_id TEXT NOT NULL DEFAULT 'default',
331
+ project_key TEXT NOT NULL,
332
+ display_name TEXT,
333
+ registered_by TEXT NOT NULL DEFAULT 'default',
334
+ visibility TEXT NOT NULL DEFAULT 'private',
335
+ created_at INTEGER NOT NULL,
336
+ updated_at INTEGER NOT NULL,
337
+ PRIMARY KEY (tenant_id, project_key)
338
+ );
339
+ INSERT INTO knowledge_projects_v2
340
+ (tenant_id, project_key, display_name, registered_by, visibility, created_at, updated_at)
341
+ SELECT COALESCE(tenant_id,'default'), project_key, display_name,
342
+ COALESCE(registered_by,'default'), COALESCE(visibility,'private'),
343
+ created_at, updated_at
344
+ FROM knowledge_projects;
345
+ DROP TABLE knowledge_projects;
346
+ ALTER TABLE knowledge_projects_v2 RENAME TO knowledge_projects;
347
+ `);
348
+ this.db.exec('COMMIT');
349
+ }
350
+ catch (err) {
351
+ try {
352
+ this.db.exec('ROLLBACK');
353
+ }
354
+ catch {
355
+ /* ignore */
356
+ }
357
+ throw err;
358
+ }
359
+ }
360
+ // knowledge_project_agents
361
+ const paSql = tableSql('knowledge_project_agents');
362
+ if (paSql && !/PRIMARY KEY\s*\(\s*tenant_id/i.test(paSql)) {
363
+ this.db.exec('BEGIN');
364
+ try {
365
+ this.db.exec(`
366
+ CREATE TABLE knowledge_project_agents_v2 (
367
+ tenant_id TEXT NOT NULL DEFAULT 'default',
368
+ gateway_id TEXT NOT NULL DEFAULT 'default',
369
+ local_agent_id TEXT NOT NULL DEFAULT '',
370
+ project_key TEXT NOT NULL,
371
+ created_at INTEGER NOT NULL,
372
+ PRIMARY KEY (tenant_id, gateway_id, local_agent_id, project_key)
373
+ );
374
+ INSERT OR IGNORE INTO knowledge_project_agents_v2
375
+ (tenant_id, gateway_id, local_agent_id, project_key, created_at)
376
+ SELECT COALESCE(tenant_id,'default'), COALESCE(gateway_id,'default'),
377
+ COALESCE(NULLIF(local_agent_id,''), COALESCE(agent_id,'')),
378
+ project_key, created_at
379
+ FROM knowledge_project_agents;
380
+ DROP TABLE knowledge_project_agents;
381
+ ALTER TABLE knowledge_project_agents_v2 RENAME TO knowledge_project_agents;
382
+ `);
383
+ this.db.exec('COMMIT');
384
+ }
385
+ catch (err) {
386
+ try {
387
+ this.db.exec('ROLLBACK');
388
+ }
389
+ catch {
390
+ /* ignore */
391
+ }
392
+ throw err;
393
+ }
394
+ }
395
+ // knowledge_agent_hidden
396
+ const ahSql = tableSql('knowledge_agent_hidden');
397
+ if (ahSql && !/PRIMARY KEY\s*\(\s*tenant_id/i.test(ahSql)) {
398
+ this.db.exec('BEGIN');
399
+ try {
400
+ this.db.exec(`
401
+ CREATE TABLE knowledge_agent_hidden_v2 (
402
+ tenant_id TEXT NOT NULL DEFAULT 'default',
403
+ gateway_id TEXT NOT NULL DEFAULT 'default',
404
+ local_agent_id TEXT NOT NULL DEFAULT '',
405
+ source_id TEXT NOT NULL,
406
+ created_at INTEGER NOT NULL,
407
+ PRIMARY KEY (tenant_id, gateway_id, local_agent_id, source_id)
408
+ );
409
+ INSERT OR IGNORE INTO knowledge_agent_hidden_v2
410
+ (tenant_id, gateway_id, local_agent_id, source_id, created_at)
411
+ SELECT COALESCE(tenant_id,'default'), COALESCE(gateway_id,'default'),
412
+ COALESCE(NULLIF(local_agent_id,''), COALESCE(agent_id,'')),
413
+ source_id, created_at
414
+ FROM knowledge_agent_hidden;
415
+ DROP TABLE knowledge_agent_hidden;
416
+ ALTER TABLE knowledge_agent_hidden_v2 RENAME TO knowledge_agent_hidden;
417
+ `);
418
+ this.db.exec('COMMIT');
419
+ }
420
+ catch (err) {
421
+ try {
422
+ this.db.exec('ROLLBACK');
423
+ }
424
+ catch {
425
+ /* ignore */
426
+ }
427
+ throw err;
428
+ }
429
+ }
430
+ }
431
+ /** 旧列 agent_id / session_id → local_* 回填(best-effort) */
432
+ backfillLegacyAgentColumns() {
433
+ const hasCol = (table, col) => {
434
+ try {
435
+ const rows = this.db.prepare(`PRAGMA table_info(${table})`).all();
436
+ return rows.some((r) => r.name === col);
437
+ }
438
+ catch {
439
+ return false;
440
+ }
441
+ };
442
+ try {
443
+ if (hasCol('knowledge_project_agents', 'agent_id')) {
444
+ this.db.exec(`UPDATE knowledge_project_agents SET local_agent_id = agent_id
445
+ WHERE local_agent_id IS NULL OR local_agent_id = ''`);
446
+ }
447
+ if (hasCol('knowledge_agent_hidden', 'agent_id')) {
448
+ this.db.exec(`UPDATE knowledge_agent_hidden SET local_agent_id = agent_id
449
+ WHERE local_agent_id IS NULL OR local_agent_id = ''`);
450
+ }
451
+ if (hasCol('knowledge_session_visibility', 'session_id')) {
452
+ this.db.exec(`UPDATE knowledge_session_visibility SET local_session_id = session_id
453
+ WHERE local_session_id IS NULL OR local_session_id = ''`);
454
+ }
455
+ }
456
+ catch {
457
+ // 旧列不存在则跳过
458
+ }
459
+ }
460
+ /**
461
+ * 保证 embeddings 表含 `bucket` 列并建索引。
462
+ *
463
+ * 必须在任何 `INSERT … bucket` / `WHERE bucket` **之前**调用;
464
+ * 且不得与 `CREATE INDEX … (bucket)` 同批执行(旧表无该列时会先炸索引)。
465
+ */
466
+ ensureEmbeddingSchema() {
467
+ const cols = this.db
198
468
  .prepare(`PRAGMA table_info(knowledge_chunk_embeddings)`)
199
469
  .all();
200
- const embNames = new Set(embCols.map((c) => c.name));
201
- if (embNames.has('embedding_json') && !embNames.has('embedding')) {
202
- this.db.exec('DROP TABLE IF EXISTS knowledge_chunk_embeddings');
470
+ const names = new Set(cols.map((c) => c.name));
471
+ if (!names.has('bucket')) {
472
+ this.db.exec(`ALTER TABLE knowledge_chunk_embeddings ADD COLUMN bucket INTEGER NOT NULL DEFAULT -1`);
473
+ }
474
+ this.db.exec(`CREATE INDEX IF NOT EXISTS idx_knowledge_chunk_embeddings_bucket
475
+ ON knowledge_chunk_embeddings(bucket)`);
476
+ }
477
+ /**
478
+ * 幂等迁移:createTables 已跑 upgradeLegacySchema;此处仅兜底
479
+ */
480
+ migrate() {
481
+ this.ensureEmbeddingSchema();
482
+ this.migrateLegacyFileIdentitySchema();
483
+ this.ensureEmbeddingSchema();
484
+ }
485
+ /**
486
+ * v2 旧库检测:knowledge_files 仍为 (source_id,path) 且无 identity_key / memberships。
487
+ * Index 非权威 → 迁 membership + identity;失败则重建空索引(源登记保留)。
488
+ */
489
+ migrateLegacyFileIdentitySchema() {
490
+ const filesCols = this.db
491
+ .prepare(`PRAGMA table_info(knowledge_files)`)
492
+ .all();
493
+ const fileNames = new Set(filesCols.map((c) => c.name));
494
+ const hasMemberships = this.db
495
+ .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='knowledge_memberships'`)
496
+ .get();
497
+ // 新库或已迁移
498
+ if (fileNames.has('identity_key') && hasMemberships)
499
+ return;
500
+ const now = Date.now();
501
+ this.db.exec('BEGIN');
502
+ try {
503
+ // 保留 sources / projects / hidden / session_visibility
504
+ // 重建索引投影表(identity 版)+ 清空残留 jobs(引用已删 file)
505
+ this.db.exec(`
506
+ DROP TABLE IF EXISTS knowledge_chunks_fts;
507
+ DROP TABLE IF EXISTS knowledge_chunk_embeddings;
508
+ DROP TABLE IF EXISTS knowledge_chunks;
509
+ DROP TABLE IF EXISTS knowledge_files;
510
+ DROP TABLE IF EXISTS knowledge_memberships;
511
+ `);
512
+ // jobs/hits 可能尚未建表(全新库)或已是新 schema
513
+ try {
514
+ this.db.exec('DELETE FROM knowledge_jobs');
515
+ this.db.exec('DELETE FROM knowledge_hits');
516
+ }
517
+ catch {
518
+ // 表不存在:createTables 随后会建
519
+ }
203
520
  this.db.exec(`
521
+ CREATE TABLE knowledge_files (
522
+ id TEXT PRIMARY KEY,
523
+ tenant_id TEXT NOT NULL DEFAULT 'default',
524
+ identity_key TEXT NOT NULL,
525
+ size INTEGER NOT NULL DEFAULT 0,
526
+ mtime INTEGER NOT NULL DEFAULT 0,
527
+ content_hash TEXT,
528
+ adapter_id TEXT,
529
+ status TEXT NOT NULL DEFAULT 'indexed',
530
+ error TEXT,
531
+ chunk_count INTEGER NOT NULL DEFAULT 0,
532
+ indexed_at INTEGER NOT NULL,
533
+ external_url TEXT,
534
+ etag TEXT,
535
+ last_modified TEXT,
536
+ UNIQUE (tenant_id, identity_key)
537
+ );
538
+
539
+ CREATE TABLE knowledge_memberships (
540
+ source_id TEXT NOT NULL,
541
+ file_id TEXT NOT NULL,
542
+ logical_path TEXT NOT NULL,
543
+ created_at INTEGER NOT NULL,
544
+ PRIMARY KEY (source_id, logical_path)
545
+ );
546
+ CREATE INDEX idx_knowledge_memberships_file ON knowledge_memberships(file_id);
547
+
548
+ CREATE TABLE knowledge_chunks (
549
+ id TEXT PRIMARY KEY,
550
+ file_id TEXT NOT NULL,
551
+ ordinal INTEGER NOT NULL,
552
+ text TEXT NOT NULL,
553
+ start_line INTEGER NOT NULL DEFAULT 1,
554
+ end_line INTEGER NOT NULL DEFAULT 1
555
+ );
556
+ CREATE INDEX idx_knowledge_chunks_file ON knowledge_chunks(file_id);
557
+
204
558
  CREATE TABLE knowledge_chunk_embeddings (
205
559
  chunk_id TEXT PRIMARY KEY,
206
560
  dimensions INTEGER NOT NULL,
207
561
  embedding BLOB NOT NULL,
562
+ bucket INTEGER NOT NULL DEFAULT -1,
208
563
  created_at INTEGER NOT NULL
209
564
  );
210
565
  `);
211
- // DROP 只进 freelist,文件不缩;大表迁移后立刻 VACUUM
566
+ this.db.exec('COMMIT');
567
+ }
568
+ catch (err) {
212
569
  try {
213
- this.db.exec('VACUUM');
570
+ this.db.exec('ROLLBACK');
214
571
  }
215
572
  catch {
216
- // busy 时跳过;可稍后手工 VACUUM
573
+ // 以原异常为准
217
574
  }
575
+ throw err;
218
576
  }
219
- // 旧库可能缺 path 复合索引(删文件曾阻塞主进程)
220
- this.db.exec(`
221
- CREATE INDEX IF NOT EXISTS idx_knowledge_files_source_path
222
- ON knowledge_files(source_id, path);
223
- CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source_path
224
- ON knowledge_chunks(source_id, path);
225
- `);
226
- const cols = this.db
227
- .prepare(`PRAGMA table_info(knowledge_files)`)
228
- .all();
229
- const names = new Set(cols.map((c) => c.name));
230
- if (!names.has('external_url')) {
231
- this.db.exec(`ALTER TABLE knowledge_files ADD COLUMN external_url TEXT`);
232
- }
233
- if (!names.has('etag')) {
234
- this.db.exec(`ALTER TABLE knowledge_files ADD COLUMN etag TEXT`);
235
- }
236
- if (!names.has('last_modified')) {
237
- this.db.exec(`ALTER TABLE knowledge_files ADD COLUMN last_modified TEXT`);
238
- }
239
- const srcCols = this.db
240
- .prepare(`PRAGMA table_info(knowledge_sources)`)
241
- .all();
242
- const srcNames = new Set(srcCols.map((c) => c.name));
243
- if (!srcNames.has('auth_ref')) {
244
- this.db.exec(`ALTER TABLE knowledge_sources ADD COLUMN auth_ref TEXT`);
245
- }
246
- if (!srcNames.has('network_json')) {
247
- this.db.exec(`ALTER TABLE knowledge_sources ADD COLUMN network_json TEXT`);
248
- }
249
- if (!srcNames.has('last_polled_at')) {
250
- this.db.exec(`ALTER TABLE knowledge_sources ADD COLUMN last_polled_at INTEGER`);
251
- }
252
- if (!srcNames.has('discover_json')) {
253
- this.db.exec(`ALTER TABLE knowledge_sources ADD COLUMN discover_json TEXT`);
254
- }
577
+ // 索引需重建:调用方 reconcile / reindex 补齐
578
+ void now;
255
579
  }
256
580
  get raw() {
257
581
  return this.db;
@@ -262,13 +586,15 @@ export class KnowledgeDatabase {
262
586
  stats() {
263
587
  const counts = {
264
588
  sources: 'SELECT COUNT(*) AS count FROM knowledge_sources',
589
+ memberships: 'SELECT COUNT(*) AS count FROM knowledge_memberships',
590
+ files: 'SELECT COUNT(*) AS count FROM knowledge_files',
265
591
  projectAgents: 'SELECT COUNT(*) AS count FROM knowledge_project_agents',
266
592
  agentHidden: 'SELECT COUNT(*) AS count FROM knowledge_agent_hidden',
267
593
  sessionVisibility: 'SELECT COUNT(*) AS count FROM knowledge_session_visibility',
268
- files: 'SELECT COUNT(*) AS count FROM knowledge_files',
269
594
  chunks: 'SELECT COUNT(*) AS count FROM knowledge_chunks',
595
+ ftsChunks: 'SELECT COUNT(*) AS count FROM knowledge_chunks_fts',
270
596
  embeddableChunks: `SELECT COUNT(*) AS count FROM knowledge_chunks c
271
- LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
597
+ LEFT JOIN knowledge_files f ON f.id = c.file_id
272
598
  WHERE (f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))`,
273
599
  embeddings: 'SELECT COUNT(*) AS count FROM knowledge_chunk_embeddings',
274
600
  hits: 'SELECT COUNT(*) AS count FROM knowledge_hits',
@@ -285,6 +611,8 @@ export class KnowledgeDatabase {
285
611
  result[key] = 0;
286
612
  }
287
613
  }
614
+ // 0=无 ANN(仅桶/FTS);1=sqlite-vec KNN —— 百万级部署必须为 1
615
+ result.sqliteVec = this._sqliteVec ? 1 : 0;
288
616
  return result;
289
617
  }
290
618
  }