@octopi-agent/engine 0.56.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +43 -1
  4. package/dist/config-schema/substrate.d.ts +42 -0
  5. package/dist/config-schema/substrate.js +58 -0
  6. package/dist/config.d.ts +39 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/agent/config-bridge.js +3 -0
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  23. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  24. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  25. package/dist/harness/capabilities/document/factory.js +37 -0
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  37. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  39. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  40. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  41. package/dist/harness/capabilities/document/worker-port.js +78 -0
  42. package/dist/harness/context/capabilities/index.d.ts +2 -2
  43. package/dist/harness/context/capabilities/index.js +2 -2
  44. package/dist/harness/context/constitution/default-agents.md +1 -0
  45. package/dist/harness/context/index.d.ts +2 -2
  46. package/dist/harness/context/index.js +1 -1
  47. package/dist/harness/context/layer-types.d.ts +26 -2
  48. package/dist/harness/context/layer-types.js +59 -11
  49. package/dist/harness/context/layers.d.ts +3 -0
  50. package/dist/harness/context/layers.js +38 -12
  51. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  52. package/dist/harness/context/system-prompt-assembler.js +2 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/index.d.ts +5 -3
  57. package/dist/harness/index.js +3 -2
  58. package/dist/harness/knowledge/adapters.d.ts +8 -5
  59. package/dist/harness/knowledge/adapters.js +53 -13
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/client.d.ts +197 -0
  62. package/dist/harness/knowledge/client.js +237 -0
  63. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  64. package/dist/harness/knowledge/connectors.d.ts +3 -3
  65. package/dist/harness/knowledge/connectors.js +14 -8
  66. package/dist/harness/knowledge/db.d.ts +42 -3
  67. package/dist/harness/knowledge/db.js +466 -83
  68. package/dist/harness/knowledge/describe.d.ts +2 -0
  69. package/dist/harness/knowledge/describe.js +5 -1
  70. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  71. package/dist/harness/knowledge/document-extract-worker.js +68 -0
  72. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  73. package/dist/harness/knowledge/embed-runner.js +171 -0
  74. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  75. package/dist/harness/knowledge/engine-thread.js +302 -0
  76. package/dist/harness/knowledge/extract-document.d.ts +33 -0
  77. package/dist/harness/knowledge/extract-document.js +86 -0
  78. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  79. package/dist/harness/knowledge/fetchers.js +52 -16
  80. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  81. package/dist/harness/knowledge/file-identity.js +172 -0
  82. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  83. package/dist/harness/knowledge/file-index-store.js +132 -0
  84. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  85. package/dist/harness/knowledge/file-limits.js +175 -0
  86. package/dist/harness/knowledge/fts.d.ts +60 -0
  87. package/dist/harness/knowledge/fts.js +218 -0
  88. package/dist/harness/knowledge/grounding.d.ts +3 -2
  89. package/dist/harness/knowledge/http-app.d.ts +50 -0
  90. package/dist/harness/knowledge/http-app.js +652 -0
  91. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  92. package/dist/harness/knowledge/http-bridge.js +17 -0
  93. package/dist/harness/knowledge/index-store.d.ts +166 -63
  94. package/dist/harness/knowledge/index-store.js +1025 -282
  95. package/dist/harness/knowledge/index.d.ts +47 -5
  96. package/dist/harness/knowledge/index.js +24 -2
  97. package/dist/harness/knowledge/ingest.d.ts +322 -49
  98. package/dist/harness/knowledge/ingest.js +1383 -267
  99. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  100. package/dist/harness/knowledge/job-control-state.js +44 -0
  101. package/dist/harness/knowledge/job-control.d.ts +35 -0
  102. package/dist/harness/knowledge/job-control.js +86 -0
  103. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  104. package/dist/harness/knowledge/job-queue.js +173 -0
  105. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  106. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  107. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  108. package/dist/harness/knowledge/local-stack.js +63 -0
  109. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  110. package/dist/harness/knowledge/membership-store.js +198 -0
  111. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  112. package/dist/harness/knowledge/network-guard.js +130 -85
  113. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  114. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  115. package/dist/harness/knowledge/query-service.d.ts +272 -0
  116. package/dist/harness/knowledge/query-service.js +326 -0
  117. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  118. package/dist/harness/knowledge/query-worker-client.js +207 -0
  119. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  120. package/dist/harness/knowledge/query-worker.js +186 -0
  121. package/dist/harness/knowledge/read-http.d.ts +35 -0
  122. package/dist/harness/knowledge/read-http.js +373 -0
  123. package/dist/harness/knowledge/retriever.d.ts +30 -5
  124. package/dist/harness/knowledge/retriever.js +98 -35
  125. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  126. package/dist/harness/knowledge/secret-scan.js +25 -2
  127. package/dist/harness/knowledge/serve.d.ts +48 -0
  128. package/dist/harness/knowledge/serve.js +336 -0
  129. package/dist/harness/knowledge/source-store.d.ts +83 -16
  130. package/dist/harness/knowledge/source-store.js +205 -78
  131. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  132. package/dist/harness/knowledge/start-service-process.js +158 -0
  133. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  134. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  135. package/dist/harness/knowledge/topics.d.ts +13 -0
  136. package/dist/harness/knowledge/topics.js +48 -0
  137. package/dist/harness/knowledge/types.d.ts +1 -1
  138. package/dist/harness/knowledge/types.js +1 -1
  139. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  140. package/dist/harness/knowledge/vector-ann.js +68 -0
  141. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  142. package/dist/harness/knowledge/writer-lock.js +71 -0
  143. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  144. package/dist/harness/knowledge/writer-service.js +355 -0
  145. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  146. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  147. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  148. package/dist/harness/knowledge/writer-worker.js +189 -0
  149. package/dist/harness/memory/index.d.ts +3 -2
  150. package/dist/harness/memory/index.js +3 -1
  151. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  152. package/dist/harness/memory/retrieval-rank.js +60 -0
  153. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  154. package/dist/harness/memory/sqlite/embedding.js +109 -16
  155. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  156. package/dist/harness/memory/sqlite/index.js +1 -1
  157. package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
  158. package/dist/harness/memory/sqlite/keyword-search.js +104 -17
  159. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  160. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  161. package/dist/harness/memory/store.js +39 -21
  162. package/dist/harness/memory/types.d.ts +7 -0
  163. package/dist/harness/run/runner.d.ts +1 -0
  164. package/dist/harness/run/runner.js +4 -2
  165. package/dist/harness/session/attachments/service.d.ts +1 -1
  166. package/dist/harness/session/attachments/service.js +1 -1
  167. package/dist/harness/session/tasks/index.d.ts +1 -1
  168. package/dist/harness/session/tasks/index.js +1 -1
  169. package/dist/harness/session/tasks/render.d.ts +10 -0
  170. package/dist/harness/session/tasks/render.js +27 -0
  171. package/dist/index.d.ts +4 -2
  172. package/dist/index.js +2 -1
  173. package/dist/worker-path.d.ts +8 -0
  174. package/dist/worker-path.js +15 -0
  175. package/package.json +2 -2
@@ -1,67 +1,263 @@
1
1
  /**
2
- * KnowledgeIndexStore — files / chunks 投影 + 关键词检索(Phase A)
2
+ * KnowledgeIndexStore — File 本位索引投影
3
3
  *
4
- * Index 非权威;可整库重跑。检索用 Memory 同款 CJK 二元组分词。
4
+ * chunk/file 挂 file_id;logical path 在 Membership。
5
+ * sourceId+path 的 API 保留为「Membership 视图」,内部按 identity 去重。
5
6
  */
6
7
  import { randomUUID, createHash } from 'node:crypto';
7
- import { tokenizeKeywordQuery, scoreKeywordFields } from '../memory/sqlite/keyword-search.js';
8
- import { cosineSimilarity } from '../memory/sqlite/vector-search.js';
8
+ import { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordCoverage, } from '../memory/sqlite/keyword-search.js';
9
+ import { KnowledgeFts } from './fts.js';
10
+ import { vectorBucket } from './vector-ann.js';
11
+ import { toVecBlob } from '../memory/sqlite/sqlite-vec.js';
9
12
  import { asChunkId, asSourceId } from './types.js';
13
+ import { normalizePathLexical } from './file-identity.js';
14
+ const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
15
+ function defaultIdentityKey(path) {
16
+ try {
17
+ return `path:${normalizePathLexical(path)}`;
18
+ }
19
+ catch {
20
+ return `path:${path.replace(/\\/g, '/')}`;
21
+ }
22
+ }
23
+ function rowToFile(row) {
24
+ return {
25
+ id: String(row.id),
26
+ sourceId: asSourceId(String(row.source_id ?? row.membership_source_id ?? '')),
27
+ path: String(row.logical_path ?? row.path ?? ''),
28
+ contentHash: String(row.content_hash ?? ''),
29
+ size: Number(row.size ?? 0),
30
+ mtime: Number(row.mtime ?? 0),
31
+ adapterId: row.adapter_id == null ? null : String(row.adapter_id),
32
+ status: String(row.status),
33
+ error: row.error == null ? undefined : String(row.error),
34
+ chunkCount: Number(row.chunk_count ?? 0),
35
+ indexedAt: Number(row.indexed_at ?? 0),
36
+ externalUrl: row.external_url == null ? undefined : String(row.external_url),
37
+ etag: row.etag == null ? undefined : String(row.etag),
38
+ lastModified: row.last_modified == null ? undefined : String(row.last_modified),
39
+ };
40
+ }
10
41
  export class KnowledgeIndexStore {
11
42
  db;
43
+ fts;
44
+ ftsBackfillPromise = null;
45
+ vecTableReady = false;
46
+ /** 同 File 写序(upsert ∥ purge):键 = identity_key */
47
+ writeLocks = new Map();
48
+ /** 大库 COUNT 短 TTL:UI 轮询 / jobControl 不必每请求全表 JOIN */
49
+ statsCache = new Map();
50
+ static STATS_TTL_MS = 1_000;
12
51
  constructor(db) {
13
52
  this.db = db;
53
+ this.fts = new KnowledgeFts(db);
54
+ }
55
+ cached(key, compute) {
56
+ const now = Date.now();
57
+ const hit = this.statsCache.get(key);
58
+ if (hit && now - hit.at < KnowledgeIndexStore.STATS_TTL_MS)
59
+ return hit.value;
60
+ const value = compute();
61
+ this.statsCache.set(key, { at: now, value });
62
+ return value;
63
+ }
64
+ /** 写路径后立刻让统计可见(收尾 force 也可走此绕过 TTL) */
65
+ invalidateStatsCache() {
66
+ this.statsCache.clear();
14
67
  }
15
68
  /**
16
- * 写入/替换某文件的 chunks(幂等:先删旧 chunk 与向量)
69
+ * 按 File identity 串行写(upsert / purge)。
70
+ * purge 在 dropChunks 让出窗口内与 upsert 交错会留下孤儿 FTS / 悬空 membership。
17
71
  */
18
- upsertFile(input) {
19
- const now = Date.now();
20
- const existing = this.db.raw
21
- .prepare('SELECT id FROM knowledge_files WHERE source_id = ? AND path = ?')
22
- .get(input.sourceId, input.path);
23
- const fileId = existing?.id ?? `kf_${randomUUID().slice(0, 12)}`;
24
- this.db.raw.exec('BEGIN');
72
+ async withFileWriteLock(key, fn) {
73
+ const prev = this.writeLocks.get(key) ?? Promise.resolve();
74
+ const run = prev.then(() => fn());
75
+ const tail = run.then(() => undefined, () => undefined);
76
+ this.writeLocks.set(key, tail);
25
77
  try {
78
+ return await run;
79
+ }
80
+ finally {
81
+ if (this.writeLocks.get(key) === tail)
82
+ this.writeLocks.delete(key);
83
+ }
84
+ }
85
+ get ftsAvailable() {
86
+ return this.fts.available;
87
+ }
88
+ get ftsBackfillRunning() {
89
+ return this.ftsBackfillPromise != null;
90
+ }
91
+ ensureFtsBackfill() {
92
+ if (!this.fts.available)
93
+ return Promise.resolve(0);
94
+ if (this.ftsBackfillPromise)
95
+ return this.ftsBackfillPromise;
96
+ const n = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks').get().n;
97
+ const ftsN = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks_fts').get().n;
98
+ if (n === 0 || ftsN > 0)
99
+ return Promise.resolve(0);
100
+ this.ftsBackfillPromise = this.fts
101
+ .rebuildFromChunksAsync()
102
+ .catch(() => 0)
103
+ .finally(() => {
104
+ this.ftsBackfillPromise = null;
105
+ });
106
+ return this.ftsBackfillPromise;
107
+ }
108
+ /** 解析或创建 File(identity 去重)+ Membership */
109
+ resolveFile(sourceId, path, meta) {
110
+ const identityKey = meta.identityKey ?? defaultIdentityKey(path);
111
+ const tenant = meta.tenantId ?? 'default';
112
+ const now = Date.now();
113
+ let row = this.db.raw
114
+ .prepare('SELECT id, size, mtime FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
115
+ .get(tenant, identityKey);
116
+ if (!row) {
117
+ const id = `kf_${randomUUID().slice(0, 12)}`;
26
118
  this.db.raw
27
- .prepare(`DELETE FROM knowledge_chunk_embeddings
28
- WHERE chunk_id IN (SELECT id FROM knowledge_chunks WHERE file_id = ?)`)
29
- .run(fileId);
30
- this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
119
+ .prepare(`INSERT INTO knowledge_files
120
+ (id, tenant_id, identity_key, size, mtime, content_hash, status, chunk_count, indexed_at)
121
+ VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, ?)`)
122
+ .run(id, tenant, identityKey, meta.size ?? 0, meta.mtime ?? 0, now);
123
+ row = { id, size: meta.size ?? 0, mtime: meta.mtime ?? 0 };
124
+ }
125
+ else if ((meta.size != null && meta.size !== row.size) ||
126
+ (meta.mtime != null && meta.mtime !== row.mtime)) {
31
127
  this.db.raw
32
- .prepare(`INSERT INTO knowledge_files (
33
- id, source_id, path, content_hash, size, mtime, adapter_id, status, error, chunk_count, indexed_at,
34
- external_url, etag, last_modified
35
- ) VALUES (?, ?, ?, ?, ?, ?, ?, 'indexed', NULL, ?, ?, ?, ?, ?)
36
- ON CONFLICT(source_id, path) DO UPDATE SET
37
- content_hash=excluded.content_hash,
38
- size=excluded.size,
39
- mtime=excluded.mtime,
40
- adapter_id=excluded.adapter_id,
41
- status='indexed',
42
- error=NULL,
43
- chunk_count=excluded.chunk_count,
44
- indexed_at=excluded.indexed_at,
45
- external_url=excluded.external_url,
46
- etag=excluded.etag,
47
- last_modified=excluded.last_modified`)
48
- .run(fileId, input.sourceId, input.path, input.contentHash, input.size, input.mtime, input.adapterId, input.chunks.length, now, input.externalUrl ?? null, input.etag ?? null, input.lastModified ?? null);
49
- const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, source_id, path, ordinal, text, start_line, end_line)
50
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
51
- for (const c of input.chunks) {
52
- insertChunk.run(`kc_${randomUUID().slice(0, 12)}`, fileId, input.sourceId, input.path, c.ordinal, c.text, c.startLine, c.endLine);
128
+ .prepare('UPDATE knowledge_files SET size = ?, mtime = ? WHERE id = ?')
129
+ .run(meta.size ?? row.size, meta.mtime ?? row.mtime, row.id);
130
+ }
131
+ const logical = path.replace(/\\/g, '/');
132
+ this.db.raw
133
+ .prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
134
+ VALUES (?, ?, ?, ?)
135
+ ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
136
+ .run(sourceId, row.id, logical, now);
137
+ return row.id;
138
+ }
139
+ /** 按 identity_key 查已有 File(不建行);供 job 入队绑定 file_id */
140
+ findFileIdByIdentity(identityKey, tenantId = 'default') {
141
+ const row = this.db.raw
142
+ .prepare('SELECT id FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
143
+ .get(tenantId, identityKey);
144
+ return row?.id ?? null;
145
+ }
146
+ /** 只挂 Membership(共享 File:已有 parse 在途时补认领,禁止双 parse) */
147
+ attachMembership(sourceId, fileId, path) {
148
+ const logical = path.replace(/\\/g, '/');
149
+ this.db.raw
150
+ .prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
151
+ VALUES (?, ?, ?, ?)
152
+ ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
153
+ .run(sourceId, fileId, logical, Date.now());
154
+ }
155
+ /**
156
+ * 替换文件索引。chunk/FTS **分批**写入并让出事件循环,
157
+ * 大 xlsx 数千 chunk 不得整文件一个同步事务堵死 Service。
158
+ *
159
+ * 语义:旧 chunk 先清;`status='indexing'` 直到全部批次落库后才 `indexed`。
160
+ * 检索不认 `indexing`(半成品)。批次失败/中止会清掉 partial chunks,
161
+ * 并留在 `indexing`(chunk_count=0)供 gap 扫描重试——不得把半成品标成 `error`
162
+ * 而让 partial 变得可搜。
163
+ */
164
+ async upsertFile(input, opts) {
165
+ const lockKey = input.identityKey ?? defaultIdentityKey(input.path);
166
+ return this.withFileWriteLock(lockKey, async () => {
167
+ return this.upsertFileLocked(input, opts);
168
+ });
169
+ }
170
+ async upsertFileLocked(input, opts) {
171
+ const now = Date.now();
172
+ // 小批量 + 批间让出:200/批 的 FTS 写在 8 文件并发 parse 时仍会长时间占住 Engine
173
+ const batchSize = Math.max(20, opts?.batchSize ?? 50);
174
+ const signal = opts?.signal;
175
+ const fileId = this.resolveFile(String(input.sourceId), input.path, {
176
+ identityKey: input.identityKey,
177
+ size: input.size,
178
+ mtime: input.mtime,
179
+ });
180
+ if (signal?.aborted) {
181
+ await this.markUpsertIncomplete(fileId, 'aborted');
182
+ throw new Error('aborted');
183
+ }
184
+ // 旧索引作废(含 embeddings/FTS/vec)— 分批让出,禁止同步连环 DELETE 冻住 Engine
185
+ {
186
+ const oldIds = this.db.raw
187
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
188
+ .all(fileId).map((r) => r.id);
189
+ await this.dropChunksAsync(oldIds);
190
+ this.db.raw.exec('BEGIN');
191
+ try {
192
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
193
+ this.db.raw
194
+ .prepare(`UPDATE knowledge_files
195
+ SET content_hash = ?, size = ?, mtime = ?, adapter_id = ?,
196
+ status = 'indexing', error = NULL, chunk_count = 0, indexed_at = ?,
197
+ external_url = ?, etag = ?, last_modified = ?
198
+ WHERE id = ?`)
199
+ .run(input.contentHash, input.size, input.mtime, input.adapterId, now, input.externalUrl ?? null, input.etag ?? null, input.lastModified ?? null, fileId);
200
+ this.db.raw.exec('COMMIT');
201
+ }
202
+ catch (err) {
203
+ try {
204
+ this.db.raw.exec('ROLLBACK');
205
+ }
206
+ catch {
207
+ // 以原异常为准
208
+ }
209
+ throw err;
53
210
  }
54
- this.db.raw.exec('COMMIT');
55
211
  }
56
- catch (err) {
212
+ for (let offset = 0; offset < input.chunks.length; offset += batchSize) {
213
+ if (signal?.aborted) {
214
+ await this.markUpsertIncomplete(fileId, 'aborted');
215
+ throw new Error('aborted');
216
+ }
217
+ const batch = input.chunks.slice(offset, offset + batchSize);
218
+ this.db.raw.exec('BEGIN');
57
219
  try {
58
- this.db.raw.exec('ROLLBACK');
220
+ const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, ordinal, text, start_line, end_line)
221
+ VALUES (?, ?, ?, ?, ?, ?)`);
222
+ const ftsRows = [];
223
+ for (const c of batch) {
224
+ const cid = `kc_${randomUUID().slice(0, 12)}`;
225
+ insertChunk.run(cid, fileId, c.ordinal, c.text, c.startLine, c.endLine);
226
+ ftsRows.push({
227
+ id: cid,
228
+ text: c.text,
229
+ path: input.path,
230
+ ...(c.ftsToks != null ? { toks: c.ftsToks } : {}),
231
+ });
232
+ }
233
+ this.fts.upsertMany(ftsRows);
234
+ this.db.raw.exec('COMMIT');
59
235
  }
60
- catch {
61
- // rollback 失败仅吞掉:上层以原异常为准,避免掩盖根因
236
+ catch (err) {
237
+ try {
238
+ this.db.raw.exec('ROLLBACK');
239
+ }
240
+ catch {
241
+ // 以原异常为准
242
+ }
243
+ const message = err instanceof Error ? err.message : String(err);
244
+ await this.markUpsertIncomplete(fileId, message);
245
+ throw err;
62
246
  }
63
- throw err;
247
+ if (offset + batchSize < input.chunks.length) {
248
+ await new Promise((r) => setImmediate(r));
249
+ }
250
+ }
251
+ if (signal?.aborted) {
252
+ await this.markUpsertIncomplete(fileId, 'aborted');
253
+ throw new Error('aborted');
64
254
  }
255
+ this.db.raw
256
+ .prepare(`UPDATE knowledge_files
257
+ SET status = 'indexed', error = NULL, chunk_count = ?, indexed_at = ?
258
+ WHERE id = ?`)
259
+ .run(input.chunks.length, now, fileId);
260
+ this.invalidateStatsCache();
65
261
  return {
66
262
  id: fileId,
67
263
  sourceId: input.sourceId,
@@ -79,322 +275,869 @@ export class KnowledgeIndexStore {
79
275
  };
80
276
  }
81
277
  /**
82
- * 标记 skipped — **不删** 已有 chunks(瞬时/边界失败保旧索引);仅元数据。
278
+ * 半写入收尾:清掉已落地的 partial chunks,回到可重试的 `indexing`。
279
+ * 不得标成 `error`——error 行上的 partial 会被检索捞出。
83
280
  */
84
- markFileSkipped(sourceId, path, reason) {
85
- this.markFileNonIndexed(sourceId, path, 'skipped', reason);
281
+ async markUpsertIncomplete(fileId, message) {
282
+ try {
283
+ const ids = this.db.raw
284
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
285
+ .all(fileId).map((r) => r.id);
286
+ await this.dropChunksAsync(ids);
287
+ this.db.raw.exec('BEGIN');
288
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
289
+ this.db.raw
290
+ .prepare(`UPDATE knowledge_files
291
+ SET status = 'indexing', chunk_count = 0, error = ?, indexed_at = ?
292
+ WHERE id = ?`)
293
+ .run(message, Date.now(), fileId);
294
+ this.db.raw.exec('COMMIT');
295
+ }
296
+ catch {
297
+ try {
298
+ this.db.raw.exec('ROLLBACK');
299
+ }
300
+ catch {
301
+ // 清理失败时至少保证调用方仍看到原异常;半成品由 gap 扫描按 indexing 重做
302
+ }
303
+ }
304
+ }
305
+ markFileSkipped(sourceId, path, reason, size, identityKey) {
306
+ this.markFileNonIndexed(sourceId, path, 'skipped', reason, size, identityKey);
307
+ }
308
+ markFileError(sourceId, path, error, size, identityKey) {
309
+ this.markFileNonIndexed(sourceId, path, 'error', error, size, identityKey);
86
310
  }
87
311
  /**
88
- * 标记 error — **不删** 已有 chunks(网络抖动不得清库);仅元数据。
89
- * 成功 `upsertFile` 才替换 chunks。
312
+ * 标记 skipped/error — **不删** 已有 chunks。
313
+ * 若已有 membership,只改状态;**不得**用 path-identity 新建 File 顶掉带 chunk 的行。
90
314
  */
91
- markFileError(sourceId, path, error) {
92
- this.markFileNonIndexed(sourceId, path, 'error', error);
93
- }
94
- markFileNonIndexed(sourceId, path, status, detail) {
95
- const now = Date.now();
315
+ markFileNonIndexed(sourceId, path, status, detail, size, identityKey) {
316
+ const logical = path.replace(/\\/g, '/');
96
317
  const existing = this.db.raw
97
- .prepare('SELECT id, chunk_count FROM knowledge_files WHERE source_id = ? AND path = ?')
98
- .get(sourceId, path);
99
- const fileId = existing?.id ?? `kf_${randomUUID().slice(0, 12)}`;
100
- const chunkCount = existing?.chunk_count ?? 0;
318
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
319
+ .get(String(sourceId), logical);
320
+ if (existing?.file_id) {
321
+ this.db.raw
322
+ .prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
323
+ .run(status, detail, Date.now(), existing.file_id);
324
+ return;
325
+ }
326
+ // 优先真实 identity(win:/unix:),避免 path: 与成功路径分裂出零认领行
327
+ const fileId = this.resolveFile(String(sourceId), path, { size, identityKey });
101
328
  this.db.raw
102
- .prepare(`INSERT INTO knowledge_files (
103
- id, source_id, path, content_hash, size, mtime, adapter_id, status, error, chunk_count, indexed_at
104
- ) VALUES (?, ?, ?, '', 0, 0, NULL, ?, ?, ?, ?)
105
- ON CONFLICT(source_id, path) DO UPDATE SET
106
- status=excluded.status,
107
- error=excluded.error,
108
- indexed_at=excluded.indexed_at`)
109
- .run(fileId, sourceId, path, status, detail, chunkCount, now);
329
+ .prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
330
+ .run(status, detail, Date.now(), fileId);
110
331
  }
111
332
  /**
112
- * 文件是否已用同一 contentHash 索引(且确有 chunk,防半写入)
333
+ * 清零认领的 path: 降级 File(错误路径与成功路径 identity 分裂残留)。
334
+ *
335
+ * @returns 删除的 File 行数
113
336
  */
337
+ purgeOrphanPathIdentityFiles() {
338
+ const res = this.db.raw
339
+ .prepare(`DELETE FROM knowledge_files
340
+ WHERE identity_key LIKE 'path:%'
341
+ AND NOT EXISTS (
342
+ SELECT 1 FROM knowledge_memberships m WHERE m.file_id = knowledge_files.id
343
+ )`)
344
+ .run();
345
+ return Number(res.changes ?? 0);
346
+ }
114
347
  isFresh(sourceId, path, contentHash) {
348
+ const logical = path.replace(/\\/g, '/');
349
+ const row = this.db.raw
350
+ .prepare(`SELECT f.content_hash AS content_hash, f.chunk_count AS chunk_count
351
+ FROM knowledge_memberships m
352
+ JOIN knowledge_files f ON f.id = m.file_id
353
+ WHERE m.source_id = ? AND m.logical_path = ? AND f.status <> 'indexing'`)
354
+ .get(String(sourceId), logical);
355
+ return row?.content_hash === contentHash && (row?.chunk_count ?? 0) > 0;
356
+ }
357
+ invalidateFileForReparse(sourceId, path) {
358
+ const logical = path.replace(/\\/g, '/');
115
359
  const row = this.db.raw
116
- .prepare(`SELECT content_hash, chunk_count FROM knowledge_files
117
- WHERE source_id = ? AND path = ? AND status = 'indexed'`)
118
- .get(sourceId, path);
119
- return (row?.content_hash === contentHash &&
120
- (row?.chunk_count ?? 0) > 0);
360
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
361
+ .get(String(sourceId), logical);
362
+ if (!row)
363
+ return;
364
+ this.db.raw
365
+ .prepare(`UPDATE knowledge_files SET content_hash = '' WHERE id = ?`)
366
+ .run(row.file_id);
121
367
  }
122
368
  getFile(sourceId, path) {
369
+ const logical = path.replace(/\\/g, '/');
123
370
  const row = this.db.raw
124
- .prepare('SELECT * FROM knowledge_files WHERE source_id = ? AND path = ?')
125
- .get(sourceId, path);
371
+ .prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
372
+ FROM knowledge_memberships m
373
+ JOIN knowledge_files f ON f.id = m.file_id
374
+ WHERE m.source_id = ? AND m.logical_path = ?`)
375
+ .get(String(sourceId), logical);
126
376
  return row ? rowToFile(row) : null;
127
377
  }
128
378
  listFiles(sourceId) {
129
379
  const rows = this.db.raw
130
- .prepare('SELECT * FROM knowledge_files WHERE source_id = ? ORDER BY path')
131
- .all(sourceId);
380
+ .prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
381
+ FROM knowledge_memberships m
382
+ JOIN knowledge_files f ON f.id = m.file_id
383
+ WHERE m.source_id = ? ORDER BY m.logical_path`)
384
+ .all(String(sourceId));
132
385
  return rows.map(rowToFile);
133
386
  }
387
+ listFilesPaged(sourceId, opts) {
388
+ const pageSize = Math.max(1, Math.min(200, opts?.pageSize ?? 50));
389
+ const page = Math.max(1, opts?.page ?? 1);
390
+ const status = opts?.status && opts.status !== 'all' ? opts.status : null;
391
+ const ext = opts?.ext && opts.ext !== 'all'
392
+ ? opts.ext.replace(/^\./, '').toLowerCase()
393
+ : null;
394
+ const q = opts?.q?.trim().toLowerCase() || null;
395
+ // SQL 过滤 + LIMIT/OFFSET:禁止全量拉表再在内存分页(1k+ 文件时 UI 轮询会拖垮 Engine/回包)
396
+ const where = ['m.source_id = ?'];
397
+ const params = [String(sourceId)];
398
+ if (status) {
399
+ where.push('f.status = ?');
400
+ params.push(status);
401
+ }
402
+ if (ext) {
403
+ where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
404
+ params.push(`%.${ext.replace(/[\\%_]/g, (c) => `\\${c}`)}`);
405
+ }
406
+ if (q) {
407
+ where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
408
+ params.push(`%${q.replace(/[\\%_]/g, (c) => `\\${c}`)}%`);
409
+ }
410
+ const whereSql = where.join(' AND ');
411
+ const totalRow = this.db.raw
412
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_memberships m
413
+ JOIN knowledge_files f ON f.id = m.file_id
414
+ WHERE ${whereSql}`)
415
+ .get(...params);
416
+ const rows = this.db.raw
417
+ .prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
418
+ FROM knowledge_memberships m
419
+ JOIN knowledge_files f ON f.id = m.file_id
420
+ WHERE ${whereSql}
421
+ ORDER BY m.logical_path
422
+ LIMIT ? OFFSET ?`)
423
+ .all(...params, pageSize, (page - 1) * pageSize);
424
+ // counts 走 SQL 聚合:禁止把整源 logical_path 拉进 JS 再计数(UI 每页轮询会打满 Engine)
425
+ const statusCounts = { indexed: 0, skipped: 0, error: 0 };
426
+ const statusRows = this.db.raw
427
+ .prepare(`SELECT f.status AS status, COUNT(*) AS n
428
+ FROM knowledge_memberships m
429
+ JOIN knowledge_files f ON f.id = m.file_id
430
+ WHERE m.source_id = ?
431
+ GROUP BY f.status`)
432
+ .all(String(sourceId));
433
+ for (const r of statusRows) {
434
+ if (r.status === 'indexed' || r.status === 'skipped' || r.status === 'error') {
435
+ statusCounts[r.status] = Number(r.n ?? 0);
436
+ }
437
+ }
438
+ // ext:basename → 后缀(目录名带点不影响);JSON split 规避 SQLite 无 rsplit
439
+ const extRows = this.db.raw
440
+ .prepare(`SELECT lower(ext) AS ext, COUNT(*) AS n FROM (
441
+ SELECT
442
+ CASE
443
+ WHEN instr(name, '.') = 0 OR name IS NULL THEN ''
444
+ ELSE json_extract('["' || replace(name, '.', '","') || '"]', '$[#-1]')
445
+ END AS ext
446
+ FROM (
447
+ SELECT
448
+ json_extract(
449
+ '["' || replace(replace(COALESCE(m.logical_path, ''), char(92), '","'), '/', '","') || '"]',
450
+ '$[#-1]'
451
+ ) AS name
452
+ FROM knowledge_memberships m
453
+ WHERE m.source_id = ?
454
+ )
455
+ )
456
+ WHERE ext IS NOT NULL AND ext != ''
457
+ GROUP BY ext`)
458
+ .all(String(sourceId));
459
+ const extCounts = extRows
460
+ .filter((r) => r.ext)
461
+ .map((r) => ({ ext: r.ext, n: Number(r.n ?? 0) }));
462
+ const items = rows.map((row) => {
463
+ const f = rowToFile(row);
464
+ const extName = f.path.includes('.')
465
+ ? (f.path.split('.').pop() ?? '').toLowerCase()
466
+ : '';
467
+ return { ...f, ext: extName };
468
+ });
469
+ return {
470
+ items,
471
+ total: Number(totalRow?.n ?? 0),
472
+ page,
473
+ pageSize,
474
+ statusCounts,
475
+ extCounts,
476
+ };
477
+ }
478
+ listFilePathsFiltered(sourceId, filter) {
479
+ const rows = this.db.raw
480
+ .prepare(`SELECT m.logical_path AS path, f.status AS status
481
+ FROM knowledge_memberships m
482
+ JOIN knowledge_files f ON f.id = m.file_id
483
+ WHERE m.source_id = ? ORDER BY m.logical_path`)
484
+ .all(String(sourceId));
485
+ let out = rows.map((r) => r.path);
486
+ if (filter?.status) {
487
+ out = rows.filter((r) => r.status === filter.status).map((r) => r.path);
488
+ }
489
+ if (filter?.ext && filter.ext !== 'all') {
490
+ const ext = filter.ext.replace(/^\./, '').toLowerCase();
491
+ out = out.filter((p) => p.toLowerCase().endsWith(`.${ext}`));
492
+ }
493
+ if (filter?.max != null)
494
+ out = out.slice(0, filter.max);
495
+ return out;
496
+ }
497
+ /** 解绑 path;File 仅在零认领时 purge(单文件同步语义,供测试/轻量路径) */
498
+ removeFile(sourceId, path) {
499
+ const logical = path.replace(/\\/g, '/');
500
+ const row = this.db.raw
501
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
502
+ .get(String(sourceId), logical);
503
+ this.db.raw
504
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
505
+ .run(String(sourceId), logical);
506
+ if (!row)
507
+ return;
508
+ const n = this.db.raw
509
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
510
+ .get(row.file_id).n;
511
+ if (n === 0)
512
+ this.purgeFileRow(row.file_id);
513
+ }
134
514
  /**
135
- * 单文件 purge(合规):chunks + embeddings + files 行
515
+ * 解绑 path + 可能的 purge(异步让出)。
516
+ * 单文件 purge 也可能删数千 chunk——必须让出,否则 Engine 调度不了 HTTP 回包。
136
517
  */
137
- purgePath(sourceId, path) {
138
- this.removeFile(sourceId, path);
518
+ async removeFileAsync(sourceId, path, opts) {
519
+ const logical = path.replace(/\\/g, '/');
520
+ const row = this.db.raw
521
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
522
+ .get(String(sourceId), logical);
523
+ this.db.raw
524
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
525
+ .run(String(sourceId), logical);
526
+ if (!row)
527
+ return;
528
+ const n = this.db.raw
529
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
530
+ .get(row.file_id).n;
531
+ if (n === 0)
532
+ await this.purgeFileRowAsync(row.file_id, opts);
139
533
  }
140
534
  /**
141
- * 源级 purge
535
+ * 批量解绑:每 `yieldEvery` 条让出事件循环。
536
+ * 删大目录(数百文件)时同步连环 DELETE 会堵死 Engine,HTTP/Query 回包都无法调度。
142
537
  */
143
- purgeSource(sourceId) {
144
- this.clearSource(sourceId);
538
+ async removeFilesAsync(sourceId, paths, opts) {
539
+ const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
540
+ let n = 0;
541
+ for (let i = 0; i < paths.length; i++) {
542
+ await this.removeFileAsync(sourceId, paths[i], opts);
543
+ n += 1;
544
+ if ((i + 1) % yieldEvery === 0) {
545
+ await new Promise((r) => setImmediate(r));
546
+ }
547
+ }
548
+ return n;
549
+ }
550
+ removeFiles(sourceId, paths) {
551
+ for (const p of paths)
552
+ this.removeFile(sourceId, p);
145
553
  }
146
554
  /**
147
- * 删除源下全部 index(卸载 / rebuild)
555
+ * 删除 path 及子树:先收集 logical_path,再分批异步解绑 + purge。
148
556
  */
149
- clearSource(sourceId) {
150
- this.db.raw
151
- .prepare(`DELETE FROM knowledge_chunk_embeddings
152
- WHERE chunk_id IN (SELECT id FROM knowledge_chunks WHERE source_id = ?)`)
153
- .run(sourceId);
154
- this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE source_id = ?').run(sourceId);
155
- this.db.raw.prepare('DELETE FROM knowledge_files WHERE source_id = ?').run(sourceId);
557
+ async removePathTreeAsync(sourceId, path, opts) {
558
+ const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
559
+ const rows = this.db.raw
560
+ .prepare(`SELECT logical_path FROM knowledge_memberships
561
+ WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
562
+ .all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
563
+ return this.removeFilesAsync(sourceId, rows.map((r) => r.logical_path), opts);
156
564
  }
157
- removeFile(sourceId, path) {
565
+ removePathTree(sourceId, path) {
566
+ const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
567
+ const rows = this.db.raw
568
+ .prepare(`SELECT logical_path FROM knowledge_memberships
569
+ WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
570
+ .all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
571
+ for (const r of rows)
572
+ this.removeFile(sourceId, r.logical_path);
573
+ }
574
+ /**
575
+ * 清空源认领:分批 purge,避免一次清数千 File 堵死事件循环。
576
+ */
577
+ async clearSourceAsync(sourceId, opts) {
578
+ const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
579
+ const rows = this.db.raw
580
+ .prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
581
+ .all(String(sourceId));
158
582
  this.db.raw
159
- .prepare(`DELETE FROM knowledge_chunk_embeddings
160
- WHERE chunk_id IN (
161
- SELECT id FROM knowledge_chunks WHERE source_id = ? AND path = ?
162
- )`)
163
- .run(sourceId, path);
583
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
584
+ .run(String(sourceId));
585
+ let purged = 0;
586
+ for (let i = 0; i < rows.length; i++) {
587
+ const fileId = rows[i].file_id;
588
+ const n = this.db.raw
589
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
590
+ .get(fileId).n;
591
+ if (n === 0) {
592
+ await this.purgeFileRowAsync(fileId, opts);
593
+ purged += 1;
594
+ }
595
+ if ((i + 1) % yieldEvery === 0) {
596
+ await new Promise((r) => setImmediate(r));
597
+ }
598
+ }
599
+ return purged;
600
+ }
601
+ clearSource(sourceId) {
602
+ const rows = this.db.raw
603
+ .prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
604
+ .all(String(sourceId));
164
605
  this.db.raw
165
- .prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?')
166
- .run(sourceId, path);
606
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
607
+ .run(String(sourceId));
608
+ for (const r of rows) {
609
+ const n = this.db.raw
610
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
611
+ .get(r.file_id).n;
612
+ if (n === 0)
613
+ this.purgeFileRow(r.file_id);
614
+ }
615
+ }
616
+ purgePath(sourceId, path) {
617
+ this.removeFile(sourceId, path);
618
+ }
619
+ /** 按 fileId 物理删除(membership 已清) */
620
+ purgeFile(fileId) {
621
+ this.purgeFileRow(fileId);
622
+ }
623
+ async purgeFileAsync(fileId, opts) {
624
+ await this.purgeFileRowAsync(fileId, opts);
625
+ }
626
+ purgeSource(sourceId) {
627
+ this.clearSource(sourceId);
628
+ }
629
+ purgeFileRow(fileId) {
630
+ const oldIds = this.db.raw
631
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
632
+ .all(fileId).map((r) => r.id);
633
+ this.dropChunks(oldIds);
634
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
635
+ this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
636
+ this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
637
+ }
638
+ /**
639
+ * purge + 分批让出。大文件数千 chunk 时 dropChunks 同步段仍可能秒级——批间 setImmediate。
640
+ * 与 upsertFile 共用 File 写锁,避免让出窗口被 parse 插入后残留孤儿 FTS。
641
+ */
642
+ async purgeFileRowAsync(fileId, opts) {
643
+ const keyRow = this.db.raw
644
+ .prepare('SELECT identity_key FROM knowledge_files WHERE id = ?')
645
+ .get(fileId);
646
+ const lockKey = keyRow?.identity_key ?? `id:${fileId}`;
647
+ await this.withFileWriteLock(lockKey, async () => {
648
+ const oldIds = this.db.raw
649
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
650
+ .all(fileId).map((r) => r.id);
651
+ await this.dropChunksAsync(oldIds);
652
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
653
+ this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
654
+ this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
655
+ await new Promise((r) => setImmediate(r));
656
+ });
657
+ void opts;
658
+ }
659
+ dropChunks(chunkIds) {
660
+ if (chunkIds.length === 0)
661
+ return;
662
+ // 分批:大文件重解析时数千 id 一次 IN 会长时间占住 Engine 事件循环
663
+ for (let i = 0; i < chunkIds.length; i += 400) {
664
+ this.dropChunksBatch(chunkIds.slice(i, i + 400));
665
+ }
666
+ }
667
+ async dropChunksAsync(chunkIds) {
668
+ if (chunkIds.length === 0)
669
+ return;
670
+ for (let i = 0; i < chunkIds.length; i += 400) {
671
+ this.dropChunksBatch(chunkIds.slice(i, i + 400));
672
+ // 批间必须让出:否则单文件数千 chunk 仍会冻住 Engine 数秒
673
+ await new Promise((r) => setImmediate(r));
674
+ }
675
+ }
676
+ dropChunksBatch(batch) {
677
+ if (batch.length === 0)
678
+ return;
679
+ const ph = batch.map(() => '?').join(',');
167
680
  this.db.raw
168
- .prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?')
169
- .run(sourceId, path);
681
+ .prepare(`DELETE FROM knowledge_chunk_embeddings WHERE chunk_id IN (${ph})`)
682
+ .run(...batch);
683
+ this.deleteKnowledgeVec(batch);
684
+ this.fts.removeMany(batch);
685
+ this.invalidateStatsCache();
686
+ }
687
+ deleteKnowledgeVec(chunkIds) {
688
+ if (!this.db.sqliteVecEnabled || chunkIds.length === 0)
689
+ return;
690
+ try {
691
+ const del = this.db.raw.prepare(`DELETE FROM knowledge_vec WHERE chunk_id IN (${chunkIds.map(() => '?').join(',')})`);
692
+ for (let i = 0; i < chunkIds.length; i += 400) {
693
+ del.run(...chunkIds.slice(i, i + 400));
694
+ }
695
+ }
696
+ catch {
697
+ // 无 vec 表时忽略
698
+ }
170
699
  }
171
700
  /**
172
- * 差量 prune:删除本轮 discover **未出现** 的 path(含 chunks)。
173
- * 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
701
+ * 对账差量删除:批量路径,内部让出事件循环。
174
702
  *
175
- * @returns 删除的文件数
703
+ * @returns 删除条数
176
704
  */
177
- pruneMissing(sourceId, keepPaths) {
178
- const keep = keepPaths instanceof Set ? keepPaths : new Set(keepPaths);
179
- if (keep.size === 0)
705
+ async pruneMissingAsync(sourceId, keepPaths, opts) {
706
+ // keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
707
+ const normalize = (p) => p.replace(/\\/g, '/');
708
+ const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
709
+ // discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
710
+ if (keep.size === 0 && !opts?.allowEmptyKeep)
180
711
  return 0;
181
712
  const files = this.listFiles(sourceId);
182
- let removed = 0;
183
- for (const f of files) {
184
- if (!keep.has(f.path)) {
185
- this.removeFile(sourceId, f.path);
186
- removed += 1;
187
- }
188
- }
189
- return removed;
713
+ const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
714
+ if (gone.length === 0)
715
+ return 0;
716
+ await this.removeFilesAsync(sourceId, gone, opts);
717
+ return gone.length;
718
+ }
719
+ pruneMissing(sourceId, keepPaths, opts) {
720
+ // keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
721
+ const normalize = (p) => p.replace(/\\/g, '/');
722
+ const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
723
+ // discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
724
+ if (keep.size === 0 && !opts?.allowEmptyKeep)
725
+ return 0;
726
+ const files = this.listFiles(sourceId);
727
+ const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
728
+ if (gone.length === 0)
729
+ return 0;
730
+ this.removeFiles(sourceId, gone);
731
+ return gone.length;
190
732
  }
191
733
  sourceStats(sourceId) {
734
+ return this.cached(`src-stats:${sourceId}`, () => this.computeSourceStats(sourceId));
735
+ }
736
+ computeSourceStats(sourceId) {
192
737
  const f = this.db.raw
193
738
  .prepare(`SELECT
194
739
  COUNT(*) AS total,
195
- SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS errors,
196
- SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped
197
- FROM knowledge_files WHERE source_id = ?`)
198
- .get(sourceId);
740
+ SUM(CASE WHEN f.status = 'error' THEN 1 ELSE 0 END) AS errors,
741
+ SUM(CASE WHEN f.status = 'skipped' THEN 1 ELSE 0 END) AS skipped
742
+ FROM knowledge_memberships m
743
+ JOIN knowledge_files f ON f.id = m.file_id
744
+ WHERE m.source_id = ?`)
745
+ .get(String(sourceId));
199
746
  const c = this.db.raw
200
- .prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
201
- .get(sourceId);
747
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
748
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
749
+ WHERE m.source_id = ?`)
750
+ .get(String(sourceId));
751
+ const ec = this.db.raw
752
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
753
+ JOIN knowledge_files f ON f.id = c.file_id
754
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
755
+ WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
756
+ .get(String(sourceId));
757
+ const e = this.db.raw
758
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
759
+ JOIN knowledge_chunks c ON c.id = e.chunk_id
760
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
761
+ WHERE m.source_id = ?`)
762
+ .get(String(sourceId));
202
763
  return {
203
764
  files: f?.total ?? 0,
204
765
  chunks: c?.n ?? 0,
766
+ embeddableChunks: ec?.n ?? 0,
767
+ embeddings: e?.n ?? 0,
205
768
  errors: f?.errors ?? 0,
206
769
  skipped: f?.skipped ?? 0,
207
770
  };
208
771
  }
209
772
  /**
210
- * 关键词检索(CJK 二元组 + Latin);限定 sourceIds(可见集)
773
+ * 关键词检索。sourceIds = 可见 Membership;chunk 按 file 去重。
211
774
  */
212
775
  search(query, opts) {
213
776
  const tokens = tokenizeKeywordQuery(query);
777
+ const tokenDetail = tokenizeKeywordDetail(query);
214
778
  if (tokens.length === 0 || opts.sourceIds.length === 0)
215
779
  return [];
216
780
  const limit = opts.limit ?? 8;
217
781
  const placeholders = opts.sourceIds.map(() => '?').join(',');
218
- const rows = this.db.raw
219
- .prepare(`SELECT id, source_id, path, ordinal, text, start_line, end_line
220
- FROM knowledge_chunks
221
- WHERE source_id IN (${placeholders})
222
- ORDER BY source_id, path, ordinal
223
- LIMIT 20000`)
224
- .all(...opts.sourceIds);
225
- const hits = [];
226
- for (const row of rows) {
227
- const score = scoreKeywordFields({ content: row.text, tags: row.path }, tokens);
228
- if (score <= 0)
229
- continue;
230
- hits.push({
231
- chunkId: asChunkId(row.id),
232
- sourceId: asSourceId(row.source_id),
233
- path: row.path,
234
- ordinal: row.ordinal,
235
- text: row.text,
236
- startLine: row.start_line,
237
- endLine: row.end_line,
238
- score,
239
- });
240
- }
241
- hits.sort((a, b) => b.score - a.score);
242
- return hits.slice(0, limit);
243
- }
244
- // ── Phase B 向量 ──
245
- /**
246
- * 按 chunk id 读取
247
- */
248
- getChunkById(chunkId) {
249
- const row = this.db.raw
250
- .prepare('SELECT id, source_id, path, text, start_line, end_line FROM knowledge_chunks WHERE id = ?')
251
- .get(chunkId);
252
- if (!row)
253
- return null;
254
- return {
255
- id: asChunkId(row.id),
256
- sourceId: asSourceId(row.source_id),
257
- path: row.path,
258
- text: row.text,
259
- startLine: row.start_line,
260
- endLine: row.end_line,
261
- };
262
- }
263
- /**
264
- * 按路径列出 chunks(可选行窗)
265
- */
266
- listChunksByPath(sourceId, path, opts) {
267
- const rows = this.db.raw
268
- .prepare(`SELECT id, path, text, start_line, end_line FROM knowledge_chunks
269
- WHERE source_id = ? AND path = ?
270
- ORDER BY ordinal`)
271
- .all(sourceId, path);
272
- let mapped = rows.map((r) => ({
273
- id: asChunkId(r.id),
274
- path: r.path,
275
- text: r.text,
276
- startLine: r.start_line,
277
- endLine: r.end_line,
278
- }));
279
- if (opts?.startLine != null) {
280
- mapped = mapped.filter((c) => c.endLine >= opts.startLine);
782
+ let rows = null;
783
+ const ftsIds = this.fts.search(query, opts.sourceIds, Math.max(limit * 4, 32));
784
+ if (ftsIds && ftsIds.length > 0) {
785
+ const idPh = ftsIds.map(() => '?').join(',');
786
+ rows = this.db.raw
787
+ .prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
788
+ m.source_id, m.logical_path
789
+ FROM knowledge_chunks c
790
+ JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
791
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
792
+ WHERE c.id IN (${idPh}) AND m.source_id IN (${placeholders})`)
793
+ .all(...ftsIds, ...opts.sourceIds);
281
794
  }
282
- if (opts?.endLine != null) {
283
- mapped = mapped.filter((c) => c.startLine <= opts.endLine);
795
+ if (!rows || rows.length === 0) {
796
+ const likeClauses = [];
797
+ const likeParams = [];
798
+ for (const token of tokens) {
799
+ const like = `%${token.toLowerCase().replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
800
+ likeClauses.push(`LOWER(c.text) LIKE ? ESCAPE '\\'`);
801
+ likeParams.push(like);
802
+ likeClauses.push(`LOWER(m.logical_path) LIKE ? ESCAPE '\\'`);
803
+ likeParams.push(like);
804
+ }
805
+ rows = this.db.raw
806
+ .prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
807
+ m.source_id, m.logical_path
808
+ FROM knowledge_chunks c
809
+ JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
810
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
811
+ WHERE m.source_id IN (${placeholders})
812
+ AND (${likeClauses.join(' OR ')})
813
+ LIMIT ?`)
814
+ .all(...opts.sourceIds, ...likeParams, Math.max(limit * 8, 64));
284
815
  }
285
- return mapped;
816
+ const byChunk = new Map();
817
+ for (const r of rows) {
818
+ let h = byChunk.get(r.id);
819
+ if (!h) {
820
+ if (byChunk.size >= limit * 3)
821
+ continue;
822
+ const score = scoreKeywordCoverage({ content: r.text, tags: r.logical_path }, tokenDetail);
823
+ h = {
824
+ chunkId: r.id,
825
+ fileId: r.file_id,
826
+ sourceIds: [],
827
+ logicalPaths: {},
828
+ path: r.logical_path,
829
+ ordinal: r.ordinal,
830
+ text: r.text,
831
+ startLine: r.start_line,
832
+ endLine: r.end_line,
833
+ score: score ?? 0,
834
+ };
835
+ byChunk.set(r.id, h);
836
+ }
837
+ if (!h.sourceIds.includes(r.source_id))
838
+ h.sourceIds.push(r.source_id);
839
+ h.logicalPaths[r.source_id] = r.logical_path;
840
+ }
841
+ return [...byChunk.values()]
842
+ .sort((a, b) => b.score - a.score)
843
+ .slice(0, limit)
844
+ .map((h) => ({
845
+ chunkId: asChunkId(h.chunkId),
846
+ sourceId: asSourceId(h.sourceIds[0] ?? ''),
847
+ path: h.path,
848
+ ordinal: h.ordinal,
849
+ text: h.text,
850
+ startLine: h.startLine,
851
+ endLine: h.endLine,
852
+ score: h.score,
853
+ // 契约 §8.4:共享 File 多绑定溯源
854
+ sourceIds: h.sourceIds,
855
+ logicalPaths: h.logicalPaths,
856
+ }));
286
857
  }
287
- /**
288
- * 写入 chunk 向量(JSON 冗余;P3 用 JS cosine)
289
- */
290
- setChunkEmbedding(chunkId, embedding) {
291
- this.db.raw
292
- .prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding_json, created_at)
293
- VALUES (?, ?, ?, ?)
294
- ON CONFLICT(chunk_id) DO UPDATE SET
295
- dimensions=excluded.dimensions,
296
- embedding_json=excluded.embedding_json,
297
- created_at=excluded.created_at`)
298
- .run(chunkId, embedding.length, JSON.stringify(embedding), Date.now());
858
+ vectorBackend() {
859
+ if (this.countEmbeddings() === 0)
860
+ return 'disabled';
861
+ return this.db.sqliteVecEnabled ? 'sqlite-vec' : 'js-bucket';
299
862
  }
300
- /**
301
- * 列出待嵌入 chunk(无向量行)
302
- */
303
- listChunksMissingEmbedding(sourceId, limit = 100) {
863
+ vectorSearch(embedding, opts) {
864
+ const sourceIds = opts.sourceIds.map((s) => String(s));
865
+ if (sourceIds.length === 0)
866
+ return [];
867
+ const limit = opts.limit ?? 8;
868
+ const emb = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
869
+ const placeholders = sourceIds.map(() => '?').join(',');
870
+ const joinChunks = `JOIN knowledge_chunks c ON c.id = %s
871
+ JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
872
+ JOIN knowledge_memberships m ON m.file_id = c.file_id`;
873
+ const where = `WHERE m.source_id IN (${placeholders})`;
874
+ const load = (ids, scores) => {
875
+ if (ids.length === 0)
876
+ return [];
877
+ const idPh = ids.map(() => '?').join(',');
878
+ const rows = this.db.raw
879
+ .prepare(`SELECT c.id, c.ordinal, c.text, c.start_line, c.end_line,
880
+ m.source_id, m.logical_path
881
+ FROM knowledge_chunks c
882
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
883
+ WHERE c.id IN (${idPh})`)
884
+ .all(...ids);
885
+ // 保持 scores 顺序(SQL IN 不保证序)
886
+ const byId = new Map(rows.map((r) => [r.id, r]));
887
+ const ordered = (scores ? ids : ids).map((id) => byId.get(id)).filter(Boolean);
888
+ return ordered.map((r) => ({
889
+ chunkId: asChunkId(r.id),
890
+ sourceId: asSourceId(r.source_id),
891
+ path: r.logical_path,
892
+ ordinal: r.ordinal,
893
+ text: r.text,
894
+ startLine: r.start_line,
895
+ endLine: r.end_line,
896
+ score: scores?.get(r.id) ?? 0,
897
+ }));
898
+ };
899
+ if (this.db.sqliteVecEnabled) {
900
+ // 必须走 vec0 ANN(MATCH + k)。禁止 ORDER BY vec_distance_cosine —— 那是全表暴力扫,
901
+ // 14 万向量会跑到 50s+,直接打穿 query timeout。
902
+ try {
903
+ const ph = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
904
+ // 先 ANN 过采,再按 Membership 裁剪,最后截断
905
+ const k = Math.min(Math.max(limit * 4, 64), 1024);
906
+ const knn = this.db.raw
907
+ .prepare(`SELECT c.id AS id, v.distance AS dist
908
+ FROM (
909
+ SELECT chunk_id, distance FROM knowledge_vec
910
+ WHERE embedding MATCH ? AND k = ?
911
+ ) v
912
+ ${joinChunks.replace('%s', 'v.chunk_id')} ${where}
913
+ ORDER BY v.distance LIMIT ?`)
914
+ .all(ph, k, ...sourceIds, limit);
915
+ // ANN 完成即权威结果(含 0 命中);不再退 JS 全表
916
+ const scores = new Map(knn.map((r) => [r.id, 1 - Number(r.dist ?? 1)]));
917
+ return load(knn.map((r) => r.id), scores);
918
+ }
919
+ catch {
920
+ // 扩展查询失败才落到有界 JS 路径
921
+ }
922
+ }
923
+ // 无 sqlite-vec 的有界 JS 路径:只取 id/embedding,不拉 text
304
924
  const rows = this.db.raw
305
- .prepare(`SELECT c.id, c.text, c.path
925
+ .prepare(`SELECT e.chunk_id AS chunk_id, e.embedding AS embedding
926
+ FROM knowledge_chunk_embeddings e
927
+ ${joinChunks.replace('%s', 'e.chunk_id')} ${where}
928
+ LIMIT 2000`)
929
+ .all(...sourceIds);
930
+ if (rows.length === 0)
931
+ return [];
932
+ const qArr = emb instanceof Float32Array ? emb : Float32Array.from(emb);
933
+ const scored = [];
934
+ for (const r of rows) {
935
+ const raw = r.embedding;
936
+ const buf = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
937
+ const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
938
+ if (vec.length !== qArr.length)
939
+ continue;
940
+ let dot = 0;
941
+ let nA = 0;
942
+ let nB = 0;
943
+ for (let d = 0; d < qArr.length; d++) {
944
+ const a = qArr[d];
945
+ const b = vec[d];
946
+ dot += a * b;
947
+ nA += a * a;
948
+ nB += b * b;
949
+ }
950
+ const den = Math.sqrt(nA) * Math.sqrt(nB);
951
+ const score = den === 0 ? 0 : dot / den;
952
+ if (score > 0)
953
+ scored.push({ id: r.chunk_id, score });
954
+ }
955
+ scored.sort((a, b) => b.score - a.score);
956
+ const top = scored.slice(0, limit);
957
+ const scores = new Map(top.map((s) => [s.id, s.score]));
958
+ return load(top.map((s) => s.id), scores);
959
+ }
960
+ listChunksMissingEmbedding(sourceId, limit = 50) {
961
+ const rows = this.db.raw
962
+ .prepare(`SELECT c.id AS id, c.text AS text
306
963
  FROM knowledge_chunks c
964
+ JOIN knowledge_files f ON f.id = c.file_id
965
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
307
966
  LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
308
- WHERE c.source_id = ? AND e.chunk_id IS NULL
309
- ORDER BY c.path, c.ordinal
967
+ WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
310
968
  LIMIT ?`)
311
- .all(sourceId, limit);
312
- return rows.map((r) => ({ id: asChunkId(r.id), text: r.text, path: r.path }));
969
+ .all(String(sourceId), limit);
970
+ return rows.map((r) => ({ chunkId: r.id, text: r.text }));
313
971
  }
314
- /**
315
- * 向量覆盖度(0–1;无 chunk 时为 1)
316
- */
317
- embeddingCoverage(sourceId) {
318
- const total = this.db.raw
319
- .prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
320
- .get(sourceId);
321
- if (!total?.n)
322
- return 1;
323
- const embedded = this.db.raw
324
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
325
- JOIN knowledge_chunks c ON c.id = e.chunk_id
326
- WHERE c.source_id = ?`)
327
- .get(sourceId);
328
- return Math.min(1, (embedded?.n ?? 0) / total.n);
972
+ /** 是否仍有缺向量 chunk(EXISTS,不拉正文) */
973
+ hasChunksMissingEmbedding(sourceId) {
974
+ return this.cached(`emb-miss:${sourceId}`, () => {
975
+ const row = this.db.raw
976
+ .prepare(`SELECT 1 AS ok
977
+ FROM knowledge_chunks c
978
+ JOIN knowledge_files f ON f.id = c.file_id
979
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
980
+ LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
981
+ WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
982
+ LIMIT 1`)
983
+ .get(String(sourceId));
984
+ return Boolean(row);
985
+ });
329
986
  }
330
987
  /**
331
- * 向量检索(JS cosine);embeddingJson 不可用的 chunk 跳过
988
+ * 批量写入嵌入。pairs: `[chunkId, number[]]`;空向量 = secret-skip 墓碑。
332
989
  */
333
- vectorSearch(queryEmbedding, opts) {
334
- if (opts.sourceIds.length === 0 || !queryEmbedding?.length)
335
- return [];
336
- const limit = opts.limit ?? 8;
337
- const placeholders = opts.sourceIds.map(() => '?').join(',');
338
- const rows = this.db.raw
339
- .prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.embedding_json
340
- FROM knowledge_chunks c
341
- JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
342
- WHERE c.source_id IN (${placeholders})
343
- LIMIT 20000`)
344
- .all(...opts.sourceIds);
345
- const hits = [];
346
- for (const row of rows) {
347
- let emb = null;
990
+ setChunkEmbeddings(pairs) {
991
+ if (!pairs || pairs.length === 0)
992
+ return;
993
+ this.db.ensureEmbeddingSchema();
994
+ const now = Date.now();
995
+ const insert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, bucket, created_at)
996
+ VALUES (?, ?, ?, ?, ?)
997
+ ON CONFLICT(chunk_id) DO UPDATE SET
998
+ dimensions = excluded.dimensions,
999
+ embedding = excluded.embedding,
1000
+ bucket = excluded.bucket`);
1001
+ this.db.raw.exec('BEGIN');
1002
+ try {
1003
+ for (const [chunkId, embedding] of pairs) {
1004
+ const vec = Float32Array.from(embedding ?? []);
1005
+ const dims = vec.length;
1006
+ const buf = Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
1007
+ insert.run(chunkId, dims, buf, dims > 0 ? vectorBucket(Array.from(vec)) : -1, now);
1008
+ this.writeKnowledgeVec(chunkId, embedding ?? []);
1009
+ }
1010
+ this.db.raw.exec('COMMIT');
1011
+ this.invalidateStatsCache();
1012
+ }
1013
+ catch (err) {
348
1014
  try {
349
- emb = JSON.parse(row.embedding_json);
1015
+ this.db.raw.exec('ROLLBACK');
350
1016
  }
351
1017
  catch {
352
- emb = null;
1018
+ // 以原异常为准
353
1019
  }
354
- if (!emb?.length || emb.length !== queryEmbedding.length)
355
- continue;
356
- const sim = cosineSimilarity(queryEmbedding, emb);
357
- if (sim <= 0)
358
- continue;
359
- hits.push({
360
- chunkId: asChunkId(row.id),
361
- sourceId: asSourceId(row.source_id),
362
- path: row.path,
363
- ordinal: row.ordinal,
364
- text: row.text,
365
- startLine: row.start_line,
366
- endLine: row.end_line,
367
- score: sim,
368
- });
369
- }
370
- hits.sort((a, b) => b.score - a.score);
371
- return hits.slice(0, limit);
1020
+ throw err;
1021
+ }
1022
+ }
1023
+ setChunkEmbedding(chunkId, embedding) {
1024
+ this.setChunkEmbeddings([[chunkId, embedding]]);
1025
+ }
1026
+ writeKnowledgeVec(chunkId, embedding) {
1027
+ if (!this.db.sqliteVecEnabled || !embedding?.length)
1028
+ return;
1029
+ try {
1030
+ const dims = embedding.length;
1031
+ if (!this.vecTableReady) {
1032
+ this.db.raw.exec(`
1033
+ CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_vec USING vec0(
1034
+ chunk_id TEXT PRIMARY KEY,
1035
+ embedding float[${dims}] distance_metric=cosine
1036
+ );
1037
+ `);
1038
+ this.vecTableReady = true;
1039
+ }
1040
+ this.db.raw.prepare('DELETE FROM knowledge_vec WHERE chunk_id = ?').run(chunkId);
1041
+ this.db.raw
1042
+ .prepare('INSERT INTO knowledge_vec (chunk_id, embedding) VALUES (?, ?)')
1043
+ .run(chunkId, toVecBlob(embedding));
1044
+ }
1045
+ catch {
1046
+ // 无 sqlite-vec 时忽略
1047
+ }
1048
+ }
1049
+ embeddingCoverage(sourceId) {
1050
+ return this.cached(`emb-cov:${sourceId ?? 'global'}`, () => this.computeEmbeddingCoverage(sourceId));
1051
+ }
1052
+ computeEmbeddingCoverage(sourceId) {
1053
+ if (sourceId) {
1054
+ const t = this.db.raw
1055
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
1056
+ JOIN knowledge_files f ON f.id = c.file_id
1057
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
1058
+ WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
1059
+ .get(String(sourceId)).n;
1060
+ if (t === 0)
1061
+ return 1;
1062
+ const e = this.db.raw
1063
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
1064
+ JOIN knowledge_chunks c ON c.id = e.chunk_id
1065
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
1066
+ WHERE m.source_id = ?`)
1067
+ .get(String(sourceId)).n;
1068
+ return Math.min(1, e / t);
1069
+ }
1070
+ const t = this.db.raw
1071
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
1072
+ JOIN knowledge_files f ON f.id = c.file_id WHERE ${EMBEDDABLE_WHERE}`)
1073
+ .get().n;
1074
+ if (t === 0)
1075
+ return 1;
1076
+ const e = this.db.raw
1077
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
1078
+ .get().n;
1079
+ return Math.min(1, e / t);
1080
+ }
1081
+ backfillVecFromEmbeddings(batch = 100) {
1082
+ if (!this.db.sqliteVecEnabled)
1083
+ return 0;
1084
+ const rows = this.db.raw
1085
+ .prepare(`SELECT chunk_id, embedding, dimensions FROM knowledge_chunk_embeddings
1086
+ WHERE dimensions > 0 LIMIT ?`)
1087
+ .all(batch);
1088
+ let n = 0;
1089
+ for (const r of rows) {
1090
+ const buf = r.embedding instanceof Uint8Array
1091
+ ? r.embedding
1092
+ : new Uint8Array(r.embedding);
1093
+ const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
1094
+ this.writeKnowledgeVec(r.chunk_id, vec);
1095
+ n += 1;
1096
+ }
1097
+ return n;
1098
+ }
1099
+ getChunk(chunkId) {
1100
+ const r = this.db.raw
1101
+ .prepare(`SELECT c.id, c.text, c.start_line, c.end_line, m.source_id, m.logical_path
1102
+ FROM knowledge_chunks c
1103
+ LEFT JOIN knowledge_memberships m ON m.file_id = c.file_id
1104
+ WHERE c.id = ? LIMIT 1`)
1105
+ .get(chunkId);
1106
+ if (!r)
1107
+ return null;
1108
+ return {
1109
+ id: r.id,
1110
+ sourceId: r.source_id ?? '',
1111
+ path: r.logical_path ?? '',
1112
+ text: r.text,
1113
+ startLine: r.start_line,
1114
+ endLine: r.end_line,
1115
+ };
1116
+ }
1117
+ listChunksByPath(sourceId, path) {
1118
+ const logical = path.replace(/\\/g, '/');
1119
+ const rows = this.db.raw
1120
+ .prepare(`SELECT c.id, c.text, c.start_line, c.end_line, c.ordinal
1121
+ FROM knowledge_chunks c
1122
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
1123
+ WHERE m.source_id = ? AND m.logical_path = ?
1124
+ ORDER BY c.ordinal`)
1125
+ .all(String(sourceId), logical);
1126
+ return rows.map((r) => ({
1127
+ id: r.id,
1128
+ text: r.text,
1129
+ startLine: r.start_line,
1130
+ endLine: r.end_line,
1131
+ ordinal: r.ordinal,
1132
+ path: logical,
1133
+ }));
1134
+ }
1135
+ countEmbeddings() {
1136
+ return this.db.raw
1137
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
1138
+ .get().n;
372
1139
  }
373
1140
  }
374
- /**
375
- * 内容指纹(新鲜度判断)
376
- *
377
- * @param content - UTF-8 文本或原始字节(二进制文档用 bytes)
378
- * @returns sha256 hex
379
- */
380
1141
  export function hashContent(content) {
381
1142
  return createHash('sha256').update(content).digest('hex');
382
1143
  }
383
- function rowToFile(row) {
384
- return {
385
- id: String(row.id),
386
- sourceId: asSourceId(String(row.source_id)),
387
- path: String(row.path),
388
- contentHash: String(row.content_hash ?? ''),
389
- size: Number(row.size ?? 0),
390
- mtime: Number(row.mtime ?? 0),
391
- adapterId: row.adapter_id == null ? null : String(row.adapter_id),
392
- status: String(row.status),
393
- error: row.error == null ? undefined : String(row.error),
394
- chunkCount: Number(row.chunk_count ?? 0),
395
- indexedAt: Number(row.indexed_at ?? 0),
396
- externalUrl: row.external_url == null ? undefined : String(row.external_url),
397
- etag: row.etag == null ? undefined : String(row.etag),
398
- lastModified: row.last_modified == null ? undefined : String(row.last_modified),
399
- };
400
- }