@octopi-agent/engine 0.58.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +38 -1
  4. package/dist/config-schema/substrate.d.ts +37 -0
  5. package/dist/config-schema/substrate.js +47 -0
  6. package/dist/config.d.ts +30 -1
  7. package/dist/harness/agent/builder.d.ts +14 -0
  8. package/dist/harness/agent/builder.js +14 -2
  9. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  10. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  21. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  23. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  24. package/dist/harness/capabilities/document/factory.js +37 -0
  25. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  26. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  27. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  37. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  39. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  40. package/dist/harness/capabilities/document/worker-port.js +78 -0
  41. package/dist/harness/context/capabilities/index.d.ts +2 -2
  42. package/dist/harness/context/capabilities/index.js +2 -2
  43. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  44. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  45. package/dist/harness/index.d.ts +4 -2
  46. package/dist/harness/index.js +2 -1
  47. package/dist/harness/knowledge/adapters.d.ts +8 -5
  48. package/dist/harness/knowledge/adapters.js +53 -13
  49. package/dist/harness/knowledge/client.d.ts +197 -0
  50. package/dist/harness/knowledge/client.js +237 -0
  51. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  52. package/dist/harness/knowledge/connectors.d.ts +3 -3
  53. package/dist/harness/knowledge/connectors.js +14 -8
  54. package/dist/harness/knowledge/db.d.ts +40 -5
  55. package/dist/harness/knowledge/db.js +441 -113
  56. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  57. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  58. package/dist/harness/knowledge/embed-runner.js +171 -0
  59. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  60. package/dist/harness/knowledge/engine-thread.js +302 -0
  61. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  62. package/dist/harness/knowledge/extract-document.js +32 -8
  63. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  64. package/dist/harness/knowledge/fetchers.js +52 -16
  65. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  66. package/dist/harness/knowledge/file-identity.js +172 -0
  67. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  68. package/dist/harness/knowledge/file-index-store.js +132 -0
  69. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  70. package/dist/harness/knowledge/file-limits.js +175 -0
  71. package/dist/harness/knowledge/fts.d.ts +60 -0
  72. package/dist/harness/knowledge/fts.js +218 -0
  73. package/dist/harness/knowledge/grounding.d.ts +3 -2
  74. package/dist/harness/knowledge/http-app.d.ts +50 -0
  75. package/dist/harness/knowledge/http-app.js +652 -0
  76. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  77. package/dist/harness/knowledge/http-bridge.js +17 -0
  78. package/dist/harness/knowledge/index-store.d.ts +158 -93
  79. package/dist/harness/knowledge/index-store.js +982 -441
  80. package/dist/harness/knowledge/index.d.ts +45 -5
  81. package/dist/harness/knowledge/index.js +23 -2
  82. package/dist/harness/knowledge/ingest.d.ts +262 -87
  83. package/dist/harness/knowledge/ingest.js +1072 -480
  84. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  85. package/dist/harness/knowledge/job-control-state.js +44 -0
  86. package/dist/harness/knowledge/job-control.d.ts +35 -0
  87. package/dist/harness/knowledge/job-control.js +86 -0
  88. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  89. package/dist/harness/knowledge/job-queue.js +173 -0
  90. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  91. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  92. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  93. package/dist/harness/knowledge/local-stack.js +63 -0
  94. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  95. package/dist/harness/knowledge/membership-store.js +198 -0
  96. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  97. package/dist/harness/knowledge/network-guard.js +130 -85
  98. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  99. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  100. package/dist/harness/knowledge/query-service.d.ts +272 -0
  101. package/dist/harness/knowledge/query-service.js +326 -0
  102. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  103. package/dist/harness/knowledge/query-worker-client.js +207 -0
  104. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  105. package/dist/harness/knowledge/query-worker.js +186 -0
  106. package/dist/harness/knowledge/read-http.d.ts +35 -0
  107. package/dist/harness/knowledge/read-http.js +373 -0
  108. package/dist/harness/knowledge/retriever.d.ts +22 -5
  109. package/dist/harness/knowledge/retriever.js +66 -35
  110. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  111. package/dist/harness/knowledge/secret-scan.js +25 -2
  112. package/dist/harness/knowledge/serve.d.ts +48 -0
  113. package/dist/harness/knowledge/serve.js +336 -0
  114. package/dist/harness/knowledge/source-store.d.ts +80 -17
  115. package/dist/harness/knowledge/source-store.js +160 -89
  116. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  117. package/dist/harness/knowledge/start-service-process.js +158 -0
  118. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  119. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  120. package/dist/harness/knowledge/types.d.ts +1 -1
  121. package/dist/harness/knowledge/types.js +1 -1
  122. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  123. package/dist/harness/knowledge/vector-ann.js +68 -0
  124. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  125. package/dist/harness/knowledge/writer-lock.js +71 -0
  126. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  127. package/dist/harness/knowledge/writer-service.js +355 -0
  128. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  129. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  130. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  131. package/dist/harness/knowledge/writer-worker.js +189 -0
  132. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  133. package/dist/harness/memory/sqlite/embedding.js +109 -16
  134. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  135. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  136. package/dist/harness/session/attachments/service.d.ts +1 -1
  137. package/dist/harness/session/attachments/service.js +1 -1
  138. package/dist/index.d.ts +4 -2
  139. package/dist/index.js +2 -1
  140. package/dist/worker-path.d.ts +8 -0
  141. package/dist/worker-path.js +15 -0
  142. package/package.json +2 -2
@@ -1,69 +1,263 @@
1
1
  /**
2
- * KnowledgeIndexStore — files / chunks 投影 + 关键词检索(Phase A)
2
+ * KnowledgeIndexStore — File 本位索引投影
3
3
  *
4
- * Index 非权威;可整库重跑。检索用 Memory 同款 CJK 二元组分词。
4
+ * chunk/file 挂 file_id;logical path 在 Membership。
5
+ * sourceId+path 的 API 保留为「Membership 视图」,内部按 identity 去重。
5
6
  */
6
7
  import { randomUUID, createHash } from 'node:crypto';
7
- import { tokenizeKeywordQuery, scoreKeywordFields } from '../memory/sqlite/keyword-search.js';
8
- import { cosineSimilarity } from '../memory/sqlite/vector-search.js';
8
+ import { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordCoverage, } from '../memory/sqlite/keyword-search.js';
9
+ import { KnowledgeFts } from './fts.js';
10
+ import { vectorBucket } from './vector-ann.js';
11
+ import { toVecBlob } from '../memory/sqlite/sqlite-vec.js';
9
12
  import { asChunkId, asSourceId } from './types.js';
10
- /** 可嵌入向量的 chunk 条件:非 code-tree(代码走 file_search / 原文) */
13
+ import { normalizePathLexical } from './file-identity.js';
11
14
  const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
15
+ function defaultIdentityKey(path) {
16
+ try {
17
+ return `path:${normalizePathLexical(path)}`;
18
+ }
19
+ catch {
20
+ return `path:${path.replace(/\\/g, '/')}`;
21
+ }
22
+ }
23
+ function rowToFile(row) {
24
+ return {
25
+ id: String(row.id),
26
+ sourceId: asSourceId(String(row.source_id ?? row.membership_source_id ?? '')),
27
+ path: String(row.logical_path ?? row.path ?? ''),
28
+ contentHash: String(row.content_hash ?? ''),
29
+ size: Number(row.size ?? 0),
30
+ mtime: Number(row.mtime ?? 0),
31
+ adapterId: row.adapter_id == null ? null : String(row.adapter_id),
32
+ status: String(row.status),
33
+ error: row.error == null ? undefined : String(row.error),
34
+ chunkCount: Number(row.chunk_count ?? 0),
35
+ indexedAt: Number(row.indexed_at ?? 0),
36
+ externalUrl: row.external_url == null ? undefined : String(row.external_url),
37
+ etag: row.etag == null ? undefined : String(row.etag),
38
+ lastModified: row.last_modified == null ? undefined : String(row.last_modified),
39
+ };
40
+ }
12
41
  export class KnowledgeIndexStore {
13
42
  db;
43
+ fts;
44
+ ftsBackfillPromise = null;
45
+ vecTableReady = false;
46
+ /** 同 File 写序(upsert ∥ purge):键 = identity_key */
47
+ writeLocks = new Map();
48
+ /** 大库 COUNT 短 TTL:UI 轮询 / jobControl 不必每请求全表 JOIN */
49
+ statsCache = new Map();
50
+ static STATS_TTL_MS = 1_000;
14
51
  constructor(db) {
15
52
  this.db = db;
53
+ this.fts = new KnowledgeFts(db);
54
+ }
55
+ cached(key, compute) {
56
+ const now = Date.now();
57
+ const hit = this.statsCache.get(key);
58
+ if (hit && now - hit.at < KnowledgeIndexStore.STATS_TTL_MS)
59
+ return hit.value;
60
+ const value = compute();
61
+ this.statsCache.set(key, { at: now, value });
62
+ return value;
63
+ }
64
+ /** 写路径后立刻让统计可见(收尾 force 也可走此绕过 TTL) */
65
+ invalidateStatsCache() {
66
+ this.statsCache.clear();
16
67
  }
17
68
  /**
18
- * 写入/替换某文件的 chunks(幂等:先删旧 chunk 与向量)
69
+ * 按 File identity 串行写(upsert / purge)。
70
+ * purge 在 dropChunks 让出窗口内与 upsert 交错会留下孤儿 FTS / 悬空 membership。
19
71
  */
20
- upsertFile(input) {
21
- const now = Date.now();
22
- const existing = this.db.raw
23
- .prepare('SELECT id FROM knowledge_files WHERE source_id = ? AND path = ?')
24
- .get(input.sourceId, input.path);
25
- const fileId = existing?.id ?? `kf_${randomUUID().slice(0, 12)}`;
26
- this.db.raw.exec('BEGIN');
72
+ async withFileWriteLock(key, fn) {
73
+ const prev = this.writeLocks.get(key) ?? Promise.resolve();
74
+ const run = prev.then(() => fn());
75
+ const tail = run.then(() => undefined, () => undefined);
76
+ this.writeLocks.set(key, tail);
27
77
  try {
78
+ return await run;
79
+ }
80
+ finally {
81
+ if (this.writeLocks.get(key) === tail)
82
+ this.writeLocks.delete(key);
83
+ }
84
+ }
85
+ get ftsAvailable() {
86
+ return this.fts.available;
87
+ }
88
+ get ftsBackfillRunning() {
89
+ return this.ftsBackfillPromise != null;
90
+ }
91
+ ensureFtsBackfill() {
92
+ if (!this.fts.available)
93
+ return Promise.resolve(0);
94
+ if (this.ftsBackfillPromise)
95
+ return this.ftsBackfillPromise;
96
+ const n = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks').get().n;
97
+ const ftsN = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks_fts').get().n;
98
+ if (n === 0 || ftsN > 0)
99
+ return Promise.resolve(0);
100
+ this.ftsBackfillPromise = this.fts
101
+ .rebuildFromChunksAsync()
102
+ .catch(() => 0)
103
+ .finally(() => {
104
+ this.ftsBackfillPromise = null;
105
+ });
106
+ return this.ftsBackfillPromise;
107
+ }
108
+ /** 解析或创建 File(identity 去重)+ Membership */
109
+ resolveFile(sourceId, path, meta) {
110
+ const identityKey = meta.identityKey ?? defaultIdentityKey(path);
111
+ const tenant = meta.tenantId ?? 'default';
112
+ const now = Date.now();
113
+ let row = this.db.raw
114
+ .prepare('SELECT id, size, mtime FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
115
+ .get(tenant, identityKey);
116
+ if (!row) {
117
+ const id = `kf_${randomUUID().slice(0, 12)}`;
28
118
  this.db.raw
29
- .prepare(`DELETE FROM knowledge_chunk_embeddings
30
- WHERE chunk_id IN (SELECT id FROM knowledge_chunks WHERE file_id = ?)`)
31
- .run(fileId);
32
- this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
119
+ .prepare(`INSERT INTO knowledge_files
120
+ (id, tenant_id, identity_key, size, mtime, content_hash, status, chunk_count, indexed_at)
121
+ VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, ?)`)
122
+ .run(id, tenant, identityKey, meta.size ?? 0, meta.mtime ?? 0, now);
123
+ row = { id, size: meta.size ?? 0, mtime: meta.mtime ?? 0 };
124
+ }
125
+ else if ((meta.size != null && meta.size !== row.size) ||
126
+ (meta.mtime != null && meta.mtime !== row.mtime)) {
33
127
  this.db.raw
34
- .prepare(`INSERT INTO knowledge_files (
35
- id, source_id, path, content_hash, size, mtime, adapter_id, status, error, chunk_count, indexed_at,
36
- external_url, etag, last_modified
37
- ) VALUES (?, ?, ?, ?, ?, ?, ?, 'indexed', NULL, ?, ?, ?, ?, ?)
38
- ON CONFLICT(source_id, path) DO UPDATE SET
39
- content_hash=excluded.content_hash,
40
- size=excluded.size,
41
- mtime=excluded.mtime,
42
- adapter_id=excluded.adapter_id,
43
- status='indexed',
44
- error=NULL,
45
- chunk_count=excluded.chunk_count,
46
- indexed_at=excluded.indexed_at,
47
- external_url=excluded.external_url,
48
- etag=excluded.etag,
49
- last_modified=excluded.last_modified`)
50
- .run(fileId, input.sourceId, input.path, input.contentHash, input.size, input.mtime, input.adapterId, input.chunks.length, now, input.externalUrl ?? null, input.etag ?? null, input.lastModified ?? null);
51
- const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, source_id, path, ordinal, text, start_line, end_line)
52
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
53
- for (const c of input.chunks) {
54
- insertChunk.run(`kc_${randomUUID().slice(0, 12)}`, fileId, input.sourceId, input.path, c.ordinal, c.text, c.startLine, c.endLine);
128
+ .prepare('UPDATE knowledge_files SET size = ?, mtime = ? WHERE id = ?')
129
+ .run(meta.size ?? row.size, meta.mtime ?? row.mtime, row.id);
130
+ }
131
+ const logical = path.replace(/\\/g, '/');
132
+ this.db.raw
133
+ .prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
134
+ VALUES (?, ?, ?, ?)
135
+ ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
136
+ .run(sourceId, row.id, logical, now);
137
+ return row.id;
138
+ }
139
+ /** 按 identity_key 查已有 File(不建行);供 job 入队绑定 file_id */
140
+ findFileIdByIdentity(identityKey, tenantId = 'default') {
141
+ const row = this.db.raw
142
+ .prepare('SELECT id FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
143
+ .get(tenantId, identityKey);
144
+ return row?.id ?? null;
145
+ }
146
+ /** 只挂 Membership(共享 File:已有 parse 在途时补认领,禁止双 parse) */
147
+ attachMembership(sourceId, fileId, path) {
148
+ const logical = path.replace(/\\/g, '/');
149
+ this.db.raw
150
+ .prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
151
+ VALUES (?, ?, ?, ?)
152
+ ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
153
+ .run(sourceId, fileId, logical, Date.now());
154
+ }
155
+ /**
156
+ * 替换文件索引。chunk/FTS **分批**写入并让出事件循环,
157
+ * 大 xlsx 数千 chunk 不得整文件一个同步事务堵死 Service。
158
+ *
159
+ * 语义:旧 chunk 先清;`status='indexing'` 直到全部批次落库后才 `indexed`。
160
+ * 检索不认 `indexing`(半成品)。批次失败/中止会清掉 partial chunks,
161
+ * 并留在 `indexing`(chunk_count=0)供 gap 扫描重试——不得把半成品标成 `error`
162
+ * 而让 partial 变得可搜。
163
+ */
164
+ async upsertFile(input, opts) {
165
+ const lockKey = input.identityKey ?? defaultIdentityKey(input.path);
166
+ return this.withFileWriteLock(lockKey, async () => {
167
+ return this.upsertFileLocked(input, opts);
168
+ });
169
+ }
170
+ async upsertFileLocked(input, opts) {
171
+ const now = Date.now();
172
+ // 小批量 + 批间让出:200/批 的 FTS 写在 8 文件并发 parse 时仍会长时间占住 Engine
173
+ const batchSize = Math.max(20, opts?.batchSize ?? 50);
174
+ const signal = opts?.signal;
175
+ const fileId = this.resolveFile(String(input.sourceId), input.path, {
176
+ identityKey: input.identityKey,
177
+ size: input.size,
178
+ mtime: input.mtime,
179
+ });
180
+ if (signal?.aborted) {
181
+ await this.markUpsertIncomplete(fileId, 'aborted');
182
+ throw new Error('aborted');
183
+ }
184
+ // 旧索引作废(含 embeddings/FTS/vec)— 分批让出,禁止同步连环 DELETE 冻住 Engine
185
+ {
186
+ const oldIds = this.db.raw
187
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
188
+ .all(fileId).map((r) => r.id);
189
+ await this.dropChunksAsync(oldIds);
190
+ this.db.raw.exec('BEGIN');
191
+ try {
192
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
193
+ this.db.raw
194
+ .prepare(`UPDATE knowledge_files
195
+ SET content_hash = ?, size = ?, mtime = ?, adapter_id = ?,
196
+ status = 'indexing', error = NULL, chunk_count = 0, indexed_at = ?,
197
+ external_url = ?, etag = ?, last_modified = ?
198
+ WHERE id = ?`)
199
+ .run(input.contentHash, input.size, input.mtime, input.adapterId, now, input.externalUrl ?? null, input.etag ?? null, input.lastModified ?? null, fileId);
200
+ this.db.raw.exec('COMMIT');
201
+ }
202
+ catch (err) {
203
+ try {
204
+ this.db.raw.exec('ROLLBACK');
205
+ }
206
+ catch {
207
+ // 以原异常为准
208
+ }
209
+ throw err;
55
210
  }
56
- this.db.raw.exec('COMMIT');
57
211
  }
58
- catch (err) {
212
+ for (let offset = 0; offset < input.chunks.length; offset += batchSize) {
213
+ if (signal?.aborted) {
214
+ await this.markUpsertIncomplete(fileId, 'aborted');
215
+ throw new Error('aborted');
216
+ }
217
+ const batch = input.chunks.slice(offset, offset + batchSize);
218
+ this.db.raw.exec('BEGIN');
59
219
  try {
60
- this.db.raw.exec('ROLLBACK');
220
+ const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, ordinal, text, start_line, end_line)
221
+ VALUES (?, ?, ?, ?, ?, ?)`);
222
+ const ftsRows = [];
223
+ for (const c of batch) {
224
+ const cid = `kc_${randomUUID().slice(0, 12)}`;
225
+ insertChunk.run(cid, fileId, c.ordinal, c.text, c.startLine, c.endLine);
226
+ ftsRows.push({
227
+ id: cid,
228
+ text: c.text,
229
+ path: input.path,
230
+ ...(c.ftsToks != null ? { toks: c.ftsToks } : {}),
231
+ });
232
+ }
233
+ this.fts.upsertMany(ftsRows);
234
+ this.db.raw.exec('COMMIT');
61
235
  }
62
- catch {
63
- // rollback 失败仅吞掉:上层以原异常为准,避免掩盖根因
236
+ catch (err) {
237
+ try {
238
+ this.db.raw.exec('ROLLBACK');
239
+ }
240
+ catch {
241
+ // 以原异常为准
242
+ }
243
+ const message = err instanceof Error ? err.message : String(err);
244
+ await this.markUpsertIncomplete(fileId, message);
245
+ throw err;
246
+ }
247
+ if (offset + batchSize < input.chunks.length) {
248
+ await new Promise((r) => setImmediate(r));
64
249
  }
65
- throw err;
66
250
  }
251
+ if (signal?.aborted) {
252
+ await this.markUpsertIncomplete(fileId, 'aborted');
253
+ throw new Error('aborted');
254
+ }
255
+ this.db.raw
256
+ .prepare(`UPDATE knowledge_files
257
+ SET status = 'indexed', error = NULL, chunk_count = ?, indexed_at = ?
258
+ WHERE id = ?`)
259
+ .run(input.chunks.length, now, fileId);
260
+ this.invalidateStatsCache();
67
261
  return {
68
262
  id: fileId,
69
263
  sourceId: input.sourceId,
@@ -81,203 +275,491 @@ export class KnowledgeIndexStore {
81
275
  };
82
276
  }
83
277
  /**
84
- * 标记 skipped — **不删** 已有 chunks(瞬时/边界失败保旧索引);仅元数据。
278
+ * 半写入收尾:清掉已落地的 partial chunks,回到可重试的 `indexing`。
279
+ * 不得标成 `error`——error 行上的 partial 会被检索捞出。
85
280
  */
86
- markFileSkipped(sourceId, path, reason) {
87
- this.markFileNonIndexed(sourceId, path, 'skipped', reason);
281
+ async markUpsertIncomplete(fileId, message) {
282
+ try {
283
+ const ids = this.db.raw
284
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
285
+ .all(fileId).map((r) => r.id);
286
+ await this.dropChunksAsync(ids);
287
+ this.db.raw.exec('BEGIN');
288
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
289
+ this.db.raw
290
+ .prepare(`UPDATE knowledge_files
291
+ SET status = 'indexing', chunk_count = 0, error = ?, indexed_at = ?
292
+ WHERE id = ?`)
293
+ .run(message, Date.now(), fileId);
294
+ this.db.raw.exec('COMMIT');
295
+ }
296
+ catch {
297
+ try {
298
+ this.db.raw.exec('ROLLBACK');
299
+ }
300
+ catch {
301
+ // 清理失败时至少保证调用方仍看到原异常;半成品由 gap 扫描按 indexing 重做
302
+ }
303
+ }
304
+ }
305
+ markFileSkipped(sourceId, path, reason, size, identityKey) {
306
+ this.markFileNonIndexed(sourceId, path, 'skipped', reason, size, identityKey);
307
+ }
308
+ markFileError(sourceId, path, error, size, identityKey) {
309
+ this.markFileNonIndexed(sourceId, path, 'error', error, size, identityKey);
88
310
  }
89
311
  /**
90
- * 标记 error — **不删** 已有 chunks(网络抖动不得清库);仅元数据。
91
- * 成功 `upsertFile` 才替换 chunks。
312
+ * 标记 skipped/error — **不删** 已有 chunks。
313
+ * 若已有 membership,只改状态;**不得**用 path-identity 新建 File 顶掉带 chunk 的行。
92
314
  */
93
- markFileError(sourceId, path, error) {
94
- this.markFileNonIndexed(sourceId, path, 'error', error);
95
- }
96
- markFileNonIndexed(sourceId, path, status, detail) {
97
- const now = Date.now();
315
+ markFileNonIndexed(sourceId, path, status, detail, size, identityKey) {
316
+ const logical = path.replace(/\\/g, '/');
98
317
  const existing = this.db.raw
99
- .prepare('SELECT id, chunk_count FROM knowledge_files WHERE source_id = ? AND path = ?')
100
- .get(sourceId, path);
101
- const fileId = existing?.id ?? `kf_${randomUUID().slice(0, 12)}`;
102
- const chunkCount = existing?.chunk_count ?? 0;
318
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
319
+ .get(String(sourceId), logical);
320
+ if (existing?.file_id) {
321
+ this.db.raw
322
+ .prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
323
+ .run(status, detail, Date.now(), existing.file_id);
324
+ return;
325
+ }
326
+ // 优先真实 identity(win:/unix:),避免 path: 与成功路径分裂出零认领行
327
+ const fileId = this.resolveFile(String(sourceId), path, { size, identityKey });
103
328
  this.db.raw
104
- .prepare(`INSERT INTO knowledge_files (
105
- id, source_id, path, content_hash, size, mtime, adapter_id, status, error, chunk_count, indexed_at
106
- ) VALUES (?, ?, ?, '', 0, 0, NULL, ?, ?, ?, ?)
107
- ON CONFLICT(source_id, path) DO UPDATE SET
108
- status=excluded.status,
109
- error=excluded.error,
110
- indexed_at=excluded.indexed_at`)
111
- .run(fileId, sourceId, path, status, detail, chunkCount, now);
329
+ .prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
330
+ .run(status, detail, Date.now(), fileId);
112
331
  }
113
332
  /**
114
- * 文件是否已用同一 contentHash 索引(且确有 chunk,防半写入)
333
+ * 清零认领的 path: 降级 File(错误路径与成功路径 identity 分裂残留)。
334
+ *
335
+ * @returns 删除的 File 行数
115
336
  */
337
+ purgeOrphanPathIdentityFiles() {
338
+ const res = this.db.raw
339
+ .prepare(`DELETE FROM knowledge_files
340
+ WHERE identity_key LIKE 'path:%'
341
+ AND NOT EXISTS (
342
+ SELECT 1 FROM knowledge_memberships m WHERE m.file_id = knowledge_files.id
343
+ )`)
344
+ .run();
345
+ return Number(res.changes ?? 0);
346
+ }
116
347
  isFresh(sourceId, path, contentHash) {
348
+ const logical = path.replace(/\\/g, '/');
349
+ const row = this.db.raw
350
+ .prepare(`SELECT f.content_hash AS content_hash, f.chunk_count AS chunk_count
351
+ FROM knowledge_memberships m
352
+ JOIN knowledge_files f ON f.id = m.file_id
353
+ WHERE m.source_id = ? AND m.logical_path = ? AND f.status <> 'indexing'`)
354
+ .get(String(sourceId), logical);
355
+ return row?.content_hash === contentHash && (row?.chunk_count ?? 0) > 0;
356
+ }
357
+ invalidateFileForReparse(sourceId, path) {
358
+ const logical = path.replace(/\\/g, '/');
117
359
  const row = this.db.raw
118
- .prepare(`SELECT content_hash, chunk_count FROM knowledge_files
119
- WHERE source_id = ? AND path = ? AND status = 'indexed'`)
120
- .get(sourceId, path);
121
- return (row?.content_hash === contentHash &&
122
- (row?.chunk_count ?? 0) > 0);
360
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
361
+ .get(String(sourceId), logical);
362
+ if (!row)
363
+ return;
364
+ this.db.raw
365
+ .prepare(`UPDATE knowledge_files SET content_hash = '' WHERE id = ?`)
366
+ .run(row.file_id);
123
367
  }
124
368
  getFile(sourceId, path) {
369
+ const logical = path.replace(/\\/g, '/');
125
370
  const row = this.db.raw
126
- .prepare('SELECT * FROM knowledge_files WHERE source_id = ? AND path = ?')
127
- .get(sourceId, path);
371
+ .prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
372
+ FROM knowledge_memberships m
373
+ JOIN knowledge_files f ON f.id = m.file_id
374
+ WHERE m.source_id = ? AND m.logical_path = ?`)
375
+ .get(String(sourceId), logical);
128
376
  return row ? rowToFile(row) : null;
129
377
  }
130
378
  listFiles(sourceId) {
131
379
  const rows = this.db.raw
132
- .prepare('SELECT * FROM knowledge_files WHERE source_id = ? ORDER BY path')
133
- .all(sourceId);
380
+ .prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
381
+ FROM knowledge_memberships m
382
+ JOIN knowledge_files f ON f.id = m.file_id
383
+ WHERE m.source_id = ? ORDER BY m.logical_path`)
384
+ .all(String(sourceId));
134
385
  return rows.map(rowToFile);
135
386
  }
387
+ listFilesPaged(sourceId, opts) {
388
+ const pageSize = Math.max(1, Math.min(200, opts?.pageSize ?? 50));
389
+ const page = Math.max(1, opts?.page ?? 1);
390
+ const status = opts?.status && opts.status !== 'all' ? opts.status : null;
391
+ const ext = opts?.ext && opts.ext !== 'all'
392
+ ? opts.ext.replace(/^\./, '').toLowerCase()
393
+ : null;
394
+ const q = opts?.q?.trim().toLowerCase() || null;
395
+ // SQL 过滤 + LIMIT/OFFSET:禁止全量拉表再在内存分页(1k+ 文件时 UI 轮询会拖垮 Engine/回包)
396
+ const where = ['m.source_id = ?'];
397
+ const params = [String(sourceId)];
398
+ if (status) {
399
+ where.push('f.status = ?');
400
+ params.push(status);
401
+ }
402
+ if (ext) {
403
+ where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
404
+ params.push(`%.${ext.replace(/[\\%_]/g, (c) => `\\${c}`)}`);
405
+ }
406
+ if (q) {
407
+ where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
408
+ params.push(`%${q.replace(/[\\%_]/g, (c) => `\\${c}`)}%`);
409
+ }
410
+ const whereSql = where.join(' AND ');
411
+ const totalRow = this.db.raw
412
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_memberships m
413
+ JOIN knowledge_files f ON f.id = m.file_id
414
+ WHERE ${whereSql}`)
415
+ .get(...params);
416
+ const rows = this.db.raw
417
+ .prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
418
+ FROM knowledge_memberships m
419
+ JOIN knowledge_files f ON f.id = m.file_id
420
+ WHERE ${whereSql}
421
+ ORDER BY m.logical_path
422
+ LIMIT ? OFFSET ?`)
423
+ .all(...params, pageSize, (page - 1) * pageSize);
424
+ // counts 走 SQL 聚合:禁止把整源 logical_path 拉进 JS 再计数(UI 每页轮询会打满 Engine)
425
+ const statusCounts = { indexed: 0, skipped: 0, error: 0 };
426
+ const statusRows = this.db.raw
427
+ .prepare(`SELECT f.status AS status, COUNT(*) AS n
428
+ FROM knowledge_memberships m
429
+ JOIN knowledge_files f ON f.id = m.file_id
430
+ WHERE m.source_id = ?
431
+ GROUP BY f.status`)
432
+ .all(String(sourceId));
433
+ for (const r of statusRows) {
434
+ if (r.status === 'indexed' || r.status === 'skipped' || r.status === 'error') {
435
+ statusCounts[r.status] = Number(r.n ?? 0);
436
+ }
437
+ }
438
+ // ext:basename → 后缀(目录名带点不影响);JSON split 规避 SQLite 无 rsplit
439
+ const extRows = this.db.raw
440
+ .prepare(`SELECT lower(ext) AS ext, COUNT(*) AS n FROM (
441
+ SELECT
442
+ CASE
443
+ WHEN instr(name, '.') = 0 OR name IS NULL THEN ''
444
+ ELSE json_extract('["' || replace(name, '.', '","') || '"]', '$[#-1]')
445
+ END AS ext
446
+ FROM (
447
+ SELECT
448
+ json_extract(
449
+ '["' || replace(replace(COALESCE(m.logical_path, ''), char(92), '","'), '/', '","') || '"]',
450
+ '$[#-1]'
451
+ ) AS name
452
+ FROM knowledge_memberships m
453
+ WHERE m.source_id = ?
454
+ )
455
+ )
456
+ WHERE ext IS NOT NULL AND ext != ''
457
+ GROUP BY ext`)
458
+ .all(String(sourceId));
459
+ const extCounts = extRows
460
+ .filter((r) => r.ext)
461
+ .map((r) => ({ ext: r.ext, n: Number(r.n ?? 0) }));
462
+ const items = rows.map((row) => {
463
+ const f = rowToFile(row);
464
+ const extName = f.path.includes('.')
465
+ ? (f.path.split('.').pop() ?? '').toLowerCase()
466
+ : '';
467
+ return { ...f, ext: extName };
468
+ });
469
+ return {
470
+ items,
471
+ total: Number(totalRow?.n ?? 0),
472
+ page,
473
+ pageSize,
474
+ statusCounts,
475
+ extCounts,
476
+ };
477
+ }
478
+ listFilePathsFiltered(sourceId, filter) {
479
+ const rows = this.db.raw
480
+ .prepare(`SELECT m.logical_path AS path, f.status AS status
481
+ FROM knowledge_memberships m
482
+ JOIN knowledge_files f ON f.id = m.file_id
483
+ WHERE m.source_id = ? ORDER BY m.logical_path`)
484
+ .all(String(sourceId));
485
+ let out = rows.map((r) => r.path);
486
+ if (filter?.status) {
487
+ out = rows.filter((r) => r.status === filter.status).map((r) => r.path);
488
+ }
489
+ if (filter?.ext && filter.ext !== 'all') {
490
+ const ext = filter.ext.replace(/^\./, '').toLowerCase();
491
+ out = out.filter((p) => p.toLowerCase().endsWith(`.${ext}`));
492
+ }
493
+ if (filter?.max != null)
494
+ out = out.slice(0, filter.max);
495
+ return out;
496
+ }
497
+ /** 解绑 path;File 仅在零认领时 purge(单文件同步语义,供测试/轻量路径) */
498
+ removeFile(sourceId, path) {
499
+ const logical = path.replace(/\\/g, '/');
500
+ const row = this.db.raw
501
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
502
+ .get(String(sourceId), logical);
503
+ this.db.raw
504
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
505
+ .run(String(sourceId), logical);
506
+ if (!row)
507
+ return;
508
+ const n = this.db.raw
509
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
510
+ .get(row.file_id).n;
511
+ if (n === 0)
512
+ this.purgeFileRow(row.file_id);
513
+ }
136
514
  /**
137
- * 单文件 purge(合规):chunks + embeddings + files 行
515
+ * 解绑 path + 可能的 purge(异步让出)。
516
+ * 单文件 purge 也可能删数千 chunk——必须让出,否则 Engine 调度不了 HTTP 回包。
138
517
  */
139
- purgePath(sourceId, path) {
140
- this.removeFile(sourceId, path);
518
+ async removeFileAsync(sourceId, path, opts) {
519
+ const logical = path.replace(/\\/g, '/');
520
+ const row = this.db.raw
521
+ .prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
522
+ .get(String(sourceId), logical);
523
+ this.db.raw
524
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
525
+ .run(String(sourceId), logical);
526
+ if (!row)
527
+ return;
528
+ const n = this.db.raw
529
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
530
+ .get(row.file_id).n;
531
+ if (n === 0)
532
+ await this.purgeFileRowAsync(row.file_id, opts);
141
533
  }
142
534
  /**
143
- * 源级 purge
535
+ * 批量解绑:每 `yieldEvery` 条让出事件循环。
536
+ * 删大目录(数百文件)时同步连环 DELETE 会堵死 Engine,HTTP/Query 回包都无法调度。
144
537
  */
145
- purgeSource(sourceId) {
146
- this.clearSource(sourceId);
538
+ async removeFilesAsync(sourceId, paths, opts) {
539
+ const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
540
+ let n = 0;
541
+ for (let i = 0; i < paths.length; i++) {
542
+ await this.removeFileAsync(sourceId, paths[i], opts);
543
+ n += 1;
544
+ if ((i + 1) % yieldEvery === 0) {
545
+ await new Promise((r) => setImmediate(r));
546
+ }
547
+ }
548
+ return n;
549
+ }
550
+ removeFiles(sourceId, paths) {
551
+ for (const p of paths)
552
+ this.removeFile(sourceId, p);
147
553
  }
148
554
  /**
149
- * 删除源下全部 index(卸载 / rebuild)
555
+ * 删除 path 及子树:先收集 logical_path,再分批异步解绑 + purge。
150
556
  */
151
- clearSource(sourceId) {
152
- this.db.raw
153
- .prepare(`DELETE FROM knowledge_chunk_embeddings
154
- WHERE chunk_id IN (SELECT id FROM knowledge_chunks WHERE source_id = ?)`)
155
- .run(sourceId);
156
- this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE source_id = ?').run(sourceId);
157
- this.db.raw.prepare('DELETE FROM knowledge_files WHERE source_id = ?').run(sourceId);
557
+ async removePathTreeAsync(sourceId, path, opts) {
558
+ const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
559
+ const rows = this.db.raw
560
+ .prepare(`SELECT logical_path FROM knowledge_memberships
561
+ WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
562
+ .all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
563
+ return this.removeFilesAsync(sourceId, rows.map((r) => r.logical_path), opts);
158
564
  }
159
- removeFile(sourceId, path) {
160
- this.db.raw
161
- .prepare(`DELETE FROM knowledge_chunk_embeddings
162
- WHERE chunk_id IN (
163
- SELECT id FROM knowledge_chunks WHERE source_id = ? AND path = ?
164
- )`)
165
- .run(sourceId, path);
166
- this.db.raw
167
- .prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?')
168
- .run(sourceId, path);
169
- this.db.raw
170
- .prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?')
171
- .run(sourceId, path);
565
+ removePathTree(sourceId, path) {
566
+ const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
567
+ const rows = this.db.raw
568
+ .prepare(`SELECT logical_path FROM knowledge_memberships
569
+ WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
570
+ .all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
571
+ for (const r of rows)
572
+ this.removeFile(sourceId, r.logical_path);
172
573
  }
173
574
  /**
174
- * 批量删除 path(同一事务,减少 sync 往返)
175
- *
176
- * @param sourceId - 知识源 id
177
- * @param paths - 精确 path 列表
575
+ * 清空源认领:分批 purge,避免一次清数千 File 堵死事件循环。
178
576
  */
179
- removeFiles(sourceId, paths) {
180
- if (paths.length === 0)
181
- return;
182
- const chunkIds = [];
183
- const select = this.db.raw.prepare(`SELECT id FROM knowledge_chunks WHERE source_id = ? AND path = ?`);
184
- for (const p of paths) {
185
- const rows = select.all(sourceId, p);
186
- for (const r of rows)
187
- chunkIds.push(r.id);
188
- }
189
- this.db.raw.exec('BEGIN');
190
- try {
191
- if (chunkIds.length > 0) {
192
- const delEmb = this.db.raw.prepare('DELETE FROM knowledge_chunk_embeddings WHERE chunk_id = ?');
193
- for (const id of chunkIds)
194
- delEmb.run(id);
577
+ async clearSourceAsync(sourceId, opts) {
578
+ const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
579
+ const rows = this.db.raw
580
+ .prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
581
+ .all(String(sourceId));
582
+ this.db.raw
583
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
584
+ .run(String(sourceId));
585
+ let purged = 0;
586
+ for (let i = 0; i < rows.length; i++) {
587
+ const fileId = rows[i].file_id;
588
+ const n = this.db.raw
589
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
590
+ .get(fileId).n;
591
+ if (n === 0) {
592
+ await this.purgeFileRowAsync(fileId, opts);
593
+ purged += 1;
195
594
  }
196
- const delChunks = this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?');
197
- const delFiles = this.db.raw.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?');
198
- for (const p of paths) {
199
- delChunks.run(sourceId, p);
200
- delFiles.run(sourceId, p);
595
+ if ((i + 1) % yieldEvery === 0) {
596
+ await new Promise((r) => setImmediate(r));
201
597
  }
202
- this.db.raw.exec('COMMIT');
203
598
  }
204
- catch (e) {
205
- this.db.raw.exec('ROLLBACK');
206
- throw e;
599
+ return purged;
600
+ }
601
+ clearSource(sourceId) {
602
+ const rows = this.db.raw
603
+ .prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
604
+ .all(String(sourceId));
605
+ this.db.raw
606
+ .prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
607
+ .run(String(sourceId));
608
+ for (const r of rows) {
609
+ const n = this.db.raw
610
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
611
+ .get(r.file_id).n;
612
+ if (n === 0)
613
+ this.purgeFileRow(r.file_id);
207
614
  }
208
615
  }
616
+ purgePath(sourceId, path) {
617
+ this.removeFile(sourceId, path);
618
+ }
619
+ /** 按 fileId 物理删除(membership 已清) */
620
+ purgeFile(fileId) {
621
+ this.purgeFileRow(fileId);
622
+ }
623
+ async purgeFileAsync(fileId, opts) {
624
+ await this.purgeFileRowAsync(fileId, opts);
625
+ }
626
+ purgeSource(sourceId) {
627
+ this.clearSource(sourceId);
628
+ }
629
+ purgeFileRow(fileId) {
630
+ const oldIds = this.db.raw
631
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
632
+ .all(fileId).map((r) => r.id);
633
+ this.dropChunks(oldIds);
634
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
635
+ this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
636
+ this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
637
+ }
209
638
  /**
210
- * 删除路径及子树(目录被移出 / 删除)
211
- *
212
- * @param sourceId - 知识源 id
213
- * @param path - 文件或目录绝对路径
639
+ * purge + 分批让出。大文件数千 chunk 时 dropChunks 同步段仍可能秒级——批间 setImmediate。
640
+ * 与 upsertFile 共用 File 写锁,避免让出窗口被 parse 插入后残留孤儿 FTS。
214
641
  */
215
- removePathTree(sourceId, path) {
216
- this.db.raw.exec('BEGIN');
642
+ async purgeFileRowAsync(fileId, opts) {
643
+ const keyRow = this.db.raw
644
+ .prepare('SELECT identity_key FROM knowledge_files WHERE id = ?')
645
+ .get(fileId);
646
+ const lockKey = keyRow?.identity_key ?? `id:${fileId}`;
647
+ await this.withFileWriteLock(lockKey, async () => {
648
+ const oldIds = this.db.raw
649
+ .prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
650
+ .all(fileId).map((r) => r.id);
651
+ await this.dropChunksAsync(oldIds);
652
+ this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
653
+ this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
654
+ this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
655
+ await new Promise((r) => setImmediate(r));
656
+ });
657
+ void opts;
658
+ }
659
+ dropChunks(chunkIds) {
660
+ if (chunkIds.length === 0)
661
+ return;
662
+ // 分批:大文件重解析时数千 id 一次 IN 会长时间占住 Engine 事件循环
663
+ for (let i = 0; i < chunkIds.length; i += 400) {
664
+ this.dropChunksBatch(chunkIds.slice(i, i + 400));
665
+ }
666
+ }
667
+ async dropChunksAsync(chunkIds) {
668
+ if (chunkIds.length === 0)
669
+ return;
670
+ for (let i = 0; i < chunkIds.length; i += 400) {
671
+ this.dropChunksBatch(chunkIds.slice(i, i + 400));
672
+ // 批间必须让出:否则单文件数千 chunk 仍会冻住 Engine 数秒
673
+ await new Promise((r) => setImmediate(r));
674
+ }
675
+ }
676
+ dropChunksBatch(batch) {
677
+ if (batch.length === 0)
678
+ return;
679
+ const ph = batch.map(() => '?').join(',');
680
+ this.db.raw
681
+ .prepare(`DELETE FROM knowledge_chunk_embeddings WHERE chunk_id IN (${ph})`)
682
+ .run(...batch);
683
+ this.deleteKnowledgeVec(batch);
684
+ this.fts.removeMany(batch);
685
+ this.invalidateStatsCache();
686
+ }
687
+ deleteKnowledgeVec(chunkIds) {
688
+ if (!this.db.sqliteVecEnabled || chunkIds.length === 0)
689
+ return;
217
690
  try {
218
- this.removeFile(sourceId, path);
219
- const likeSlash = `${path}/%`;
220
- const likeBack = `${path}\\%`;
221
- this.db.raw
222
- .prepare(`DELETE FROM knowledge_chunk_embeddings
223
- WHERE chunk_id IN (
224
- SELECT id FROM knowledge_chunks
225
- WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)
226
- )`)
227
- .run(sourceId, likeSlash, likeBack);
228
- this.db.raw
229
- .prepare(`DELETE FROM knowledge_chunks WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
230
- .run(sourceId, likeSlash, likeBack);
231
- this.db.raw
232
- .prepare(`DELETE FROM knowledge_files WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
233
- .run(sourceId, likeSlash, likeBack);
234
- this.db.raw.exec('COMMIT');
691
+ const del = this.db.raw.prepare(`DELETE FROM knowledge_vec WHERE chunk_id IN (${chunkIds.map(() => '?').join(',')})`);
692
+ for (let i = 0; i < chunkIds.length; i += 400) {
693
+ del.run(...chunkIds.slice(i, i + 400));
694
+ }
235
695
  }
236
- catch (e) {
237
- this.db.raw.exec('ROLLBACK');
238
- throw e;
696
+ catch {
697
+ // 无 vec 表时忽略
239
698
  }
240
699
  }
241
700
  /**
242
- * 差量 prune:删除本轮 discover **未出现** 的 path(含 chunks)。
243
- * 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
701
+ * 对账差量删除:批量路径,内部让出事件循环。
244
702
  *
245
- * @param keepPaths - 应保留的 path 集
246
- * @param opts.allowEmptyKeep - walk 成功且目录为空时允许清空(默认 false)
247
- * @returns 删除的文件数
703
+ * @returns 删除条数
248
704
  */
705
+ async pruneMissingAsync(sourceId, keepPaths, opts) {
706
+ // keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
707
+ const normalize = (p) => p.replace(/\\/g, '/');
708
+ const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
709
+ // discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
710
+ if (keep.size === 0 && !opts?.allowEmptyKeep)
711
+ return 0;
712
+ const files = this.listFiles(sourceId);
713
+ const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
714
+ if (gone.length === 0)
715
+ return 0;
716
+ await this.removeFilesAsync(sourceId, gone, opts);
717
+ return gone.length;
718
+ }
249
719
  pruneMissing(sourceId, keepPaths, opts) {
250
- const keep = keepPaths instanceof Set ? keepPaths : new Set(keepPaths);
720
+ // keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
721
+ const normalize = (p) => p.replace(/\\/g, '/');
722
+ const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
723
+ // discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
251
724
  if (keep.size === 0 && !opts?.allowEmptyKeep)
252
725
  return 0;
253
726
  const files = this.listFiles(sourceId);
254
- const gone = files.filter((f) => !keep.has(f.path)).map((f) => f.path);
727
+ const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
255
728
  if (gone.length === 0)
256
729
  return 0;
257
730
  this.removeFiles(sourceId, gone);
258
731
  return gone.length;
259
732
  }
260
733
  sourceStats(sourceId) {
734
+ return this.cached(`src-stats:${sourceId}`, () => this.computeSourceStats(sourceId));
735
+ }
736
+ computeSourceStats(sourceId) {
261
737
  const f = this.db.raw
262
738
  .prepare(`SELECT
263
739
  COUNT(*) AS total,
264
- SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS errors,
265
- SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped
266
- FROM knowledge_files WHERE source_id = ?`)
267
- .get(sourceId);
740
+ SUM(CASE WHEN f.status = 'error' THEN 1 ELSE 0 END) AS errors,
741
+ SUM(CASE WHEN f.status = 'skipped' THEN 1 ELSE 0 END) AS skipped
742
+ FROM knowledge_memberships m
743
+ JOIN knowledge_files f ON f.id = m.file_id
744
+ WHERE m.source_id = ?`)
745
+ .get(String(sourceId));
268
746
  const c = this.db.raw
269
- .prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
270
- .get(sourceId);
747
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
748
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
749
+ WHERE m.source_id = ?`)
750
+ .get(String(sourceId));
271
751
  const ec = this.db.raw
272
752
  .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
273
- LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
274
- WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
275
- .get(sourceId);
753
+ JOIN knowledge_files f ON f.id = c.file_id
754
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
755
+ WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
756
+ .get(String(sourceId));
276
757
  const e = this.db.raw
277
758
  .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
278
759
  JOIN knowledge_chunks c ON c.id = e.chunk_id
279
- WHERE c.source_id = ?`)
280
- .get(sourceId);
760
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
761
+ WHERE m.source_id = ?`)
762
+ .get(String(sourceId));
281
763
  return {
282
764
  files: f?.total ?? 0,
283
765
  chunks: c?.n ?? 0,
@@ -288,315 +770,374 @@ export class KnowledgeIndexStore {
288
770
  };
289
771
  }
290
772
  /**
291
- * 关键词检索(CJK 二元组 + Latin);限定 sourceIds(可见集)
773
+ * 关键词检索。sourceIds = 可见 Membership;chunk 按 file 去重。
292
774
  */
293
775
  search(query, opts) {
294
776
  const tokens = tokenizeKeywordQuery(query);
777
+ const tokenDetail = tokenizeKeywordDetail(query);
295
778
  if (tokens.length === 0 || opts.sourceIds.length === 0)
296
779
  return [];
297
780
  const limit = opts.limit ?? 8;
298
781
  const placeholders = opts.sourceIds.map(() => '?').join(',');
782
+ let rows = null;
783
+ const ftsIds = this.fts.search(query, opts.sourceIds, Math.max(limit * 4, 32));
784
+ if (ftsIds && ftsIds.length > 0) {
785
+ const idPh = ftsIds.map(() => '?').join(',');
786
+ rows = this.db.raw
787
+ .prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
788
+ m.source_id, m.logical_path
789
+ FROM knowledge_chunks c
790
+ JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
791
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
792
+ WHERE c.id IN (${idPh}) AND m.source_id IN (${placeholders})`)
793
+ .all(...ftsIds, ...opts.sourceIds);
794
+ }
795
+ if (!rows || rows.length === 0) {
796
+ const likeClauses = [];
797
+ const likeParams = [];
798
+ for (const token of tokens) {
799
+ const like = `%${token.toLowerCase().replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
800
+ likeClauses.push(`LOWER(c.text) LIKE ? ESCAPE '\\'`);
801
+ likeParams.push(like);
802
+ likeClauses.push(`LOWER(m.logical_path) LIKE ? ESCAPE '\\'`);
803
+ likeParams.push(like);
804
+ }
805
+ rows = this.db.raw
806
+ .prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
807
+ m.source_id, m.logical_path
808
+ FROM knowledge_chunks c
809
+ JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
810
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
811
+ WHERE m.source_id IN (${placeholders})
812
+ AND (${likeClauses.join(' OR ')})
813
+ LIMIT ?`)
814
+ .all(...opts.sourceIds, ...likeParams, Math.max(limit * 8, 64));
815
+ }
816
+ const byChunk = new Map();
817
+ for (const r of rows) {
818
+ let h = byChunk.get(r.id);
819
+ if (!h) {
820
+ if (byChunk.size >= limit * 3)
821
+ continue;
822
+ const score = scoreKeywordCoverage({ content: r.text, tags: r.logical_path }, tokenDetail);
823
+ h = {
824
+ chunkId: r.id,
825
+ fileId: r.file_id,
826
+ sourceIds: [],
827
+ logicalPaths: {},
828
+ path: r.logical_path,
829
+ ordinal: r.ordinal,
830
+ text: r.text,
831
+ startLine: r.start_line,
832
+ endLine: r.end_line,
833
+ score: score ?? 0,
834
+ };
835
+ byChunk.set(r.id, h);
836
+ }
837
+ if (!h.sourceIds.includes(r.source_id))
838
+ h.sourceIds.push(r.source_id);
839
+ h.logicalPaths[r.source_id] = r.logical_path;
840
+ }
841
+ return [...byChunk.values()]
842
+ .sort((a, b) => b.score - a.score)
843
+ .slice(0, limit)
844
+ .map((h) => ({
845
+ chunkId: asChunkId(h.chunkId),
846
+ sourceId: asSourceId(h.sourceIds[0] ?? ''),
847
+ path: h.path,
848
+ ordinal: h.ordinal,
849
+ text: h.text,
850
+ startLine: h.startLine,
851
+ endLine: h.endLine,
852
+ score: h.score,
853
+ // 契约 §8.4:共享 File 多绑定溯源
854
+ sourceIds: h.sourceIds,
855
+ logicalPaths: h.logicalPaths,
856
+ }));
857
+ }
858
+ vectorBackend() {
859
+ if (this.countEmbeddings() === 0)
860
+ return 'disabled';
861
+ return this.db.sqliteVecEnabled ? 'sqlite-vec' : 'js-bucket';
862
+ }
863
+ vectorSearch(embedding, opts) {
864
+ const sourceIds = opts.sourceIds.map((s) => String(s));
865
+ if (sourceIds.length === 0)
866
+ return [];
867
+ const limit = opts.limit ?? 8;
868
+ const emb = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
869
+ const placeholders = sourceIds.map(() => '?').join(',');
870
+ const joinChunks = `JOIN knowledge_chunks c ON c.id = %s
871
+ JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
872
+ JOIN knowledge_memberships m ON m.file_id = c.file_id`;
873
+ const where = `WHERE m.source_id IN (${placeholders})`;
874
+ const load = (ids, scores) => {
875
+ if (ids.length === 0)
876
+ return [];
877
+ const idPh = ids.map(() => '?').join(',');
878
+ const rows = this.db.raw
879
+ .prepare(`SELECT c.id, c.ordinal, c.text, c.start_line, c.end_line,
880
+ m.source_id, m.logical_path
881
+ FROM knowledge_chunks c
882
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
883
+ WHERE c.id IN (${idPh})`)
884
+ .all(...ids);
885
+ // 保持 scores 顺序(SQL IN 不保证序)
886
+ const byId = new Map(rows.map((r) => [r.id, r]));
887
+ const ordered = (scores ? ids : ids).map((id) => byId.get(id)).filter(Boolean);
888
+ return ordered.map((r) => ({
889
+ chunkId: asChunkId(r.id),
890
+ sourceId: asSourceId(r.source_id),
891
+ path: r.logical_path,
892
+ ordinal: r.ordinal,
893
+ text: r.text,
894
+ startLine: r.start_line,
895
+ endLine: r.end_line,
896
+ score: scores?.get(r.id) ?? 0,
897
+ }));
898
+ };
899
+ if (this.db.sqliteVecEnabled) {
900
+ // 必须走 vec0 ANN(MATCH + k)。禁止 ORDER BY vec_distance_cosine —— 那是全表暴力扫,
901
+ // 14 万向量会跑到 50s+,直接打穿 query timeout。
902
+ try {
903
+ const ph = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
904
+ // 先 ANN 过采,再按 Membership 裁剪,最后截断
905
+ const k = Math.min(Math.max(limit * 4, 64), 1024);
906
+ const knn = this.db.raw
907
+ .prepare(`SELECT c.id AS id, v.distance AS dist
908
+ FROM (
909
+ SELECT chunk_id, distance FROM knowledge_vec
910
+ WHERE embedding MATCH ? AND k = ?
911
+ ) v
912
+ ${joinChunks.replace('%s', 'v.chunk_id')} ${where}
913
+ ORDER BY v.distance LIMIT ?`)
914
+ .all(ph, k, ...sourceIds, limit);
915
+ // ANN 完成即权威结果(含 0 命中);不再退 JS 全表
916
+ const scores = new Map(knn.map((r) => [r.id, 1 - Number(r.dist ?? 1)]));
917
+ return load(knn.map((r) => r.id), scores);
918
+ }
919
+ catch {
920
+ // 扩展查询失败才落到有界 JS 路径
921
+ }
922
+ }
923
+ // 无 sqlite-vec 的有界 JS 路径:只取 id/embedding,不拉 text
299
924
  const rows = this.db.raw
300
- .prepare(`SELECT id, source_id, path, ordinal, text, start_line, end_line
301
- FROM knowledge_chunks
302
- WHERE source_id IN (${placeholders})
303
- ORDER BY source_id, path, ordinal
304
- LIMIT 20000`)
305
- .all(...opts.sourceIds);
306
- const hits = [];
307
- for (const row of rows) {
308
- const score = scoreKeywordFields({ content: row.text, tags: row.path }, tokens);
309
- if (score <= 0)
925
+ .prepare(`SELECT e.chunk_id AS chunk_id, e.embedding AS embedding
926
+ FROM knowledge_chunk_embeddings e
927
+ ${joinChunks.replace('%s', 'e.chunk_id')} ${where}
928
+ LIMIT 2000`)
929
+ .all(...sourceIds);
930
+ if (rows.length === 0)
931
+ return [];
932
+ const qArr = emb instanceof Float32Array ? emb : Float32Array.from(emb);
933
+ const scored = [];
934
+ for (const r of rows) {
935
+ const raw = r.embedding;
936
+ const buf = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
937
+ const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
938
+ if (vec.length !== qArr.length)
310
939
  continue;
311
- hits.push({
312
- chunkId: asChunkId(row.id),
313
- sourceId: asSourceId(row.source_id),
314
- path: row.path,
315
- ordinal: row.ordinal,
316
- text: row.text,
317
- startLine: row.start_line,
318
- endLine: row.end_line,
319
- score,
320
- });
321
- }
322
- hits.sort((a, b) => b.score - a.score);
323
- return hits.slice(0, limit);
324
- }
325
- // ── Phase B 向量 ──
326
- /**
327
- * 按 chunk id 读取
328
- */
329
- getChunkById(chunkId) {
330
- const row = this.db.raw
331
- .prepare('SELECT id, source_id, path, text, start_line, end_line FROM knowledge_chunks WHERE id = ?')
332
- .get(chunkId);
333
- if (!row)
334
- return null;
335
- return {
336
- id: asChunkId(row.id),
337
- sourceId: asSourceId(row.source_id),
338
- path: row.path,
339
- text: row.text,
340
- startLine: row.start_line,
341
- endLine: row.end_line,
342
- };
940
+ let dot = 0;
941
+ let nA = 0;
942
+ let nB = 0;
943
+ for (let d = 0; d < qArr.length; d++) {
944
+ const a = qArr[d];
945
+ const b = vec[d];
946
+ dot += a * b;
947
+ nA += a * a;
948
+ nB += b * b;
949
+ }
950
+ const den = Math.sqrt(nA) * Math.sqrt(nB);
951
+ const score = den === 0 ? 0 : dot / den;
952
+ if (score > 0)
953
+ scored.push({ id: r.chunk_id, score });
954
+ }
955
+ scored.sort((a, b) => b.score - a.score);
956
+ const top = scored.slice(0, limit);
957
+ const scores = new Map(top.map((s) => [s.id, s.score]));
958
+ return load(top.map((s) => s.id), scores);
343
959
  }
344
- /**
345
- * 按路径列出 chunks(可选行窗)
346
- */
347
- listChunksByPath(sourceId, path, opts) {
960
+ listChunksMissingEmbedding(sourceId, limit = 50) {
348
961
  const rows = this.db.raw
349
- .prepare(`SELECT id, path, text, start_line, end_line FROM knowledge_chunks
350
- WHERE source_id = ? AND path = ?
351
- ORDER BY ordinal`)
352
- .all(sourceId, path);
353
- let mapped = rows.map((r) => ({
354
- id: asChunkId(r.id),
355
- path: r.path,
356
- text: r.text,
357
- startLine: r.start_line,
358
- endLine: r.end_line,
359
- }));
360
- if (opts?.startLine != null) {
361
- mapped = mapped.filter((c) => c.endLine >= opts.startLine);
362
- }
363
- if (opts?.endLine != null) {
364
- mapped = mapped.filter((c) => c.startLine <= opts.endLine);
365
- }
366
- return mapped;
962
+ .prepare(`SELECT c.id AS id, c.text AS text
963
+ FROM knowledge_chunks c
964
+ JOIN knowledge_files f ON f.id = c.file_id
965
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
966
+ LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
967
+ WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
968
+ LIMIT ?`)
969
+ .all(String(sourceId), limit);
970
+ return rows.map((r) => ({ chunkId: r.id, text: r.text }));
367
971
  }
368
- /**
369
- * 写入 chunk 向量(Float32 BLOB;sqlite-vec 可用时同步进 vec 表)
370
- */
371
- setChunkEmbedding(chunkId, embedding) {
372
- this.setChunkEmbeddings([[chunkId, embedding]]);
972
+ /** 是否仍有缺向量 chunk(EXISTS,不拉正文) */
973
+ hasChunksMissingEmbedding(sourceId) {
974
+ return this.cached(`emb-miss:${sourceId}`, () => {
975
+ const row = this.db.raw
976
+ .prepare(`SELECT 1 AS ok
977
+ FROM knowledge_chunks c
978
+ JOIN knowledge_files f ON f.id = c.file_id
979
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
980
+ LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
981
+ WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
982
+ LIMIT 1`)
983
+ .get(String(sourceId));
984
+ return Boolean(row);
985
+ });
373
986
  }
374
987
  /**
375
- * 批量写向量(单事务,减少与 parse 写路径的锁竞争)
376
- *
377
- * @param rows - [chunkId, embedding] 列表
988
+ * 批量写入嵌入。pairs: `[chunkId, number[]]`;空向量 = secret-skip 墓碑。
378
989
  */
379
- setChunkEmbeddings(rows) {
380
- if (rows.length === 0)
990
+ setChunkEmbeddings(pairs) {
991
+ if (!pairs || pairs.length === 0)
381
992
  return;
382
- const upsert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, created_at)
383
- VALUES (?, ?, ?, ?)
993
+ this.db.ensureEmbeddingSchema();
994
+ const now = Date.now();
995
+ const insert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, bucket, created_at)
996
+ VALUES (?, ?, ?, ?, ?)
384
997
  ON CONFLICT(chunk_id) DO UPDATE SET
385
- dimensions=excluded.dimensions,
386
- embedding=excluded.embedding,
387
- created_at=excluded.created_at`);
998
+ dimensions = excluded.dimensions,
999
+ embedding = excluded.embedding,
1000
+ bucket = excluded.bucket`);
388
1001
  this.db.raw.exec('BEGIN');
389
1002
  try {
390
- const now = Date.now();
391
- for (const [chunkId, embedding] of rows) {
392
- upsert.run(chunkId, embedding.length, toF32Blob(embedding), now);
393
- this.upsertKnowledgeVec(chunkId, embedding);
1003
+ for (const [chunkId, embedding] of pairs) {
1004
+ const vec = Float32Array.from(embedding ?? []);
1005
+ const dims = vec.length;
1006
+ const buf = Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
1007
+ insert.run(chunkId, dims, buf, dims > 0 ? vectorBucket(Array.from(vec)) : -1, now);
1008
+ this.writeKnowledgeVec(chunkId, embedding ?? []);
394
1009
  }
395
1010
  this.db.raw.exec('COMMIT');
1011
+ this.invalidateStatsCache();
396
1012
  }
397
- catch (e) {
398
- this.db.raw.exec('ROLLBACK');
399
- throw e;
400
- }
401
- }
402
- /** sqlite-vec 表名 */
403
- static VEC_TABLE = 'knowledge_vec';
404
- /** 确保 vec0 虚拟表(维度首次写入时确定) */
405
- ensureVecTable(dimensions) {
406
- if (!this.db.sqliteVecEnabled)
407
- return false;
408
- try {
409
- const row = this.db.raw
410
- .prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
411
- .get(KnowledgeIndexStore.VEC_TABLE);
412
- if (row?.sql) {
413
- const m = /float\[(\d+)\]/i.exec(row.sql);
414
- return !m || Number(m[1]) === dimensions;
1013
+ catch (err) {
1014
+ try {
1015
+ this.db.raw.exec('ROLLBACK');
415
1016
  }
416
- this.db.raw.exec(`
417
- CREATE VIRTUAL TABLE IF NOT EXISTS ${KnowledgeIndexStore.VEC_TABLE} USING vec0(
418
- chunk_id TEXT PRIMARY KEY,
419
- embedding float[${dimensions}] distance_metric=cosine
420
- );
421
- `);
422
- return true;
423
- }
424
- catch {
425
- return false;
1017
+ catch {
1018
+ // 以原异常为准
1019
+ }
1020
+ throw err;
426
1021
  }
427
1022
  }
428
- upsertKnowledgeVec(chunkId, embedding) {
429
- if (!this.ensureVecTable(embedding.length))
1023
+ setChunkEmbedding(chunkId, embedding) {
1024
+ this.setChunkEmbeddings([[chunkId, embedding]]);
1025
+ }
1026
+ writeKnowledgeVec(chunkId, embedding) {
1027
+ if (!this.db.sqliteVecEnabled || !embedding?.length)
430
1028
  return;
431
1029
  try {
1030
+ const dims = embedding.length;
1031
+ if (!this.vecTableReady) {
1032
+ this.db.raw.exec(`
1033
+ CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_vec USING vec0(
1034
+ chunk_id TEXT PRIMARY KEY,
1035
+ embedding float[${dims}] distance_metric=cosine
1036
+ );
1037
+ `);
1038
+ this.vecTableReady = true;
1039
+ }
1040
+ this.db.raw.prepare('DELETE FROM knowledge_vec WHERE chunk_id = ?').run(chunkId);
432
1041
  this.db.raw
433
- .prepare(`INSERT INTO ${KnowledgeIndexStore.VEC_TABLE} (chunk_id, embedding)
434
- VALUES (?, ?)
435
- ON CONFLICT(chunk_id) DO UPDATE SET embedding = excluded.embedding`)
436
- .run(String(chunkId), toF32Blob(embedding));
1042
+ .prepare('INSERT INTO knowledge_vec (chunk_id, embedding) VALUES (?, ?)')
1043
+ .run(chunkId, toVecBlob(embedding));
437
1044
  }
438
1045
  catch {
439
- // vec 表不可用时忽略,检索走 JS 路径
1046
+ // 无 sqlite-vec 时忽略
440
1047
  }
441
1048
  }
442
- /**
443
- * 列出待嵌入 chunk(无向量行)
444
- *
445
- * **代码不进向量**:原文件已可 file_search;片段易失真。
446
- * 仅文档/正文类(text / markdown / document:* 等)参与 embedding。
447
- */
448
- listChunksMissingEmbedding(sourceId, limit = 100) {
449
- const rows = this.db.raw
450
- .prepare(`SELECT c.id, c.text, c.path
451
- FROM knowledge_chunks c
452
- LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
453
- LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
454
- WHERE c.source_id = ? AND e.chunk_id IS NULL
455
- AND ${EMBEDDABLE_WHERE}
456
- ORDER BY c.path, c.ordinal
457
- LIMIT ?`)
458
- .all(sourceId, limit);
459
- return rows.map((r) => ({ id: asChunkId(r.id), text: r.text, path: r.path }));
460
- }
461
- /**
462
- * 向量覆盖度(0–1;无可嵌入 chunk 时为 1)
463
- */
464
1049
  embeddingCoverage(sourceId) {
465
- const total = this.db.raw
1050
+ return this.cached(`emb-cov:${sourceId ?? 'global'}`, () => this.computeEmbeddingCoverage(sourceId));
1051
+ }
1052
+ computeEmbeddingCoverage(sourceId) {
1053
+ if (sourceId) {
1054
+ const t = this.db.raw
1055
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
1056
+ JOIN knowledge_files f ON f.id = c.file_id
1057
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
1058
+ WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
1059
+ .get(String(sourceId)).n;
1060
+ if (t === 0)
1061
+ return 1;
1062
+ const e = this.db.raw
1063
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
1064
+ JOIN knowledge_chunks c ON c.id = e.chunk_id
1065
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
1066
+ WHERE m.source_id = ?`)
1067
+ .get(String(sourceId)).n;
1068
+ return Math.min(1, e / t);
1069
+ }
1070
+ const t = this.db.raw
466
1071
  .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
467
- LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
468
- WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
469
- .get(sourceId);
470
- if (!total?.n)
1072
+ JOIN knowledge_files f ON f.id = c.file_id WHERE ${EMBEDDABLE_WHERE}`)
1073
+ .get().n;
1074
+ if (t === 0)
471
1075
  return 1;
472
- const embedded = this.db.raw
473
- .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
474
- JOIN knowledge_chunks c ON c.id = e.chunk_id
475
- LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
476
- WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
477
- .get(sourceId);
478
- return Math.min(1, (embedded?.n ?? 0) / total.n);
1076
+ const e = this.db.raw
1077
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
1078
+ .get().n;
1079
+ return Math.min(1, e / t);
479
1080
  }
480
- /**
481
- * 向量检索:优先 sqlite-vec KNN;否则 JS cosine(BLOB)
482
- */
483
- vectorSearch(queryEmbedding, opts) {
484
- if (opts.sourceIds.length === 0 || !queryEmbedding?.length)
485
- return [];
486
- const limit = opts.limit ?? 8;
487
- // vec 表可能尚未回填;0 命中时退 JS,避免空结果短路
488
- const viaVec = this.vectorSearchVec(queryEmbedding, opts, limit);
489
- if (viaVec && viaVec.length > 0)
490
- return viaVec;
491
- return this.vectorSearchJs(queryEmbedding, opts, limit);
492
- }
493
- /** sqlite-vec KNN(不可用时返回 null) */
494
- vectorSearchVec(queryEmbedding, opts, limit) {
1081
+ backfillVecFromEmbeddings(batch = 100) {
495
1082
  if (!this.db.sqliteVecEnabled)
496
- return null;
497
- if (!this.ensureVecTable(queryEmbedding.length))
498
- return null;
499
- const placeholders = opts.sourceIds.map(() => '?').join(',');
500
- try {
501
- const rows = this.db.raw
502
- .prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, v.distance AS distance
503
- FROM (
504
- SELECT chunk_id, distance
505
- FROM ${KnowledgeIndexStore.VEC_TABLE}
506
- WHERE embedding MATCH ?
507
- AND k = ?
508
- ) v
509
- JOIN knowledge_chunks c ON c.id = v.chunk_id
510
- WHERE c.source_id IN (${placeholders})
511
- ORDER BY v.distance ASC`)
512
- .all(toF32Blob(queryEmbedding), Math.max(limit * 2, 16), ...opts.sourceIds);
513
- return rows.slice(0, limit).map((row) => ({
514
- chunkId: asChunkId(row.id),
515
- sourceId: asSourceId(row.source_id),
516
- path: row.path,
517
- ordinal: row.ordinal,
518
- text: row.text,
519
- startLine: row.start_line,
520
- endLine: row.end_line,
521
- score: 1 - Number(row.distance),
522
- }));
1083
+ return 0;
1084
+ const rows = this.db.raw
1085
+ .prepare(`SELECT chunk_id, embedding, dimensions FROM knowledge_chunk_embeddings
1086
+ WHERE dimensions > 0 LIMIT ?`)
1087
+ .all(batch);
1088
+ let n = 0;
1089
+ for (const r of rows) {
1090
+ const buf = r.embedding instanceof Uint8Array
1091
+ ? r.embedding
1092
+ : new Uint8Array(r.embedding);
1093
+ const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
1094
+ this.writeKnowledgeVec(r.chunk_id, vec);
1095
+ n += 1;
523
1096
  }
524
- catch {
1097
+ return n;
1098
+ }
1099
+ getChunk(chunkId) {
1100
+ const r = this.db.raw
1101
+ .prepare(`SELECT c.id, c.text, c.start_line, c.end_line, m.source_id, m.logical_path
1102
+ FROM knowledge_chunks c
1103
+ LEFT JOIN knowledge_memberships m ON m.file_id = c.file_id
1104
+ WHERE c.id = ? LIMIT 1`)
1105
+ .get(chunkId);
1106
+ if (!r)
525
1107
  return null;
526
- }
1108
+ return {
1109
+ id: r.id,
1110
+ sourceId: r.source_id ?? '',
1111
+ path: r.logical_path ?? '',
1112
+ text: r.text,
1113
+ startLine: r.start_line,
1114
+ endLine: r.end_line,
1115
+ };
527
1116
  }
528
- /** JS 余弦(Float32 BLOB;无 JSON.parse) */
529
- vectorSearchJs(queryEmbedding, opts, limit) {
530
- const placeholders = opts.sourceIds.map(() => '?').join(',');
1117
+ listChunksByPath(sourceId, path) {
1118
+ const logical = path.replace(/\\/g, '/');
531
1119
  const rows = this.db.raw
532
- .prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.embedding
1120
+ .prepare(`SELECT c.id, c.text, c.start_line, c.end_line, c.ordinal
533
1121
  FROM knowledge_chunks c
534
- JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
535
- WHERE c.source_id IN (${placeholders})
536
- LIMIT 200000`)
537
- .all(...opts.sourceIds);
538
- const hits = [];
539
- for (const row of rows) {
540
- const emb = fromF32Blob(row.embedding);
541
- if (!emb?.length || emb.length !== queryEmbedding.length)
542
- continue;
543
- const sim = cosineSimilarity(queryEmbedding, emb);
544
- if (sim <= 0)
545
- continue;
546
- hits.push({
547
- chunkId: asChunkId(row.id),
548
- sourceId: asSourceId(row.source_id),
549
- path: row.path,
550
- ordinal: row.ordinal,
551
- text: row.text,
552
- startLine: row.start_line,
553
- endLine: row.end_line,
554
- score: sim,
555
- });
556
- }
557
- hits.sort((a, b) => b.score - a.score);
558
- return hits.slice(0, limit);
1122
+ JOIN knowledge_memberships m ON m.file_id = c.file_id
1123
+ WHERE m.source_id = ? AND m.logical_path = ?
1124
+ ORDER BY c.ordinal`)
1125
+ .all(String(sourceId), logical);
1126
+ return rows.map((r) => ({
1127
+ id: r.id,
1128
+ text: r.text,
1129
+ startLine: r.start_line,
1130
+ endLine: r.end_line,
1131
+ ordinal: r.ordinal,
1132
+ path: logical,
1133
+ }));
1134
+ }
1135
+ countEmbeddings() {
1136
+ return this.db.raw
1137
+ .prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
1138
+ .get().n;
559
1139
  }
560
1140
  }
561
- /** number[] → float32 blob */
562
- function toF32Blob(embedding) {
563
- const f32 = Float32Array.from(embedding);
564
- return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
565
- }
566
- /** float32 blob → number[] */
567
- function fromF32Blob(buf) {
568
- if (!buf || buf.byteLength === 0)
569
- return null;
570
- if (buf.byteLength % 4 !== 0)
571
- return null;
572
- const u8 = buf instanceof Buffer ? new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) : buf;
573
- const f32 = new Float32Array(u8.buffer, u8.byteOffset, u8.byteLength / 4);
574
- return Array.from(f32);
575
- }
576
- /**
577
- * 内容指纹(新鲜度判断)
578
- *
579
- * @param content - UTF-8 文本或原始字节(二进制文档用 bytes)
580
- * @returns sha256 hex
581
- */
582
1141
  export function hashContent(content) {
583
1142
  return createHash('sha256').update(content).digest('hex');
584
1143
  }
585
- function rowToFile(row) {
586
- return {
587
- id: String(row.id),
588
- sourceId: asSourceId(String(row.source_id)),
589
- path: String(row.path),
590
- contentHash: String(row.content_hash ?? ''),
591
- size: Number(row.size ?? 0),
592
- mtime: Number(row.mtime ?? 0),
593
- adapterId: row.adapter_id == null ? null : String(row.adapter_id),
594
- status: String(row.status),
595
- error: row.error == null ? undefined : String(row.error),
596
- chunkCount: Number(row.chunk_count ?? 0),
597
- indexedAt: Number(row.indexed_at ?? 0),
598
- externalUrl: row.external_url == null ? undefined : String(row.external_url),
599
- etag: row.etag == null ? undefined : String(row.etag),
600
- lastModified: row.last_modified == null ? undefined : String(row.last_modified),
601
- };
602
- }