@octopi-agent/engine 0.56.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +43 -1
- package/dist/config-schema/substrate.d.ts +42 -0
- package/dist/config-schema/substrate.js +58 -0
- package/dist/config.d.ts +39 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +115 -5
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +5 -3
- package/dist/harness/index.js +3 -2
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +42 -3
- package/dist/harness/knowledge/db.js +466 -83
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +68 -0
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +33 -0
- package/dist/harness/knowledge/extract-document.js +86 -0
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +166 -63
- package/dist/harness/knowledge/index-store.js +1025 -282
- package/dist/harness/knowledge/index.d.ts +47 -5
- package/dist/harness/knowledge/index.js +24 -2
- package/dist/harness/knowledge/ingest.d.ts +322 -49
- package/dist/harness/knowledge/ingest.js +1383 -267
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +30 -5
- package/dist/harness/knowledge/retriever.js +98 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +83 -16
- package/dist/harness/knowledge/source-store.js +205 -78
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +41 -2
- package/dist/harness/memory/sqlite/keyword-search.js +104 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -1,67 +1,263 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIndexStore —
|
|
2
|
+
* KnowledgeIndexStore — File 本位索引投影
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* chunk/file 挂 file_id;logical path 在 Membership。
|
|
5
|
+
* sourceId+path 的 API 保留为「Membership 视图」,内部按 identity 去重。
|
|
5
6
|
*/
|
|
6
7
|
import { randomUUID, createHash } from 'node:crypto';
|
|
7
|
-
import { tokenizeKeywordQuery,
|
|
8
|
-
import {
|
|
8
|
+
import { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordCoverage, } from '../memory/sqlite/keyword-search.js';
|
|
9
|
+
import { KnowledgeFts } from './fts.js';
|
|
10
|
+
import { vectorBucket } from './vector-ann.js';
|
|
11
|
+
import { toVecBlob } from '../memory/sqlite/sqlite-vec.js';
|
|
9
12
|
import { asChunkId, asSourceId } from './types.js';
|
|
13
|
+
import { normalizePathLexical } from './file-identity.js';
|
|
14
|
+
const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
|
|
15
|
+
function defaultIdentityKey(path) {
|
|
16
|
+
try {
|
|
17
|
+
return `path:${normalizePathLexical(path)}`;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return `path:${path.replace(/\\/g, '/')}`;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function rowToFile(row) {
|
|
24
|
+
return {
|
|
25
|
+
id: String(row.id),
|
|
26
|
+
sourceId: asSourceId(String(row.source_id ?? row.membership_source_id ?? '')),
|
|
27
|
+
path: String(row.logical_path ?? row.path ?? ''),
|
|
28
|
+
contentHash: String(row.content_hash ?? ''),
|
|
29
|
+
size: Number(row.size ?? 0),
|
|
30
|
+
mtime: Number(row.mtime ?? 0),
|
|
31
|
+
adapterId: row.adapter_id == null ? null : String(row.adapter_id),
|
|
32
|
+
status: String(row.status),
|
|
33
|
+
error: row.error == null ? undefined : String(row.error),
|
|
34
|
+
chunkCount: Number(row.chunk_count ?? 0),
|
|
35
|
+
indexedAt: Number(row.indexed_at ?? 0),
|
|
36
|
+
externalUrl: row.external_url == null ? undefined : String(row.external_url),
|
|
37
|
+
etag: row.etag == null ? undefined : String(row.etag),
|
|
38
|
+
lastModified: row.last_modified == null ? undefined : String(row.last_modified),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
10
41
|
export class KnowledgeIndexStore {
|
|
11
42
|
db;
|
|
43
|
+
fts;
|
|
44
|
+
ftsBackfillPromise = null;
|
|
45
|
+
vecTableReady = false;
|
|
46
|
+
/** 同 File 写序(upsert ∥ purge):键 = identity_key */
|
|
47
|
+
writeLocks = new Map();
|
|
48
|
+
/** 大库 COUNT 短 TTL:UI 轮询 / jobControl 不必每请求全表 JOIN */
|
|
49
|
+
statsCache = new Map();
|
|
50
|
+
static STATS_TTL_MS = 1_000;
|
|
12
51
|
constructor(db) {
|
|
13
52
|
this.db = db;
|
|
53
|
+
this.fts = new KnowledgeFts(db);
|
|
54
|
+
}
|
|
55
|
+
cached(key, compute) {
|
|
56
|
+
const now = Date.now();
|
|
57
|
+
const hit = this.statsCache.get(key);
|
|
58
|
+
if (hit && now - hit.at < KnowledgeIndexStore.STATS_TTL_MS)
|
|
59
|
+
return hit.value;
|
|
60
|
+
const value = compute();
|
|
61
|
+
this.statsCache.set(key, { at: now, value });
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
/** 写路径后立刻让统计可见(收尾 force 也可走此绕过 TTL) */
|
|
65
|
+
invalidateStatsCache() {
|
|
66
|
+
this.statsCache.clear();
|
|
14
67
|
}
|
|
15
68
|
/**
|
|
16
|
-
*
|
|
69
|
+
* 按 File identity 串行写(upsert / purge)。
|
|
70
|
+
* purge 在 dropChunks 让出窗口内与 upsert 交错会留下孤儿 FTS / 悬空 membership。
|
|
17
71
|
*/
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const fileId = existing?.id ?? `kf_${randomUUID().slice(0, 12)}`;
|
|
24
|
-
this.db.raw.exec('BEGIN');
|
|
72
|
+
async withFileWriteLock(key, fn) {
|
|
73
|
+
const prev = this.writeLocks.get(key) ?? Promise.resolve();
|
|
74
|
+
const run = prev.then(() => fn());
|
|
75
|
+
const tail = run.then(() => undefined, () => undefined);
|
|
76
|
+
this.writeLocks.set(key, tail);
|
|
25
77
|
try {
|
|
78
|
+
return await run;
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
if (this.writeLocks.get(key) === tail)
|
|
82
|
+
this.writeLocks.delete(key);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
get ftsAvailable() {
|
|
86
|
+
return this.fts.available;
|
|
87
|
+
}
|
|
88
|
+
get ftsBackfillRunning() {
|
|
89
|
+
return this.ftsBackfillPromise != null;
|
|
90
|
+
}
|
|
91
|
+
ensureFtsBackfill() {
|
|
92
|
+
if (!this.fts.available)
|
|
93
|
+
return Promise.resolve(0);
|
|
94
|
+
if (this.ftsBackfillPromise)
|
|
95
|
+
return this.ftsBackfillPromise;
|
|
96
|
+
const n = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks').get().n;
|
|
97
|
+
const ftsN = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks_fts').get().n;
|
|
98
|
+
if (n === 0 || ftsN > 0)
|
|
99
|
+
return Promise.resolve(0);
|
|
100
|
+
this.ftsBackfillPromise = this.fts
|
|
101
|
+
.rebuildFromChunksAsync()
|
|
102
|
+
.catch(() => 0)
|
|
103
|
+
.finally(() => {
|
|
104
|
+
this.ftsBackfillPromise = null;
|
|
105
|
+
});
|
|
106
|
+
return this.ftsBackfillPromise;
|
|
107
|
+
}
|
|
108
|
+
/** 解析或创建 File(identity 去重)+ Membership */
|
|
109
|
+
resolveFile(sourceId, path, meta) {
|
|
110
|
+
const identityKey = meta.identityKey ?? defaultIdentityKey(path);
|
|
111
|
+
const tenant = meta.tenantId ?? 'default';
|
|
112
|
+
const now = Date.now();
|
|
113
|
+
let row = this.db.raw
|
|
114
|
+
.prepare('SELECT id, size, mtime FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
|
|
115
|
+
.get(tenant, identityKey);
|
|
116
|
+
if (!row) {
|
|
117
|
+
const id = `kf_${randomUUID().slice(0, 12)}`;
|
|
26
118
|
this.db.raw
|
|
27
|
-
.prepare(`
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
119
|
+
.prepare(`INSERT INTO knowledge_files
|
|
120
|
+
(id, tenant_id, identity_key, size, mtime, content_hash, status, chunk_count, indexed_at)
|
|
121
|
+
VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, ?)`)
|
|
122
|
+
.run(id, tenant, identityKey, meta.size ?? 0, meta.mtime ?? 0, now);
|
|
123
|
+
row = { id, size: meta.size ?? 0, mtime: meta.mtime ?? 0 };
|
|
124
|
+
}
|
|
125
|
+
else if ((meta.size != null && meta.size !== row.size) ||
|
|
126
|
+
(meta.mtime != null && meta.mtime !== row.mtime)) {
|
|
31
127
|
this.db.raw
|
|
32
|
-
.prepare(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
128
|
+
.prepare('UPDATE knowledge_files SET size = ?, mtime = ? WHERE id = ?')
|
|
129
|
+
.run(meta.size ?? row.size, meta.mtime ?? row.mtime, row.id);
|
|
130
|
+
}
|
|
131
|
+
const logical = path.replace(/\\/g, '/');
|
|
132
|
+
this.db.raw
|
|
133
|
+
.prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
|
|
134
|
+
VALUES (?, ?, ?, ?)
|
|
135
|
+
ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
|
|
136
|
+
.run(sourceId, row.id, logical, now);
|
|
137
|
+
return row.id;
|
|
138
|
+
}
|
|
139
|
+
/** 按 identity_key 查已有 File(不建行);供 job 入队绑定 file_id */
|
|
140
|
+
findFileIdByIdentity(identityKey, tenantId = 'default') {
|
|
141
|
+
const row = this.db.raw
|
|
142
|
+
.prepare('SELECT id FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
|
|
143
|
+
.get(tenantId, identityKey);
|
|
144
|
+
return row?.id ?? null;
|
|
145
|
+
}
|
|
146
|
+
/** 只挂 Membership(共享 File:已有 parse 在途时补认领,禁止双 parse) */
|
|
147
|
+
attachMembership(sourceId, fileId, path) {
|
|
148
|
+
const logical = path.replace(/\\/g, '/');
|
|
149
|
+
this.db.raw
|
|
150
|
+
.prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
|
|
151
|
+
VALUES (?, ?, ?, ?)
|
|
152
|
+
ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
|
|
153
|
+
.run(sourceId, fileId, logical, Date.now());
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* 替换文件索引。chunk/FTS **分批**写入并让出事件循环,
|
|
157
|
+
* 大 xlsx 数千 chunk 不得整文件一个同步事务堵死 Service。
|
|
158
|
+
*
|
|
159
|
+
* 语义:旧 chunk 先清;`status='indexing'` 直到全部批次落库后才 `indexed`。
|
|
160
|
+
* 检索不认 `indexing`(半成品)。批次失败/中止会清掉 partial chunks,
|
|
161
|
+
* 并留在 `indexing`(chunk_count=0)供 gap 扫描重试——不得把半成品标成 `error`
|
|
162
|
+
* 而让 partial 变得可搜。
|
|
163
|
+
*/
|
|
164
|
+
async upsertFile(input, opts) {
|
|
165
|
+
const lockKey = input.identityKey ?? defaultIdentityKey(input.path);
|
|
166
|
+
return this.withFileWriteLock(lockKey, async () => {
|
|
167
|
+
return this.upsertFileLocked(input, opts);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
async upsertFileLocked(input, opts) {
|
|
171
|
+
const now = Date.now();
|
|
172
|
+
// 小批量 + 批间让出:200/批 的 FTS 写在 8 文件并发 parse 时仍会长时间占住 Engine
|
|
173
|
+
const batchSize = Math.max(20, opts?.batchSize ?? 50);
|
|
174
|
+
const signal = opts?.signal;
|
|
175
|
+
const fileId = this.resolveFile(String(input.sourceId), input.path, {
|
|
176
|
+
identityKey: input.identityKey,
|
|
177
|
+
size: input.size,
|
|
178
|
+
mtime: input.mtime,
|
|
179
|
+
});
|
|
180
|
+
if (signal?.aborted) {
|
|
181
|
+
await this.markUpsertIncomplete(fileId, 'aborted');
|
|
182
|
+
throw new Error('aborted');
|
|
183
|
+
}
|
|
184
|
+
// 旧索引作废(含 embeddings/FTS/vec)— 分批让出,禁止同步连环 DELETE 冻住 Engine
|
|
185
|
+
{
|
|
186
|
+
const oldIds = this.db.raw
|
|
187
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
188
|
+
.all(fileId).map((r) => r.id);
|
|
189
|
+
await this.dropChunksAsync(oldIds);
|
|
190
|
+
this.db.raw.exec('BEGIN');
|
|
191
|
+
try {
|
|
192
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
193
|
+
this.db.raw
|
|
194
|
+
.prepare(`UPDATE knowledge_files
|
|
195
|
+
SET content_hash = ?, size = ?, mtime = ?, adapter_id = ?,
|
|
196
|
+
status = 'indexing', error = NULL, chunk_count = 0, indexed_at = ?,
|
|
197
|
+
external_url = ?, etag = ?, last_modified = ?
|
|
198
|
+
WHERE id = ?`)
|
|
199
|
+
.run(input.contentHash, input.size, input.mtime, input.adapterId, now, input.externalUrl ?? null, input.etag ?? null, input.lastModified ?? null, fileId);
|
|
200
|
+
this.db.raw.exec('COMMIT');
|
|
201
|
+
}
|
|
202
|
+
catch (err) {
|
|
203
|
+
try {
|
|
204
|
+
this.db.raw.exec('ROLLBACK');
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
// 以原异常为准
|
|
208
|
+
}
|
|
209
|
+
throw err;
|
|
53
210
|
}
|
|
54
|
-
this.db.raw.exec('COMMIT');
|
|
55
211
|
}
|
|
56
|
-
|
|
212
|
+
for (let offset = 0; offset < input.chunks.length; offset += batchSize) {
|
|
213
|
+
if (signal?.aborted) {
|
|
214
|
+
await this.markUpsertIncomplete(fileId, 'aborted');
|
|
215
|
+
throw new Error('aborted');
|
|
216
|
+
}
|
|
217
|
+
const batch = input.chunks.slice(offset, offset + batchSize);
|
|
218
|
+
this.db.raw.exec('BEGIN');
|
|
57
219
|
try {
|
|
58
|
-
this.db.raw.
|
|
220
|
+
const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, ordinal, text, start_line, end_line)
|
|
221
|
+
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
222
|
+
const ftsRows = [];
|
|
223
|
+
for (const c of batch) {
|
|
224
|
+
const cid = `kc_${randomUUID().slice(0, 12)}`;
|
|
225
|
+
insertChunk.run(cid, fileId, c.ordinal, c.text, c.startLine, c.endLine);
|
|
226
|
+
ftsRows.push({
|
|
227
|
+
id: cid,
|
|
228
|
+
text: c.text,
|
|
229
|
+
path: input.path,
|
|
230
|
+
...(c.ftsToks != null ? { toks: c.ftsToks } : {}),
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
this.fts.upsertMany(ftsRows);
|
|
234
|
+
this.db.raw.exec('COMMIT');
|
|
59
235
|
}
|
|
60
|
-
catch {
|
|
61
|
-
|
|
236
|
+
catch (err) {
|
|
237
|
+
try {
|
|
238
|
+
this.db.raw.exec('ROLLBACK');
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
// 以原异常为准
|
|
242
|
+
}
|
|
243
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
244
|
+
await this.markUpsertIncomplete(fileId, message);
|
|
245
|
+
throw err;
|
|
62
246
|
}
|
|
63
|
-
|
|
247
|
+
if (offset + batchSize < input.chunks.length) {
|
|
248
|
+
await new Promise((r) => setImmediate(r));
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (signal?.aborted) {
|
|
252
|
+
await this.markUpsertIncomplete(fileId, 'aborted');
|
|
253
|
+
throw new Error('aborted');
|
|
64
254
|
}
|
|
255
|
+
this.db.raw
|
|
256
|
+
.prepare(`UPDATE knowledge_files
|
|
257
|
+
SET status = 'indexed', error = NULL, chunk_count = ?, indexed_at = ?
|
|
258
|
+
WHERE id = ?`)
|
|
259
|
+
.run(input.chunks.length, now, fileId);
|
|
260
|
+
this.invalidateStatsCache();
|
|
65
261
|
return {
|
|
66
262
|
id: fileId,
|
|
67
263
|
sourceId: input.sourceId,
|
|
@@ -79,322 +275,869 @@ export class KnowledgeIndexStore {
|
|
|
79
275
|
};
|
|
80
276
|
}
|
|
81
277
|
/**
|
|
82
|
-
*
|
|
278
|
+
* 半写入收尾:清掉已落地的 partial chunks,回到可重试的 `indexing`。
|
|
279
|
+
* 不得标成 `error`——error 行上的 partial 会被检索捞出。
|
|
83
280
|
*/
|
|
84
|
-
|
|
85
|
-
|
|
281
|
+
async markUpsertIncomplete(fileId, message) {
|
|
282
|
+
try {
|
|
283
|
+
const ids = this.db.raw
|
|
284
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
285
|
+
.all(fileId).map((r) => r.id);
|
|
286
|
+
await this.dropChunksAsync(ids);
|
|
287
|
+
this.db.raw.exec('BEGIN');
|
|
288
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
289
|
+
this.db.raw
|
|
290
|
+
.prepare(`UPDATE knowledge_files
|
|
291
|
+
SET status = 'indexing', chunk_count = 0, error = ?, indexed_at = ?
|
|
292
|
+
WHERE id = ?`)
|
|
293
|
+
.run(message, Date.now(), fileId);
|
|
294
|
+
this.db.raw.exec('COMMIT');
|
|
295
|
+
}
|
|
296
|
+
catch {
|
|
297
|
+
try {
|
|
298
|
+
this.db.raw.exec('ROLLBACK');
|
|
299
|
+
}
|
|
300
|
+
catch {
|
|
301
|
+
// 清理失败时至少保证调用方仍看到原异常;半成品由 gap 扫描按 indexing 重做
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
markFileSkipped(sourceId, path, reason, size, identityKey) {
|
|
306
|
+
this.markFileNonIndexed(sourceId, path, 'skipped', reason, size, identityKey);
|
|
307
|
+
}
|
|
308
|
+
markFileError(sourceId, path, error, size, identityKey) {
|
|
309
|
+
this.markFileNonIndexed(sourceId, path, 'error', error, size, identityKey);
|
|
86
310
|
}
|
|
87
311
|
/**
|
|
88
|
-
* 标记 error — **不删** 已有 chunks
|
|
89
|
-
*
|
|
312
|
+
* 标记 skipped/error — **不删** 已有 chunks。
|
|
313
|
+
* 若已有 membership,只改状态;**不得**用 path-identity 新建 File 顶掉带 chunk 的行。
|
|
90
314
|
*/
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
}
|
|
94
|
-
markFileNonIndexed(sourceId, path, status, detail) {
|
|
95
|
-
const now = Date.now();
|
|
315
|
+
markFileNonIndexed(sourceId, path, status, detail, size, identityKey) {
|
|
316
|
+
const logical = path.replace(/\\/g, '/');
|
|
96
317
|
const existing = this.db.raw
|
|
97
|
-
.prepare(
|
|
98
|
-
.get(sourceId,
|
|
99
|
-
|
|
100
|
-
|
|
318
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
319
|
+
.get(String(sourceId), logical);
|
|
320
|
+
if (existing?.file_id) {
|
|
321
|
+
this.db.raw
|
|
322
|
+
.prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
|
|
323
|
+
.run(status, detail, Date.now(), existing.file_id);
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
// 优先真实 identity(win:/unix:),避免 path: 与成功路径分裂出零认领行
|
|
327
|
+
const fileId = this.resolveFile(String(sourceId), path, { size, identityKey });
|
|
101
328
|
this.db.raw
|
|
102
|
-
.prepare(
|
|
103
|
-
|
|
104
|
-
) VALUES (?, ?, ?, '', 0, 0, NULL, ?, ?, ?, ?)
|
|
105
|
-
ON CONFLICT(source_id, path) DO UPDATE SET
|
|
106
|
-
status=excluded.status,
|
|
107
|
-
error=excluded.error,
|
|
108
|
-
indexed_at=excluded.indexed_at`)
|
|
109
|
-
.run(fileId, sourceId, path, status, detail, chunkCount, now);
|
|
329
|
+
.prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
|
|
330
|
+
.run(status, detail, Date.now(), fileId);
|
|
110
331
|
}
|
|
111
332
|
/**
|
|
112
|
-
*
|
|
333
|
+
* 清零认领的 path: 降级 File(错误路径与成功路径 identity 分裂残留)。
|
|
334
|
+
*
|
|
335
|
+
* @returns 删除的 File 行数
|
|
113
336
|
*/
|
|
337
|
+
purgeOrphanPathIdentityFiles() {
|
|
338
|
+
const res = this.db.raw
|
|
339
|
+
.prepare(`DELETE FROM knowledge_files
|
|
340
|
+
WHERE identity_key LIKE 'path:%'
|
|
341
|
+
AND NOT EXISTS (
|
|
342
|
+
SELECT 1 FROM knowledge_memberships m WHERE m.file_id = knowledge_files.id
|
|
343
|
+
)`)
|
|
344
|
+
.run();
|
|
345
|
+
return Number(res.changes ?? 0);
|
|
346
|
+
}
|
|
114
347
|
isFresh(sourceId, path, contentHash) {
|
|
348
|
+
const logical = path.replace(/\\/g, '/');
|
|
349
|
+
const row = this.db.raw
|
|
350
|
+
.prepare(`SELECT f.content_hash AS content_hash, f.chunk_count AS chunk_count
|
|
351
|
+
FROM knowledge_memberships m
|
|
352
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
353
|
+
WHERE m.source_id = ? AND m.logical_path = ? AND f.status <> 'indexing'`)
|
|
354
|
+
.get(String(sourceId), logical);
|
|
355
|
+
return row?.content_hash === contentHash && (row?.chunk_count ?? 0) > 0;
|
|
356
|
+
}
|
|
357
|
+
invalidateFileForReparse(sourceId, path) {
|
|
358
|
+
const logical = path.replace(/\\/g, '/');
|
|
115
359
|
const row = this.db.raw
|
|
116
|
-
.prepare(`SELECT
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
360
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
361
|
+
.get(String(sourceId), logical);
|
|
362
|
+
if (!row)
|
|
363
|
+
return;
|
|
364
|
+
this.db.raw
|
|
365
|
+
.prepare(`UPDATE knowledge_files SET content_hash = '' WHERE id = ?`)
|
|
366
|
+
.run(row.file_id);
|
|
121
367
|
}
|
|
122
368
|
getFile(sourceId, path) {
|
|
369
|
+
const logical = path.replace(/\\/g, '/');
|
|
123
370
|
const row = this.db.raw
|
|
124
|
-
.prepare(
|
|
125
|
-
|
|
371
|
+
.prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
|
|
372
|
+
FROM knowledge_memberships m
|
|
373
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
374
|
+
WHERE m.source_id = ? AND m.logical_path = ?`)
|
|
375
|
+
.get(String(sourceId), logical);
|
|
126
376
|
return row ? rowToFile(row) : null;
|
|
127
377
|
}
|
|
128
378
|
listFiles(sourceId) {
|
|
129
379
|
const rows = this.db.raw
|
|
130
|
-
.prepare(
|
|
131
|
-
|
|
380
|
+
.prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
|
|
381
|
+
FROM knowledge_memberships m
|
|
382
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
383
|
+
WHERE m.source_id = ? ORDER BY m.logical_path`)
|
|
384
|
+
.all(String(sourceId));
|
|
132
385
|
return rows.map(rowToFile);
|
|
133
386
|
}
|
|
387
|
+
listFilesPaged(sourceId, opts) {
|
|
388
|
+
const pageSize = Math.max(1, Math.min(200, opts?.pageSize ?? 50));
|
|
389
|
+
const page = Math.max(1, opts?.page ?? 1);
|
|
390
|
+
const status = opts?.status && opts.status !== 'all' ? opts.status : null;
|
|
391
|
+
const ext = opts?.ext && opts.ext !== 'all'
|
|
392
|
+
? opts.ext.replace(/^\./, '').toLowerCase()
|
|
393
|
+
: null;
|
|
394
|
+
const q = opts?.q?.trim().toLowerCase() || null;
|
|
395
|
+
// SQL 过滤 + LIMIT/OFFSET:禁止全量拉表再在内存分页(1k+ 文件时 UI 轮询会拖垮 Engine/回包)
|
|
396
|
+
const where = ['m.source_id = ?'];
|
|
397
|
+
const params = [String(sourceId)];
|
|
398
|
+
if (status) {
|
|
399
|
+
where.push('f.status = ?');
|
|
400
|
+
params.push(status);
|
|
401
|
+
}
|
|
402
|
+
if (ext) {
|
|
403
|
+
where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
|
|
404
|
+
params.push(`%.${ext.replace(/[\\%_]/g, (c) => `\\${c}`)}`);
|
|
405
|
+
}
|
|
406
|
+
if (q) {
|
|
407
|
+
where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
|
|
408
|
+
params.push(`%${q.replace(/[\\%_]/g, (c) => `\\${c}`)}%`);
|
|
409
|
+
}
|
|
410
|
+
const whereSql = where.join(' AND ');
|
|
411
|
+
const totalRow = this.db.raw
|
|
412
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_memberships m
|
|
413
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
414
|
+
WHERE ${whereSql}`)
|
|
415
|
+
.get(...params);
|
|
416
|
+
const rows = this.db.raw
|
|
417
|
+
.prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
|
|
418
|
+
FROM knowledge_memberships m
|
|
419
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
420
|
+
WHERE ${whereSql}
|
|
421
|
+
ORDER BY m.logical_path
|
|
422
|
+
LIMIT ? OFFSET ?`)
|
|
423
|
+
.all(...params, pageSize, (page - 1) * pageSize);
|
|
424
|
+
// counts 走 SQL 聚合:禁止把整源 logical_path 拉进 JS 再计数(UI 每页轮询会打满 Engine)
|
|
425
|
+
const statusCounts = { indexed: 0, skipped: 0, error: 0 };
|
|
426
|
+
const statusRows = this.db.raw
|
|
427
|
+
.prepare(`SELECT f.status AS status, COUNT(*) AS n
|
|
428
|
+
FROM knowledge_memberships m
|
|
429
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
430
|
+
WHERE m.source_id = ?
|
|
431
|
+
GROUP BY f.status`)
|
|
432
|
+
.all(String(sourceId));
|
|
433
|
+
for (const r of statusRows) {
|
|
434
|
+
if (r.status === 'indexed' || r.status === 'skipped' || r.status === 'error') {
|
|
435
|
+
statusCounts[r.status] = Number(r.n ?? 0);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
// ext:basename → 后缀(目录名带点不影响);JSON split 规避 SQLite 无 rsplit
|
|
439
|
+
const extRows = this.db.raw
|
|
440
|
+
.prepare(`SELECT lower(ext) AS ext, COUNT(*) AS n FROM (
|
|
441
|
+
SELECT
|
|
442
|
+
CASE
|
|
443
|
+
WHEN instr(name, '.') = 0 OR name IS NULL THEN ''
|
|
444
|
+
ELSE json_extract('["' || replace(name, '.', '","') || '"]', '$[#-1]')
|
|
445
|
+
END AS ext
|
|
446
|
+
FROM (
|
|
447
|
+
SELECT
|
|
448
|
+
json_extract(
|
|
449
|
+
'["' || replace(replace(COALESCE(m.logical_path, ''), char(92), '","'), '/', '","') || '"]',
|
|
450
|
+
'$[#-1]'
|
|
451
|
+
) AS name
|
|
452
|
+
FROM knowledge_memberships m
|
|
453
|
+
WHERE m.source_id = ?
|
|
454
|
+
)
|
|
455
|
+
)
|
|
456
|
+
WHERE ext IS NOT NULL AND ext != ''
|
|
457
|
+
GROUP BY ext`)
|
|
458
|
+
.all(String(sourceId));
|
|
459
|
+
const extCounts = extRows
|
|
460
|
+
.filter((r) => r.ext)
|
|
461
|
+
.map((r) => ({ ext: r.ext, n: Number(r.n ?? 0) }));
|
|
462
|
+
const items = rows.map((row) => {
|
|
463
|
+
const f = rowToFile(row);
|
|
464
|
+
const extName = f.path.includes('.')
|
|
465
|
+
? (f.path.split('.').pop() ?? '').toLowerCase()
|
|
466
|
+
: '';
|
|
467
|
+
return { ...f, ext: extName };
|
|
468
|
+
});
|
|
469
|
+
return {
|
|
470
|
+
items,
|
|
471
|
+
total: Number(totalRow?.n ?? 0),
|
|
472
|
+
page,
|
|
473
|
+
pageSize,
|
|
474
|
+
statusCounts,
|
|
475
|
+
extCounts,
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
listFilePathsFiltered(sourceId, filter) {
|
|
479
|
+
const rows = this.db.raw
|
|
480
|
+
.prepare(`SELECT m.logical_path AS path, f.status AS status
|
|
481
|
+
FROM knowledge_memberships m
|
|
482
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
483
|
+
WHERE m.source_id = ? ORDER BY m.logical_path`)
|
|
484
|
+
.all(String(sourceId));
|
|
485
|
+
let out = rows.map((r) => r.path);
|
|
486
|
+
if (filter?.status) {
|
|
487
|
+
out = rows.filter((r) => r.status === filter.status).map((r) => r.path);
|
|
488
|
+
}
|
|
489
|
+
if (filter?.ext && filter.ext !== 'all') {
|
|
490
|
+
const ext = filter.ext.replace(/^\./, '').toLowerCase();
|
|
491
|
+
out = out.filter((p) => p.toLowerCase().endsWith(`.${ext}`));
|
|
492
|
+
}
|
|
493
|
+
if (filter?.max != null)
|
|
494
|
+
out = out.slice(0, filter.max);
|
|
495
|
+
return out;
|
|
496
|
+
}
|
|
497
|
+
/** 解绑 path;File 仅在零认领时 purge(单文件同步语义,供测试/轻量路径) */
|
|
498
|
+
removeFile(sourceId, path) {
|
|
499
|
+
const logical = path.replace(/\\/g, '/');
|
|
500
|
+
const row = this.db.raw
|
|
501
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
502
|
+
.get(String(sourceId), logical);
|
|
503
|
+
this.db.raw
|
|
504
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
|
|
505
|
+
.run(String(sourceId), logical);
|
|
506
|
+
if (!row)
|
|
507
|
+
return;
|
|
508
|
+
const n = this.db.raw
|
|
509
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
510
|
+
.get(row.file_id).n;
|
|
511
|
+
if (n === 0)
|
|
512
|
+
this.purgeFileRow(row.file_id);
|
|
513
|
+
}
|
|
134
514
|
/**
|
|
135
|
-
*
|
|
515
|
+
* 解绑 path + 可能的 purge(异步让出)。
|
|
516
|
+
* 单文件 purge 也可能删数千 chunk——必须让出,否则 Engine 调度不了 HTTP 回包。
|
|
136
517
|
*/
|
|
137
|
-
|
|
138
|
-
|
|
518
|
+
async removeFileAsync(sourceId, path, opts) {
|
|
519
|
+
const logical = path.replace(/\\/g, '/');
|
|
520
|
+
const row = this.db.raw
|
|
521
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
522
|
+
.get(String(sourceId), logical);
|
|
523
|
+
this.db.raw
|
|
524
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
|
|
525
|
+
.run(String(sourceId), logical);
|
|
526
|
+
if (!row)
|
|
527
|
+
return;
|
|
528
|
+
const n = this.db.raw
|
|
529
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
530
|
+
.get(row.file_id).n;
|
|
531
|
+
if (n === 0)
|
|
532
|
+
await this.purgeFileRowAsync(row.file_id, opts);
|
|
139
533
|
}
|
|
140
534
|
/**
|
|
141
|
-
*
|
|
535
|
+
* 批量解绑:每 `yieldEvery` 条让出事件循环。
|
|
536
|
+
* 删大目录(数百文件)时同步连环 DELETE 会堵死 Engine,HTTP/Query 回包都无法调度。
|
|
142
537
|
*/
|
|
143
|
-
|
|
144
|
-
|
|
538
|
+
async removeFilesAsync(sourceId, paths, opts) {
|
|
539
|
+
const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
|
|
540
|
+
let n = 0;
|
|
541
|
+
for (let i = 0; i < paths.length; i++) {
|
|
542
|
+
await this.removeFileAsync(sourceId, paths[i], opts);
|
|
543
|
+
n += 1;
|
|
544
|
+
if ((i + 1) % yieldEvery === 0) {
|
|
545
|
+
await new Promise((r) => setImmediate(r));
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return n;
|
|
549
|
+
}
|
|
550
|
+
removeFiles(sourceId, paths) {
|
|
551
|
+
for (const p of paths)
|
|
552
|
+
this.removeFile(sourceId, p);
|
|
145
553
|
}
|
|
146
554
|
/**
|
|
147
|
-
*
|
|
555
|
+
* 删除 path 及子树:先收集 logical_path,再分批异步解绑 + purge。
|
|
148
556
|
*/
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
this.
|
|
557
|
+
async removePathTreeAsync(sourceId, path, opts) {
|
|
558
|
+
const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
|
|
559
|
+
const rows = this.db.raw
|
|
560
|
+
.prepare(`SELECT logical_path FROM knowledge_memberships
|
|
561
|
+
WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
|
|
562
|
+
.all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
|
|
563
|
+
return this.removeFilesAsync(sourceId, rows.map((r) => r.logical_path), opts);
|
|
156
564
|
}
|
|
157
|
-
|
|
565
|
+
removePathTree(sourceId, path) {
|
|
566
|
+
const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
|
|
567
|
+
const rows = this.db.raw
|
|
568
|
+
.prepare(`SELECT logical_path FROM knowledge_memberships
|
|
569
|
+
WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
|
|
570
|
+
.all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
|
|
571
|
+
for (const r of rows)
|
|
572
|
+
this.removeFile(sourceId, r.logical_path);
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* 清空源认领:分批 purge,避免一次清数千 File 堵死事件循环。
|
|
576
|
+
*/
|
|
577
|
+
async clearSourceAsync(sourceId, opts) {
|
|
578
|
+
const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
|
|
579
|
+
const rows = this.db.raw
|
|
580
|
+
.prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
|
|
581
|
+
.all(String(sourceId));
|
|
158
582
|
this.db.raw
|
|
159
|
-
.prepare(
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
.
|
|
583
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
|
|
584
|
+
.run(String(sourceId));
|
|
585
|
+
let purged = 0;
|
|
586
|
+
for (let i = 0; i < rows.length; i++) {
|
|
587
|
+
const fileId = rows[i].file_id;
|
|
588
|
+
const n = this.db.raw
|
|
589
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
590
|
+
.get(fileId).n;
|
|
591
|
+
if (n === 0) {
|
|
592
|
+
await this.purgeFileRowAsync(fileId, opts);
|
|
593
|
+
purged += 1;
|
|
594
|
+
}
|
|
595
|
+
if ((i + 1) % yieldEvery === 0) {
|
|
596
|
+
await new Promise((r) => setImmediate(r));
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
return purged;
|
|
600
|
+
}
|
|
601
|
+
clearSource(sourceId) {
|
|
602
|
+
const rows = this.db.raw
|
|
603
|
+
.prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
|
|
604
|
+
.all(String(sourceId));
|
|
164
605
|
this.db.raw
|
|
165
|
-
.prepare('DELETE FROM
|
|
166
|
-
.run(sourceId
|
|
606
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
|
|
607
|
+
.run(String(sourceId));
|
|
608
|
+
for (const r of rows) {
|
|
609
|
+
const n = this.db.raw
|
|
610
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
611
|
+
.get(r.file_id).n;
|
|
612
|
+
if (n === 0)
|
|
613
|
+
this.purgeFileRow(r.file_id);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
purgePath(sourceId, path) {
|
|
617
|
+
this.removeFile(sourceId, path);
|
|
618
|
+
}
|
|
619
|
+
/** 按 fileId 物理删除(membership 已清) */
|
|
620
|
+
purgeFile(fileId) {
|
|
621
|
+
this.purgeFileRow(fileId);
|
|
622
|
+
}
|
|
623
|
+
async purgeFileAsync(fileId, opts) {
|
|
624
|
+
await this.purgeFileRowAsync(fileId, opts);
|
|
625
|
+
}
|
|
626
|
+
purgeSource(sourceId) {
|
|
627
|
+
this.clearSource(sourceId);
|
|
628
|
+
}
|
|
629
|
+
purgeFileRow(fileId) {
|
|
630
|
+
const oldIds = this.db.raw
|
|
631
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
632
|
+
.all(fileId).map((r) => r.id);
|
|
633
|
+
this.dropChunks(oldIds);
|
|
634
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
635
|
+
this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
|
|
636
|
+
this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* purge + 分批让出。大文件数千 chunk 时 dropChunks 同步段仍可能秒级——批间 setImmediate。
|
|
640
|
+
* 与 upsertFile 共用 File 写锁,避免让出窗口被 parse 插入后残留孤儿 FTS。
|
|
641
|
+
*/
|
|
642
|
+
async purgeFileRowAsync(fileId, opts) {
|
|
643
|
+
const keyRow = this.db.raw
|
|
644
|
+
.prepare('SELECT identity_key FROM knowledge_files WHERE id = ?')
|
|
645
|
+
.get(fileId);
|
|
646
|
+
const lockKey = keyRow?.identity_key ?? `id:${fileId}`;
|
|
647
|
+
await this.withFileWriteLock(lockKey, async () => {
|
|
648
|
+
const oldIds = this.db.raw
|
|
649
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
650
|
+
.all(fileId).map((r) => r.id);
|
|
651
|
+
await this.dropChunksAsync(oldIds);
|
|
652
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
653
|
+
this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
|
|
654
|
+
this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
|
|
655
|
+
await new Promise((r) => setImmediate(r));
|
|
656
|
+
});
|
|
657
|
+
void opts;
|
|
658
|
+
}
|
|
659
|
+
dropChunks(chunkIds) {
|
|
660
|
+
if (chunkIds.length === 0)
|
|
661
|
+
return;
|
|
662
|
+
// 分批:大文件重解析时数千 id 一次 IN 会长时间占住 Engine 事件循环
|
|
663
|
+
for (let i = 0; i < chunkIds.length; i += 400) {
|
|
664
|
+
this.dropChunksBatch(chunkIds.slice(i, i + 400));
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
async dropChunksAsync(chunkIds) {
|
|
668
|
+
if (chunkIds.length === 0)
|
|
669
|
+
return;
|
|
670
|
+
for (let i = 0; i < chunkIds.length; i += 400) {
|
|
671
|
+
this.dropChunksBatch(chunkIds.slice(i, i + 400));
|
|
672
|
+
// 批间必须让出:否则单文件数千 chunk 仍会冻住 Engine 数秒
|
|
673
|
+
await new Promise((r) => setImmediate(r));
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
dropChunksBatch(batch) {
|
|
677
|
+
if (batch.length === 0)
|
|
678
|
+
return;
|
|
679
|
+
const ph = batch.map(() => '?').join(',');
|
|
167
680
|
this.db.raw
|
|
168
|
-
.prepare(
|
|
169
|
-
.run(
|
|
681
|
+
.prepare(`DELETE FROM knowledge_chunk_embeddings WHERE chunk_id IN (${ph})`)
|
|
682
|
+
.run(...batch);
|
|
683
|
+
this.deleteKnowledgeVec(batch);
|
|
684
|
+
this.fts.removeMany(batch);
|
|
685
|
+
this.invalidateStatsCache();
|
|
686
|
+
}
|
|
687
|
+
deleteKnowledgeVec(chunkIds) {
|
|
688
|
+
if (!this.db.sqliteVecEnabled || chunkIds.length === 0)
|
|
689
|
+
return;
|
|
690
|
+
try {
|
|
691
|
+
const del = this.db.raw.prepare(`DELETE FROM knowledge_vec WHERE chunk_id IN (${chunkIds.map(() => '?').join(',')})`);
|
|
692
|
+
for (let i = 0; i < chunkIds.length; i += 400) {
|
|
693
|
+
del.run(...chunkIds.slice(i, i + 400));
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
catch {
|
|
697
|
+
// 无 vec 表时忽略
|
|
698
|
+
}
|
|
170
699
|
}
|
|
171
700
|
/**
|
|
172
|
-
*
|
|
173
|
-
* 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
|
|
701
|
+
* 对账差量删除:批量路径,内部让出事件循环。
|
|
174
702
|
*
|
|
175
|
-
* @returns
|
|
703
|
+
* @returns 删除条数
|
|
176
704
|
*/
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
705
|
+
async pruneMissingAsync(sourceId, keepPaths, opts) {
|
|
706
|
+
// keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
|
|
707
|
+
const normalize = (p) => p.replace(/\\/g, '/');
|
|
708
|
+
const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
|
|
709
|
+
// discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
|
|
710
|
+
if (keep.size === 0 && !opts?.allowEmptyKeep)
|
|
180
711
|
return 0;
|
|
181
712
|
const files = this.listFiles(sourceId);
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
713
|
+
const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
|
|
714
|
+
if (gone.length === 0)
|
|
715
|
+
return 0;
|
|
716
|
+
await this.removeFilesAsync(sourceId, gone, opts);
|
|
717
|
+
return gone.length;
|
|
718
|
+
}
|
|
719
|
+
pruneMissing(sourceId, keepPaths, opts) {
|
|
720
|
+
// keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
|
|
721
|
+
const normalize = (p) => p.replace(/\\/g, '/');
|
|
722
|
+
const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
|
|
723
|
+
// discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
|
|
724
|
+
if (keep.size === 0 && !opts?.allowEmptyKeep)
|
|
725
|
+
return 0;
|
|
726
|
+
const files = this.listFiles(sourceId);
|
|
727
|
+
const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
|
|
728
|
+
if (gone.length === 0)
|
|
729
|
+
return 0;
|
|
730
|
+
this.removeFiles(sourceId, gone);
|
|
731
|
+
return gone.length;
|
|
190
732
|
}
|
|
191
733
|
sourceStats(sourceId) {
|
|
734
|
+
return this.cached(`src-stats:${sourceId}`, () => this.computeSourceStats(sourceId));
|
|
735
|
+
}
|
|
736
|
+
computeSourceStats(sourceId) {
|
|
192
737
|
const f = this.db.raw
|
|
193
738
|
.prepare(`SELECT
|
|
194
739
|
COUNT(*) AS total,
|
|
195
|
-
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS errors,
|
|
196
|
-
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped
|
|
197
|
-
FROM
|
|
198
|
-
|
|
740
|
+
SUM(CASE WHEN f.status = 'error' THEN 1 ELSE 0 END) AS errors,
|
|
741
|
+
SUM(CASE WHEN f.status = 'skipped' THEN 1 ELSE 0 END) AS skipped
|
|
742
|
+
FROM knowledge_memberships m
|
|
743
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
744
|
+
WHERE m.source_id = ?`)
|
|
745
|
+
.get(String(sourceId));
|
|
199
746
|
const c = this.db.raw
|
|
200
|
-
.prepare(
|
|
201
|
-
|
|
747
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
748
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
749
|
+
WHERE m.source_id = ?`)
|
|
750
|
+
.get(String(sourceId));
|
|
751
|
+
const ec = this.db.raw
|
|
752
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
753
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
754
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
755
|
+
WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
756
|
+
.get(String(sourceId));
|
|
757
|
+
const e = this.db.raw
|
|
758
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
759
|
+
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
760
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
761
|
+
WHERE m.source_id = ?`)
|
|
762
|
+
.get(String(sourceId));
|
|
202
763
|
return {
|
|
203
764
|
files: f?.total ?? 0,
|
|
204
765
|
chunks: c?.n ?? 0,
|
|
766
|
+
embeddableChunks: ec?.n ?? 0,
|
|
767
|
+
embeddings: e?.n ?? 0,
|
|
205
768
|
errors: f?.errors ?? 0,
|
|
206
769
|
skipped: f?.skipped ?? 0,
|
|
207
770
|
};
|
|
208
771
|
}
|
|
209
772
|
/**
|
|
210
|
-
*
|
|
773
|
+
* 关键词检索。sourceIds = 可见 Membership;chunk 按 file 去重。
|
|
211
774
|
*/
|
|
212
775
|
search(query, opts) {
|
|
213
776
|
const tokens = tokenizeKeywordQuery(query);
|
|
777
|
+
const tokenDetail = tokenizeKeywordDetail(query);
|
|
214
778
|
if (tokens.length === 0 || opts.sourceIds.length === 0)
|
|
215
779
|
return [];
|
|
216
780
|
const limit = opts.limit ?? 8;
|
|
217
781
|
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
hits.push({
|
|
231
|
-
chunkId: asChunkId(row.id),
|
|
232
|
-
sourceId: asSourceId(row.source_id),
|
|
233
|
-
path: row.path,
|
|
234
|
-
ordinal: row.ordinal,
|
|
235
|
-
text: row.text,
|
|
236
|
-
startLine: row.start_line,
|
|
237
|
-
endLine: row.end_line,
|
|
238
|
-
score,
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
hits.sort((a, b) => b.score - a.score);
|
|
242
|
-
return hits.slice(0, limit);
|
|
243
|
-
}
|
|
244
|
-
// ── Phase B 向量 ──
|
|
245
|
-
/**
|
|
246
|
-
* 按 chunk id 读取
|
|
247
|
-
*/
|
|
248
|
-
getChunkById(chunkId) {
|
|
249
|
-
const row = this.db.raw
|
|
250
|
-
.prepare('SELECT id, source_id, path, text, start_line, end_line FROM knowledge_chunks WHERE id = ?')
|
|
251
|
-
.get(chunkId);
|
|
252
|
-
if (!row)
|
|
253
|
-
return null;
|
|
254
|
-
return {
|
|
255
|
-
id: asChunkId(row.id),
|
|
256
|
-
sourceId: asSourceId(row.source_id),
|
|
257
|
-
path: row.path,
|
|
258
|
-
text: row.text,
|
|
259
|
-
startLine: row.start_line,
|
|
260
|
-
endLine: row.end_line,
|
|
261
|
-
};
|
|
262
|
-
}
|
|
263
|
-
/**
|
|
264
|
-
* 按路径列出 chunks(可选行窗)
|
|
265
|
-
*/
|
|
266
|
-
listChunksByPath(sourceId, path, opts) {
|
|
267
|
-
const rows = this.db.raw
|
|
268
|
-
.prepare(`SELECT id, path, text, start_line, end_line FROM knowledge_chunks
|
|
269
|
-
WHERE source_id = ? AND path = ?
|
|
270
|
-
ORDER BY ordinal`)
|
|
271
|
-
.all(sourceId, path);
|
|
272
|
-
let mapped = rows.map((r) => ({
|
|
273
|
-
id: asChunkId(r.id),
|
|
274
|
-
path: r.path,
|
|
275
|
-
text: r.text,
|
|
276
|
-
startLine: r.start_line,
|
|
277
|
-
endLine: r.end_line,
|
|
278
|
-
}));
|
|
279
|
-
if (opts?.startLine != null) {
|
|
280
|
-
mapped = mapped.filter((c) => c.endLine >= opts.startLine);
|
|
782
|
+
let rows = null;
|
|
783
|
+
const ftsIds = this.fts.search(query, opts.sourceIds, Math.max(limit * 4, 32));
|
|
784
|
+
if (ftsIds && ftsIds.length > 0) {
|
|
785
|
+
const idPh = ftsIds.map(() => '?').join(',');
|
|
786
|
+
rows = this.db.raw
|
|
787
|
+
.prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
|
|
788
|
+
m.source_id, m.logical_path
|
|
789
|
+
FROM knowledge_chunks c
|
|
790
|
+
JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
|
|
791
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
792
|
+
WHERE c.id IN (${idPh}) AND m.source_id IN (${placeholders})`)
|
|
793
|
+
.all(...ftsIds, ...opts.sourceIds);
|
|
281
794
|
}
|
|
282
|
-
if (
|
|
283
|
-
|
|
795
|
+
if (!rows || rows.length === 0) {
|
|
796
|
+
const likeClauses = [];
|
|
797
|
+
const likeParams = [];
|
|
798
|
+
for (const token of tokens) {
|
|
799
|
+
const like = `%${token.toLowerCase().replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
|
|
800
|
+
likeClauses.push(`LOWER(c.text) LIKE ? ESCAPE '\\'`);
|
|
801
|
+
likeParams.push(like);
|
|
802
|
+
likeClauses.push(`LOWER(m.logical_path) LIKE ? ESCAPE '\\'`);
|
|
803
|
+
likeParams.push(like);
|
|
804
|
+
}
|
|
805
|
+
rows = this.db.raw
|
|
806
|
+
.prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
|
|
807
|
+
m.source_id, m.logical_path
|
|
808
|
+
FROM knowledge_chunks c
|
|
809
|
+
JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
|
|
810
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
811
|
+
WHERE m.source_id IN (${placeholders})
|
|
812
|
+
AND (${likeClauses.join(' OR ')})
|
|
813
|
+
LIMIT ?`)
|
|
814
|
+
.all(...opts.sourceIds, ...likeParams, Math.max(limit * 8, 64));
|
|
284
815
|
}
|
|
285
|
-
|
|
816
|
+
const byChunk = new Map();
|
|
817
|
+
for (const r of rows) {
|
|
818
|
+
let h = byChunk.get(r.id);
|
|
819
|
+
if (!h) {
|
|
820
|
+
if (byChunk.size >= limit * 3)
|
|
821
|
+
continue;
|
|
822
|
+
const score = scoreKeywordCoverage({ content: r.text, tags: r.logical_path }, tokenDetail);
|
|
823
|
+
h = {
|
|
824
|
+
chunkId: r.id,
|
|
825
|
+
fileId: r.file_id,
|
|
826
|
+
sourceIds: [],
|
|
827
|
+
logicalPaths: {},
|
|
828
|
+
path: r.logical_path,
|
|
829
|
+
ordinal: r.ordinal,
|
|
830
|
+
text: r.text,
|
|
831
|
+
startLine: r.start_line,
|
|
832
|
+
endLine: r.end_line,
|
|
833
|
+
score: score ?? 0,
|
|
834
|
+
};
|
|
835
|
+
byChunk.set(r.id, h);
|
|
836
|
+
}
|
|
837
|
+
if (!h.sourceIds.includes(r.source_id))
|
|
838
|
+
h.sourceIds.push(r.source_id);
|
|
839
|
+
h.logicalPaths[r.source_id] = r.logical_path;
|
|
840
|
+
}
|
|
841
|
+
return [...byChunk.values()]
|
|
842
|
+
.sort((a, b) => b.score - a.score)
|
|
843
|
+
.slice(0, limit)
|
|
844
|
+
.map((h) => ({
|
|
845
|
+
chunkId: asChunkId(h.chunkId),
|
|
846
|
+
sourceId: asSourceId(h.sourceIds[0] ?? ''),
|
|
847
|
+
path: h.path,
|
|
848
|
+
ordinal: h.ordinal,
|
|
849
|
+
text: h.text,
|
|
850
|
+
startLine: h.startLine,
|
|
851
|
+
endLine: h.endLine,
|
|
852
|
+
score: h.score,
|
|
853
|
+
// 契约 §8.4:共享 File 多绑定溯源
|
|
854
|
+
sourceIds: h.sourceIds,
|
|
855
|
+
logicalPaths: h.logicalPaths,
|
|
856
|
+
}));
|
|
286
857
|
}
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
this.db.raw
|
|
292
|
-
.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding_json, created_at)
|
|
293
|
-
VALUES (?, ?, ?, ?)
|
|
294
|
-
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
295
|
-
dimensions=excluded.dimensions,
|
|
296
|
-
embedding_json=excluded.embedding_json,
|
|
297
|
-
created_at=excluded.created_at`)
|
|
298
|
-
.run(chunkId, embedding.length, JSON.stringify(embedding), Date.now());
|
|
858
|
+
vectorBackend() {
|
|
859
|
+
if (this.countEmbeddings() === 0)
|
|
860
|
+
return 'disabled';
|
|
861
|
+
return this.db.sqliteVecEnabled ? 'sqlite-vec' : 'js-bucket';
|
|
299
862
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
863
|
+
vectorSearch(embedding, opts) {
|
|
864
|
+
const sourceIds = opts.sourceIds.map((s) => String(s));
|
|
865
|
+
if (sourceIds.length === 0)
|
|
866
|
+
return [];
|
|
867
|
+
const limit = opts.limit ?? 8;
|
|
868
|
+
const emb = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
|
|
869
|
+
const placeholders = sourceIds.map(() => '?').join(',');
|
|
870
|
+
const joinChunks = `JOIN knowledge_chunks c ON c.id = %s
|
|
871
|
+
JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
|
|
872
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id`;
|
|
873
|
+
const where = `WHERE m.source_id IN (${placeholders})`;
|
|
874
|
+
const load = (ids, scores) => {
|
|
875
|
+
if (ids.length === 0)
|
|
876
|
+
return [];
|
|
877
|
+
const idPh = ids.map(() => '?').join(',');
|
|
878
|
+
const rows = this.db.raw
|
|
879
|
+
.prepare(`SELECT c.id, c.ordinal, c.text, c.start_line, c.end_line,
|
|
880
|
+
m.source_id, m.logical_path
|
|
881
|
+
FROM knowledge_chunks c
|
|
882
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
883
|
+
WHERE c.id IN (${idPh})`)
|
|
884
|
+
.all(...ids);
|
|
885
|
+
// 保持 scores 顺序(SQL IN 不保证序)
|
|
886
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
887
|
+
const ordered = (scores ? ids : ids).map((id) => byId.get(id)).filter(Boolean);
|
|
888
|
+
return ordered.map((r) => ({
|
|
889
|
+
chunkId: asChunkId(r.id),
|
|
890
|
+
sourceId: asSourceId(r.source_id),
|
|
891
|
+
path: r.logical_path,
|
|
892
|
+
ordinal: r.ordinal,
|
|
893
|
+
text: r.text,
|
|
894
|
+
startLine: r.start_line,
|
|
895
|
+
endLine: r.end_line,
|
|
896
|
+
score: scores?.get(r.id) ?? 0,
|
|
897
|
+
}));
|
|
898
|
+
};
|
|
899
|
+
if (this.db.sqliteVecEnabled) {
|
|
900
|
+
// 必须走 vec0 ANN(MATCH + k)。禁止 ORDER BY vec_distance_cosine —— 那是全表暴力扫,
|
|
901
|
+
// 14 万向量会跑到 50s+,直接打穿 query timeout。
|
|
902
|
+
try {
|
|
903
|
+
const ph = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
|
|
904
|
+
// 先 ANN 过采,再按 Membership 裁剪,最后截断
|
|
905
|
+
const k = Math.min(Math.max(limit * 4, 64), 1024);
|
|
906
|
+
const knn = this.db.raw
|
|
907
|
+
.prepare(`SELECT c.id AS id, v.distance AS dist
|
|
908
|
+
FROM (
|
|
909
|
+
SELECT chunk_id, distance FROM knowledge_vec
|
|
910
|
+
WHERE embedding MATCH ? AND k = ?
|
|
911
|
+
) v
|
|
912
|
+
${joinChunks.replace('%s', 'v.chunk_id')} ${where}
|
|
913
|
+
ORDER BY v.distance LIMIT ?`)
|
|
914
|
+
.all(ph, k, ...sourceIds, limit);
|
|
915
|
+
// ANN 完成即权威结果(含 0 命中);不再退 JS 全表
|
|
916
|
+
const scores = new Map(knn.map((r) => [r.id, 1 - Number(r.dist ?? 1)]));
|
|
917
|
+
return load(knn.map((r) => r.id), scores);
|
|
918
|
+
}
|
|
919
|
+
catch {
|
|
920
|
+
// 扩展查询失败才落到有界 JS 路径
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
// 无 sqlite-vec 的有界 JS 路径:只取 id/embedding,不拉 text
|
|
304
924
|
const rows = this.db.raw
|
|
305
|
-
.prepare(`SELECT
|
|
925
|
+
.prepare(`SELECT e.chunk_id AS chunk_id, e.embedding AS embedding
|
|
926
|
+
FROM knowledge_chunk_embeddings e
|
|
927
|
+
${joinChunks.replace('%s', 'e.chunk_id')} ${where}
|
|
928
|
+
LIMIT 2000`)
|
|
929
|
+
.all(...sourceIds);
|
|
930
|
+
if (rows.length === 0)
|
|
931
|
+
return [];
|
|
932
|
+
const qArr = emb instanceof Float32Array ? emb : Float32Array.from(emb);
|
|
933
|
+
const scored = [];
|
|
934
|
+
for (const r of rows) {
|
|
935
|
+
const raw = r.embedding;
|
|
936
|
+
const buf = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
|
|
937
|
+
const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
938
|
+
if (vec.length !== qArr.length)
|
|
939
|
+
continue;
|
|
940
|
+
let dot = 0;
|
|
941
|
+
let nA = 0;
|
|
942
|
+
let nB = 0;
|
|
943
|
+
for (let d = 0; d < qArr.length; d++) {
|
|
944
|
+
const a = qArr[d];
|
|
945
|
+
const b = vec[d];
|
|
946
|
+
dot += a * b;
|
|
947
|
+
nA += a * a;
|
|
948
|
+
nB += b * b;
|
|
949
|
+
}
|
|
950
|
+
const den = Math.sqrt(nA) * Math.sqrt(nB);
|
|
951
|
+
const score = den === 0 ? 0 : dot / den;
|
|
952
|
+
if (score > 0)
|
|
953
|
+
scored.push({ id: r.chunk_id, score });
|
|
954
|
+
}
|
|
955
|
+
scored.sort((a, b) => b.score - a.score);
|
|
956
|
+
const top = scored.slice(0, limit);
|
|
957
|
+
const scores = new Map(top.map((s) => [s.id, s.score]));
|
|
958
|
+
return load(top.map((s) => s.id), scores);
|
|
959
|
+
}
|
|
960
|
+
listChunksMissingEmbedding(sourceId, limit = 50) {
|
|
961
|
+
const rows = this.db.raw
|
|
962
|
+
.prepare(`SELECT c.id AS id, c.text AS text
|
|
306
963
|
FROM knowledge_chunks c
|
|
964
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
965
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
307
966
|
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
308
|
-
WHERE
|
|
309
|
-
ORDER BY c.path, c.ordinal
|
|
967
|
+
WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
|
|
310
968
|
LIMIT ?`)
|
|
311
|
-
.all(sourceId, limit);
|
|
312
|
-
return rows.map((r) => ({
|
|
969
|
+
.all(String(sourceId), limit);
|
|
970
|
+
return rows.map((r) => ({ chunkId: r.id, text: r.text }));
|
|
313
971
|
}
|
|
314
|
-
/**
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
return Math.min(1, (embedded?.n ?? 0) / total.n);
|
|
972
|
+
/** 是否仍有缺向量 chunk(EXISTS,不拉正文) */
|
|
973
|
+
hasChunksMissingEmbedding(sourceId) {
|
|
974
|
+
return this.cached(`emb-miss:${sourceId}`, () => {
|
|
975
|
+
const row = this.db.raw
|
|
976
|
+
.prepare(`SELECT 1 AS ok
|
|
977
|
+
FROM knowledge_chunks c
|
|
978
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
979
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
980
|
+
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
981
|
+
WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
|
|
982
|
+
LIMIT 1`)
|
|
983
|
+
.get(String(sourceId));
|
|
984
|
+
return Boolean(row);
|
|
985
|
+
});
|
|
329
986
|
}
|
|
330
987
|
/**
|
|
331
|
-
*
|
|
988
|
+
* 批量写入嵌入。pairs: `[chunkId, number[]]`;空向量 = secret-skip 墓碑。
|
|
332
989
|
*/
|
|
333
|
-
|
|
334
|
-
if (
|
|
335
|
-
return
|
|
336
|
-
|
|
337
|
-
const
|
|
338
|
-
const
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
990
|
+
setChunkEmbeddings(pairs) {
|
|
991
|
+
if (!pairs || pairs.length === 0)
|
|
992
|
+
return;
|
|
993
|
+
this.db.ensureEmbeddingSchema();
|
|
994
|
+
const now = Date.now();
|
|
995
|
+
const insert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, bucket, created_at)
|
|
996
|
+
VALUES (?, ?, ?, ?, ?)
|
|
997
|
+
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
998
|
+
dimensions = excluded.dimensions,
|
|
999
|
+
embedding = excluded.embedding,
|
|
1000
|
+
bucket = excluded.bucket`);
|
|
1001
|
+
this.db.raw.exec('BEGIN');
|
|
1002
|
+
try {
|
|
1003
|
+
for (const [chunkId, embedding] of pairs) {
|
|
1004
|
+
const vec = Float32Array.from(embedding ?? []);
|
|
1005
|
+
const dims = vec.length;
|
|
1006
|
+
const buf = Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
1007
|
+
insert.run(chunkId, dims, buf, dims > 0 ? vectorBucket(Array.from(vec)) : -1, now);
|
|
1008
|
+
this.writeKnowledgeVec(chunkId, embedding ?? []);
|
|
1009
|
+
}
|
|
1010
|
+
this.db.raw.exec('COMMIT');
|
|
1011
|
+
this.invalidateStatsCache();
|
|
1012
|
+
}
|
|
1013
|
+
catch (err) {
|
|
348
1014
|
try {
|
|
349
|
-
|
|
1015
|
+
this.db.raw.exec('ROLLBACK');
|
|
350
1016
|
}
|
|
351
1017
|
catch {
|
|
352
|
-
|
|
1018
|
+
// 以原异常为准
|
|
353
1019
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
1020
|
+
throw err;
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
setChunkEmbedding(chunkId, embedding) {
|
|
1024
|
+
this.setChunkEmbeddings([[chunkId, embedding]]);
|
|
1025
|
+
}
|
|
1026
|
+
writeKnowledgeVec(chunkId, embedding) {
|
|
1027
|
+
if (!this.db.sqliteVecEnabled || !embedding?.length)
|
|
1028
|
+
return;
|
|
1029
|
+
try {
|
|
1030
|
+
const dims = embedding.length;
|
|
1031
|
+
if (!this.vecTableReady) {
|
|
1032
|
+
this.db.raw.exec(`
|
|
1033
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_vec USING vec0(
|
|
1034
|
+
chunk_id TEXT PRIMARY KEY,
|
|
1035
|
+
embedding float[${dims}] distance_metric=cosine
|
|
1036
|
+
);
|
|
1037
|
+
`);
|
|
1038
|
+
this.vecTableReady = true;
|
|
1039
|
+
}
|
|
1040
|
+
this.db.raw.prepare('DELETE FROM knowledge_vec WHERE chunk_id = ?').run(chunkId);
|
|
1041
|
+
this.db.raw
|
|
1042
|
+
.prepare('INSERT INTO knowledge_vec (chunk_id, embedding) VALUES (?, ?)')
|
|
1043
|
+
.run(chunkId, toVecBlob(embedding));
|
|
1044
|
+
}
|
|
1045
|
+
catch {
|
|
1046
|
+
// 无 sqlite-vec 时忽略
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
embeddingCoverage(sourceId) {
|
|
1050
|
+
return this.cached(`emb-cov:${sourceId ?? 'global'}`, () => this.computeEmbeddingCoverage(sourceId));
|
|
1051
|
+
}
|
|
1052
|
+
computeEmbeddingCoverage(sourceId) {
|
|
1053
|
+
if (sourceId) {
|
|
1054
|
+
const t = this.db.raw
|
|
1055
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
1056
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
1057
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1058
|
+
WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
1059
|
+
.get(String(sourceId)).n;
|
|
1060
|
+
if (t === 0)
|
|
1061
|
+
return 1;
|
|
1062
|
+
const e = this.db.raw
|
|
1063
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
1064
|
+
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
1065
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1066
|
+
WHERE m.source_id = ?`)
|
|
1067
|
+
.get(String(sourceId)).n;
|
|
1068
|
+
return Math.min(1, e / t);
|
|
1069
|
+
}
|
|
1070
|
+
const t = this.db.raw
|
|
1071
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
1072
|
+
JOIN knowledge_files f ON f.id = c.file_id WHERE ${EMBEDDABLE_WHERE}`)
|
|
1073
|
+
.get().n;
|
|
1074
|
+
if (t === 0)
|
|
1075
|
+
return 1;
|
|
1076
|
+
const e = this.db.raw
|
|
1077
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
|
|
1078
|
+
.get().n;
|
|
1079
|
+
return Math.min(1, e / t);
|
|
1080
|
+
}
|
|
1081
|
+
backfillVecFromEmbeddings(batch = 100) {
|
|
1082
|
+
if (!this.db.sqliteVecEnabled)
|
|
1083
|
+
return 0;
|
|
1084
|
+
const rows = this.db.raw
|
|
1085
|
+
.prepare(`SELECT chunk_id, embedding, dimensions FROM knowledge_chunk_embeddings
|
|
1086
|
+
WHERE dimensions > 0 LIMIT ?`)
|
|
1087
|
+
.all(batch);
|
|
1088
|
+
let n = 0;
|
|
1089
|
+
for (const r of rows) {
|
|
1090
|
+
const buf = r.embedding instanceof Uint8Array
|
|
1091
|
+
? r.embedding
|
|
1092
|
+
: new Uint8Array(r.embedding);
|
|
1093
|
+
const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
|
|
1094
|
+
this.writeKnowledgeVec(r.chunk_id, vec);
|
|
1095
|
+
n += 1;
|
|
1096
|
+
}
|
|
1097
|
+
return n;
|
|
1098
|
+
}
|
|
1099
|
+
getChunk(chunkId) {
|
|
1100
|
+
const r = this.db.raw
|
|
1101
|
+
.prepare(`SELECT c.id, c.text, c.start_line, c.end_line, m.source_id, m.logical_path
|
|
1102
|
+
FROM knowledge_chunks c
|
|
1103
|
+
LEFT JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1104
|
+
WHERE c.id = ? LIMIT 1`)
|
|
1105
|
+
.get(chunkId);
|
|
1106
|
+
if (!r)
|
|
1107
|
+
return null;
|
|
1108
|
+
return {
|
|
1109
|
+
id: r.id,
|
|
1110
|
+
sourceId: r.source_id ?? '',
|
|
1111
|
+
path: r.logical_path ?? '',
|
|
1112
|
+
text: r.text,
|
|
1113
|
+
startLine: r.start_line,
|
|
1114
|
+
endLine: r.end_line,
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
1117
|
+
listChunksByPath(sourceId, path) {
|
|
1118
|
+
const logical = path.replace(/\\/g, '/');
|
|
1119
|
+
const rows = this.db.raw
|
|
1120
|
+
.prepare(`SELECT c.id, c.text, c.start_line, c.end_line, c.ordinal
|
|
1121
|
+
FROM knowledge_chunks c
|
|
1122
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1123
|
+
WHERE m.source_id = ? AND m.logical_path = ?
|
|
1124
|
+
ORDER BY c.ordinal`)
|
|
1125
|
+
.all(String(sourceId), logical);
|
|
1126
|
+
return rows.map((r) => ({
|
|
1127
|
+
id: r.id,
|
|
1128
|
+
text: r.text,
|
|
1129
|
+
startLine: r.start_line,
|
|
1130
|
+
endLine: r.end_line,
|
|
1131
|
+
ordinal: r.ordinal,
|
|
1132
|
+
path: logical,
|
|
1133
|
+
}));
|
|
1134
|
+
}
|
|
1135
|
+
countEmbeddings() {
|
|
1136
|
+
return this.db.raw
|
|
1137
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
|
|
1138
|
+
.get().n;
|
|
372
1139
|
}
|
|
373
1140
|
}
|
|
374
|
-
/**
|
|
375
|
-
* 内容指纹(新鲜度判断)
|
|
376
|
-
*
|
|
377
|
-
* @param content - UTF-8 文本或原始字节(二进制文档用 bytes)
|
|
378
|
-
* @returns sha256 hex
|
|
379
|
-
*/
|
|
380
1141
|
export function hashContent(content) {
|
|
381
1142
|
return createHash('sha256').update(content).digest('hex');
|
|
382
1143
|
}
|
|
383
|
-
function rowToFile(row) {
|
|
384
|
-
return {
|
|
385
|
-
id: String(row.id),
|
|
386
|
-
sourceId: asSourceId(String(row.source_id)),
|
|
387
|
-
path: String(row.path),
|
|
388
|
-
contentHash: String(row.content_hash ?? ''),
|
|
389
|
-
size: Number(row.size ?? 0),
|
|
390
|
-
mtime: Number(row.mtime ?? 0),
|
|
391
|
-
adapterId: row.adapter_id == null ? null : String(row.adapter_id),
|
|
392
|
-
status: String(row.status),
|
|
393
|
-
error: row.error == null ? undefined : String(row.error),
|
|
394
|
-
chunkCount: Number(row.chunk_count ?? 0),
|
|
395
|
-
indexedAt: Number(row.indexed_at ?? 0),
|
|
396
|
-
externalUrl: row.external_url == null ? undefined : String(row.external_url),
|
|
397
|
-
etag: row.etag == null ? undefined : String(row.etag),
|
|
398
|
-
lastModified: row.last_modified == null ? undefined : String(row.last_modified),
|
|
399
|
-
};
|
|
400
|
-
}
|