@octopi-agent/engine 0.58.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +1 -1
- package/dist/config-schema/context.js +1 -1
- package/dist/config-schema/engine.d.ts +38 -1
- package/dist/config-schema/substrate.d.ts +37 -0
- package/dist/config-schema/substrate.js +47 -0
- package/dist/config.d.ts +30 -1
- package/dist/harness/agent/builder.d.ts +14 -0
- package/dist/harness/agent/builder.js +14 -2
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
- package/dist/harness/capabilities/document/factory.d.ts +33 -0
- package/dist/harness/capabilities/document/factory.js +37 -0
- package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
- package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
- package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
- package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
- package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
- package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
- package/dist/harness/capabilities/document/worker-port.js +78 -0
- package/dist/harness/context/capabilities/index.d.ts +2 -2
- package/dist/harness/context/capabilities/index.js +2 -2
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
- package/dist/harness/index.d.ts +4 -2
- package/dist/harness/index.js +2 -1
- package/dist/harness/knowledge/adapters.d.ts +8 -5
- package/dist/harness/knowledge/adapters.js +53 -13
- package/dist/harness/knowledge/client.d.ts +197 -0
- package/dist/harness/knowledge/client.js +237 -0
- package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
- package/dist/harness/knowledge/connectors.d.ts +3 -3
- package/dist/harness/knowledge/connectors.js +14 -8
- package/dist/harness/knowledge/db.d.ts +40 -5
- package/dist/harness/knowledge/db.js +441 -113
- package/dist/harness/knowledge/document-extract-worker.js +55 -12
- package/dist/harness/knowledge/embed-runner.d.ts +53 -0
- package/dist/harness/knowledge/embed-runner.js +171 -0
- package/dist/harness/knowledge/engine-thread.d.ts +1 -0
- package/dist/harness/knowledge/engine-thread.js +302 -0
- package/dist/harness/knowledge/extract-document.d.ts +21 -5
- package/dist/harness/knowledge/extract-document.js +32 -8
- package/dist/harness/knowledge/fetchers.d.ts +14 -3
- package/dist/harness/knowledge/fetchers.js +52 -16
- package/dist/harness/knowledge/file-identity.d.ts +53 -0
- package/dist/harness/knowledge/file-identity.js +172 -0
- package/dist/harness/knowledge/file-index-store.d.ts +50 -0
- package/dist/harness/knowledge/file-index-store.js +132 -0
- package/dist/harness/knowledge/file-limits.d.ts +86 -0
- package/dist/harness/knowledge/file-limits.js +175 -0
- package/dist/harness/knowledge/fts.d.ts +60 -0
- package/dist/harness/knowledge/fts.js +218 -0
- package/dist/harness/knowledge/grounding.d.ts +3 -2
- package/dist/harness/knowledge/http-app.d.ts +50 -0
- package/dist/harness/knowledge/http-app.js +652 -0
- package/dist/harness/knowledge/http-bridge.d.ts +49 -0
- package/dist/harness/knowledge/http-bridge.js +17 -0
- package/dist/harness/knowledge/index-store.d.ts +158 -93
- package/dist/harness/knowledge/index-store.js +982 -441
- package/dist/harness/knowledge/index.d.ts +45 -5
- package/dist/harness/knowledge/index.js +23 -2
- package/dist/harness/knowledge/ingest.d.ts +262 -87
- package/dist/harness/knowledge/ingest.js +1072 -480
- package/dist/harness/knowledge/job-control-state.d.ts +29 -0
- package/dist/harness/knowledge/job-control-state.js +44 -0
- package/dist/harness/knowledge/job-control.d.ts +35 -0
- package/dist/harness/knowledge/job-control.js +86 -0
- package/dist/harness/knowledge/job-queue.d.ts +62 -0
- package/dist/harness/knowledge/job-queue.js +173 -0
- package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
- package/dist/harness/knowledge/local-stack.d.ts +27 -0
- package/dist/harness/knowledge/local-stack.js +63 -0
- package/dist/harness/knowledge/membership-store.d.ts +65 -0
- package/dist/harness/knowledge/membership-store.js +198 -0
- package/dist/harness/knowledge/network-guard.d.ts +11 -2
- package/dist/harness/knowledge/network-guard.js +130 -85
- package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
- package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
- package/dist/harness/knowledge/query-service.d.ts +272 -0
- package/dist/harness/knowledge/query-service.js +326 -0
- package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
- package/dist/harness/knowledge/query-worker-client.js +207 -0
- package/dist/harness/knowledge/query-worker.d.ts +1 -0
- package/dist/harness/knowledge/query-worker.js +186 -0
- package/dist/harness/knowledge/read-http.d.ts +35 -0
- package/dist/harness/knowledge/read-http.js +373 -0
- package/dist/harness/knowledge/retriever.d.ts +22 -5
- package/dist/harness/knowledge/retriever.js +66 -35
- package/dist/harness/knowledge/secret-scan.d.ts +13 -2
- package/dist/harness/knowledge/secret-scan.js +25 -2
- package/dist/harness/knowledge/serve.d.ts +48 -0
- package/dist/harness/knowledge/serve.js +336 -0
- package/dist/harness/knowledge/source-store.d.ts +80 -17
- package/dist/harness/knowledge/source-store.js +160 -89
- package/dist/harness/knowledge/start-service-process.d.ts +49 -0
- package/dist/harness/knowledge/start-service-process.js +158 -0
- package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
- package/dist/harness/knowledge/text-parse-worker.js +54 -0
- package/dist/harness/knowledge/types.d.ts +1 -1
- package/dist/harness/knowledge/types.js +1 -1
- package/dist/harness/knowledge/vector-ann.d.ts +42 -0
- package/dist/harness/knowledge/vector-ann.js +68 -0
- package/dist/harness/knowledge/writer-lock.d.ts +13 -0
- package/dist/harness/knowledge/writer-lock.js +71 -0
- package/dist/harness/knowledge/writer-service.d.ts +194 -0
- package/dist/harness/knowledge/writer-service.js +355 -0
- package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
- package/dist/harness/knowledge/writer-worker-client.js +381 -0
- package/dist/harness/knowledge/writer-worker.d.ts +1 -0
- package/dist/harness/knowledge/writer-worker.js +189 -0
- package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
- package/dist/harness/memory/sqlite/embedding.js +109 -16
- package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
- package/dist/harness/memory/sqlite/keyword-search.js +64 -0
- package/dist/harness/session/attachments/service.d.ts +1 -1
- package/dist/harness/session/attachments/service.js +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.js +2 -1
- package/dist/worker-path.d.ts +8 -0
- package/dist/worker-path.js +15 -0
- package/package.json +2 -2
|
@@ -1,69 +1,263 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* KnowledgeIndexStore —
|
|
2
|
+
* KnowledgeIndexStore — File 本位索引投影
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* chunk/file 挂 file_id;logical path 在 Membership。
|
|
5
|
+
* sourceId+path 的 API 保留为「Membership 视图」,内部按 identity 去重。
|
|
5
6
|
*/
|
|
6
7
|
import { randomUUID, createHash } from 'node:crypto';
|
|
7
|
-
import { tokenizeKeywordQuery,
|
|
8
|
-
import {
|
|
8
|
+
import { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordCoverage, } from '../memory/sqlite/keyword-search.js';
|
|
9
|
+
import { KnowledgeFts } from './fts.js';
|
|
10
|
+
import { vectorBucket } from './vector-ann.js';
|
|
11
|
+
import { toVecBlob } from '../memory/sqlite/sqlite-vec.js';
|
|
9
12
|
import { asChunkId, asSourceId } from './types.js';
|
|
10
|
-
|
|
13
|
+
import { normalizePathLexical } from './file-identity.js';
|
|
11
14
|
const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
|
|
15
|
+
function defaultIdentityKey(path) {
|
|
16
|
+
try {
|
|
17
|
+
return `path:${normalizePathLexical(path)}`;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return `path:${path.replace(/\\/g, '/')}`;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function rowToFile(row) {
|
|
24
|
+
return {
|
|
25
|
+
id: String(row.id),
|
|
26
|
+
sourceId: asSourceId(String(row.source_id ?? row.membership_source_id ?? '')),
|
|
27
|
+
path: String(row.logical_path ?? row.path ?? ''),
|
|
28
|
+
contentHash: String(row.content_hash ?? ''),
|
|
29
|
+
size: Number(row.size ?? 0),
|
|
30
|
+
mtime: Number(row.mtime ?? 0),
|
|
31
|
+
adapterId: row.adapter_id == null ? null : String(row.adapter_id),
|
|
32
|
+
status: String(row.status),
|
|
33
|
+
error: row.error == null ? undefined : String(row.error),
|
|
34
|
+
chunkCount: Number(row.chunk_count ?? 0),
|
|
35
|
+
indexedAt: Number(row.indexed_at ?? 0),
|
|
36
|
+
externalUrl: row.external_url == null ? undefined : String(row.external_url),
|
|
37
|
+
etag: row.etag == null ? undefined : String(row.etag),
|
|
38
|
+
lastModified: row.last_modified == null ? undefined : String(row.last_modified),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
12
41
|
export class KnowledgeIndexStore {
|
|
13
42
|
db;
|
|
43
|
+
fts;
|
|
44
|
+
ftsBackfillPromise = null;
|
|
45
|
+
vecTableReady = false;
|
|
46
|
+
/** 同 File 写序(upsert ∥ purge):键 = identity_key */
|
|
47
|
+
writeLocks = new Map();
|
|
48
|
+
/** 大库 COUNT 短 TTL:UI 轮询 / jobControl 不必每请求全表 JOIN */
|
|
49
|
+
statsCache = new Map();
|
|
50
|
+
static STATS_TTL_MS = 1_000;
|
|
14
51
|
constructor(db) {
|
|
15
52
|
this.db = db;
|
|
53
|
+
this.fts = new KnowledgeFts(db);
|
|
54
|
+
}
|
|
55
|
+
cached(key, compute) {
|
|
56
|
+
const now = Date.now();
|
|
57
|
+
const hit = this.statsCache.get(key);
|
|
58
|
+
if (hit && now - hit.at < KnowledgeIndexStore.STATS_TTL_MS)
|
|
59
|
+
return hit.value;
|
|
60
|
+
const value = compute();
|
|
61
|
+
this.statsCache.set(key, { at: now, value });
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
/** 写路径后立刻让统计可见(收尾 force 也可走此绕过 TTL) */
|
|
65
|
+
invalidateStatsCache() {
|
|
66
|
+
this.statsCache.clear();
|
|
16
67
|
}
|
|
17
68
|
/**
|
|
18
|
-
*
|
|
69
|
+
* 按 File identity 串行写(upsert / purge)。
|
|
70
|
+
* purge 在 dropChunks 让出窗口内与 upsert 交错会留下孤儿 FTS / 悬空 membership。
|
|
19
71
|
*/
|
|
20
|
-
|
|
21
|
-
const
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const fileId = existing?.id ?? `kf_${randomUUID().slice(0, 12)}`;
|
|
26
|
-
this.db.raw.exec('BEGIN');
|
|
72
|
+
async withFileWriteLock(key, fn) {
|
|
73
|
+
const prev = this.writeLocks.get(key) ?? Promise.resolve();
|
|
74
|
+
const run = prev.then(() => fn());
|
|
75
|
+
const tail = run.then(() => undefined, () => undefined);
|
|
76
|
+
this.writeLocks.set(key, tail);
|
|
27
77
|
try {
|
|
78
|
+
return await run;
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
if (this.writeLocks.get(key) === tail)
|
|
82
|
+
this.writeLocks.delete(key);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
get ftsAvailable() {
|
|
86
|
+
return this.fts.available;
|
|
87
|
+
}
|
|
88
|
+
get ftsBackfillRunning() {
|
|
89
|
+
return this.ftsBackfillPromise != null;
|
|
90
|
+
}
|
|
91
|
+
ensureFtsBackfill() {
|
|
92
|
+
if (!this.fts.available)
|
|
93
|
+
return Promise.resolve(0);
|
|
94
|
+
if (this.ftsBackfillPromise)
|
|
95
|
+
return this.ftsBackfillPromise;
|
|
96
|
+
const n = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks').get().n;
|
|
97
|
+
const ftsN = this.db.raw.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks_fts').get().n;
|
|
98
|
+
if (n === 0 || ftsN > 0)
|
|
99
|
+
return Promise.resolve(0);
|
|
100
|
+
this.ftsBackfillPromise = this.fts
|
|
101
|
+
.rebuildFromChunksAsync()
|
|
102
|
+
.catch(() => 0)
|
|
103
|
+
.finally(() => {
|
|
104
|
+
this.ftsBackfillPromise = null;
|
|
105
|
+
});
|
|
106
|
+
return this.ftsBackfillPromise;
|
|
107
|
+
}
|
|
108
|
+
/** 解析或创建 File(identity 去重)+ Membership */
|
|
109
|
+
resolveFile(sourceId, path, meta) {
|
|
110
|
+
const identityKey = meta.identityKey ?? defaultIdentityKey(path);
|
|
111
|
+
const tenant = meta.tenantId ?? 'default';
|
|
112
|
+
const now = Date.now();
|
|
113
|
+
let row = this.db.raw
|
|
114
|
+
.prepare('SELECT id, size, mtime FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
|
|
115
|
+
.get(tenant, identityKey);
|
|
116
|
+
if (!row) {
|
|
117
|
+
const id = `kf_${randomUUID().slice(0, 12)}`;
|
|
28
118
|
this.db.raw
|
|
29
|
-
.prepare(`
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
119
|
+
.prepare(`INSERT INTO knowledge_files
|
|
120
|
+
(id, tenant_id, identity_key, size, mtime, content_hash, status, chunk_count, indexed_at)
|
|
121
|
+
VALUES (?, ?, ?, ?, ?, NULL, 'pending', 0, ?)`)
|
|
122
|
+
.run(id, tenant, identityKey, meta.size ?? 0, meta.mtime ?? 0, now);
|
|
123
|
+
row = { id, size: meta.size ?? 0, mtime: meta.mtime ?? 0 };
|
|
124
|
+
}
|
|
125
|
+
else if ((meta.size != null && meta.size !== row.size) ||
|
|
126
|
+
(meta.mtime != null && meta.mtime !== row.mtime)) {
|
|
33
127
|
this.db.raw
|
|
34
|
-
.prepare(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
128
|
+
.prepare('UPDATE knowledge_files SET size = ?, mtime = ? WHERE id = ?')
|
|
129
|
+
.run(meta.size ?? row.size, meta.mtime ?? row.mtime, row.id);
|
|
130
|
+
}
|
|
131
|
+
const logical = path.replace(/\\/g, '/');
|
|
132
|
+
this.db.raw
|
|
133
|
+
.prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
|
|
134
|
+
VALUES (?, ?, ?, ?)
|
|
135
|
+
ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
|
|
136
|
+
.run(sourceId, row.id, logical, now);
|
|
137
|
+
return row.id;
|
|
138
|
+
}
|
|
139
|
+
/** 按 identity_key 查已有 File(不建行);供 job 入队绑定 file_id */
|
|
140
|
+
findFileIdByIdentity(identityKey, tenantId = 'default') {
|
|
141
|
+
const row = this.db.raw
|
|
142
|
+
.prepare('SELECT id FROM knowledge_files WHERE tenant_id = ? AND identity_key = ?')
|
|
143
|
+
.get(tenantId, identityKey);
|
|
144
|
+
return row?.id ?? null;
|
|
145
|
+
}
|
|
146
|
+
/** 只挂 Membership(共享 File:已有 parse 在途时补认领,禁止双 parse) */
|
|
147
|
+
attachMembership(sourceId, fileId, path) {
|
|
148
|
+
const logical = path.replace(/\\/g, '/');
|
|
149
|
+
this.db.raw
|
|
150
|
+
.prepare(`INSERT INTO knowledge_memberships (source_id, file_id, logical_path, created_at)
|
|
151
|
+
VALUES (?, ?, ?, ?)
|
|
152
|
+
ON CONFLICT(source_id, logical_path) DO UPDATE SET file_id = excluded.file_id`)
|
|
153
|
+
.run(sourceId, fileId, logical, Date.now());
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* 替换文件索引。chunk/FTS **分批**写入并让出事件循环,
|
|
157
|
+
* 大 xlsx 数千 chunk 不得整文件一个同步事务堵死 Service。
|
|
158
|
+
*
|
|
159
|
+
* 语义:旧 chunk 先清;`status='indexing'` 直到全部批次落库后才 `indexed`。
|
|
160
|
+
* 检索不认 `indexing`(半成品)。批次失败/中止会清掉 partial chunks,
|
|
161
|
+
* 并留在 `indexing`(chunk_count=0)供 gap 扫描重试——不得把半成品标成 `error`
|
|
162
|
+
* 而让 partial 变得可搜。
|
|
163
|
+
*/
|
|
164
|
+
async upsertFile(input, opts) {
|
|
165
|
+
const lockKey = input.identityKey ?? defaultIdentityKey(input.path);
|
|
166
|
+
return this.withFileWriteLock(lockKey, async () => {
|
|
167
|
+
return this.upsertFileLocked(input, opts);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
async upsertFileLocked(input, opts) {
|
|
171
|
+
const now = Date.now();
|
|
172
|
+
// 小批量 + 批间让出:200/批 的 FTS 写在 8 文件并发 parse 时仍会长时间占住 Engine
|
|
173
|
+
const batchSize = Math.max(20, opts?.batchSize ?? 50);
|
|
174
|
+
const signal = opts?.signal;
|
|
175
|
+
const fileId = this.resolveFile(String(input.sourceId), input.path, {
|
|
176
|
+
identityKey: input.identityKey,
|
|
177
|
+
size: input.size,
|
|
178
|
+
mtime: input.mtime,
|
|
179
|
+
});
|
|
180
|
+
if (signal?.aborted) {
|
|
181
|
+
await this.markUpsertIncomplete(fileId, 'aborted');
|
|
182
|
+
throw new Error('aborted');
|
|
183
|
+
}
|
|
184
|
+
// 旧索引作废(含 embeddings/FTS/vec)— 分批让出,禁止同步连环 DELETE 冻住 Engine
|
|
185
|
+
{
|
|
186
|
+
const oldIds = this.db.raw
|
|
187
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
188
|
+
.all(fileId).map((r) => r.id);
|
|
189
|
+
await this.dropChunksAsync(oldIds);
|
|
190
|
+
this.db.raw.exec('BEGIN');
|
|
191
|
+
try {
|
|
192
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
193
|
+
this.db.raw
|
|
194
|
+
.prepare(`UPDATE knowledge_files
|
|
195
|
+
SET content_hash = ?, size = ?, mtime = ?, adapter_id = ?,
|
|
196
|
+
status = 'indexing', error = NULL, chunk_count = 0, indexed_at = ?,
|
|
197
|
+
external_url = ?, etag = ?, last_modified = ?
|
|
198
|
+
WHERE id = ?`)
|
|
199
|
+
.run(input.contentHash, input.size, input.mtime, input.adapterId, now, input.externalUrl ?? null, input.etag ?? null, input.lastModified ?? null, fileId);
|
|
200
|
+
this.db.raw.exec('COMMIT');
|
|
201
|
+
}
|
|
202
|
+
catch (err) {
|
|
203
|
+
try {
|
|
204
|
+
this.db.raw.exec('ROLLBACK');
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
// 以原异常为准
|
|
208
|
+
}
|
|
209
|
+
throw err;
|
|
55
210
|
}
|
|
56
|
-
this.db.raw.exec('COMMIT');
|
|
57
211
|
}
|
|
58
|
-
|
|
212
|
+
for (let offset = 0; offset < input.chunks.length; offset += batchSize) {
|
|
213
|
+
if (signal?.aborted) {
|
|
214
|
+
await this.markUpsertIncomplete(fileId, 'aborted');
|
|
215
|
+
throw new Error('aborted');
|
|
216
|
+
}
|
|
217
|
+
const batch = input.chunks.slice(offset, offset + batchSize);
|
|
218
|
+
this.db.raw.exec('BEGIN');
|
|
59
219
|
try {
|
|
60
|
-
this.db.raw.
|
|
220
|
+
const insertChunk = this.db.raw.prepare(`INSERT INTO knowledge_chunks (id, file_id, ordinal, text, start_line, end_line)
|
|
221
|
+
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
222
|
+
const ftsRows = [];
|
|
223
|
+
for (const c of batch) {
|
|
224
|
+
const cid = `kc_${randomUUID().slice(0, 12)}`;
|
|
225
|
+
insertChunk.run(cid, fileId, c.ordinal, c.text, c.startLine, c.endLine);
|
|
226
|
+
ftsRows.push({
|
|
227
|
+
id: cid,
|
|
228
|
+
text: c.text,
|
|
229
|
+
path: input.path,
|
|
230
|
+
...(c.ftsToks != null ? { toks: c.ftsToks } : {}),
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
this.fts.upsertMany(ftsRows);
|
|
234
|
+
this.db.raw.exec('COMMIT');
|
|
61
235
|
}
|
|
62
|
-
catch {
|
|
63
|
-
|
|
236
|
+
catch (err) {
|
|
237
|
+
try {
|
|
238
|
+
this.db.raw.exec('ROLLBACK');
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
// 以原异常为准
|
|
242
|
+
}
|
|
243
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
244
|
+
await this.markUpsertIncomplete(fileId, message);
|
|
245
|
+
throw err;
|
|
246
|
+
}
|
|
247
|
+
if (offset + batchSize < input.chunks.length) {
|
|
248
|
+
await new Promise((r) => setImmediate(r));
|
|
64
249
|
}
|
|
65
|
-
throw err;
|
|
66
250
|
}
|
|
251
|
+
if (signal?.aborted) {
|
|
252
|
+
await this.markUpsertIncomplete(fileId, 'aborted');
|
|
253
|
+
throw new Error('aborted');
|
|
254
|
+
}
|
|
255
|
+
this.db.raw
|
|
256
|
+
.prepare(`UPDATE knowledge_files
|
|
257
|
+
SET status = 'indexed', error = NULL, chunk_count = ?, indexed_at = ?
|
|
258
|
+
WHERE id = ?`)
|
|
259
|
+
.run(input.chunks.length, now, fileId);
|
|
260
|
+
this.invalidateStatsCache();
|
|
67
261
|
return {
|
|
68
262
|
id: fileId,
|
|
69
263
|
sourceId: input.sourceId,
|
|
@@ -81,203 +275,491 @@ export class KnowledgeIndexStore {
|
|
|
81
275
|
};
|
|
82
276
|
}
|
|
83
277
|
/**
|
|
84
|
-
*
|
|
278
|
+
* 半写入收尾:清掉已落地的 partial chunks,回到可重试的 `indexing`。
|
|
279
|
+
* 不得标成 `error`——error 行上的 partial 会被检索捞出。
|
|
85
280
|
*/
|
|
86
|
-
|
|
87
|
-
|
|
281
|
+
async markUpsertIncomplete(fileId, message) {
|
|
282
|
+
try {
|
|
283
|
+
const ids = this.db.raw
|
|
284
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
285
|
+
.all(fileId).map((r) => r.id);
|
|
286
|
+
await this.dropChunksAsync(ids);
|
|
287
|
+
this.db.raw.exec('BEGIN');
|
|
288
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
289
|
+
this.db.raw
|
|
290
|
+
.prepare(`UPDATE knowledge_files
|
|
291
|
+
SET status = 'indexing', chunk_count = 0, error = ?, indexed_at = ?
|
|
292
|
+
WHERE id = ?`)
|
|
293
|
+
.run(message, Date.now(), fileId);
|
|
294
|
+
this.db.raw.exec('COMMIT');
|
|
295
|
+
}
|
|
296
|
+
catch {
|
|
297
|
+
try {
|
|
298
|
+
this.db.raw.exec('ROLLBACK');
|
|
299
|
+
}
|
|
300
|
+
catch {
|
|
301
|
+
// 清理失败时至少保证调用方仍看到原异常;半成品由 gap 扫描按 indexing 重做
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
markFileSkipped(sourceId, path, reason, size, identityKey) {
|
|
306
|
+
this.markFileNonIndexed(sourceId, path, 'skipped', reason, size, identityKey);
|
|
307
|
+
}
|
|
308
|
+
markFileError(sourceId, path, error, size, identityKey) {
|
|
309
|
+
this.markFileNonIndexed(sourceId, path, 'error', error, size, identityKey);
|
|
88
310
|
}
|
|
89
311
|
/**
|
|
90
|
-
* 标记 error — **不删** 已有 chunks
|
|
91
|
-
*
|
|
312
|
+
* 标记 skipped/error — **不删** 已有 chunks。
|
|
313
|
+
* 若已有 membership,只改状态;**不得**用 path-identity 新建 File 顶掉带 chunk 的行。
|
|
92
314
|
*/
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
markFileNonIndexed(sourceId, path, status, detail) {
|
|
97
|
-
const now = Date.now();
|
|
315
|
+
markFileNonIndexed(sourceId, path, status, detail, size, identityKey) {
|
|
316
|
+
const logical = path.replace(/\\/g, '/');
|
|
98
317
|
const existing = this.db.raw
|
|
99
|
-
.prepare(
|
|
100
|
-
.get(sourceId,
|
|
101
|
-
|
|
102
|
-
|
|
318
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
319
|
+
.get(String(sourceId), logical);
|
|
320
|
+
if (existing?.file_id) {
|
|
321
|
+
this.db.raw
|
|
322
|
+
.prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
|
|
323
|
+
.run(status, detail, Date.now(), existing.file_id);
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
// 优先真实 identity(win:/unix:),避免 path: 与成功路径分裂出零认领行
|
|
327
|
+
const fileId = this.resolveFile(String(sourceId), path, { size, identityKey });
|
|
103
328
|
this.db.raw
|
|
104
|
-
.prepare(
|
|
105
|
-
|
|
106
|
-
) VALUES (?, ?, ?, '', 0, 0, NULL, ?, ?, ?, ?)
|
|
107
|
-
ON CONFLICT(source_id, path) DO UPDATE SET
|
|
108
|
-
status=excluded.status,
|
|
109
|
-
error=excluded.error,
|
|
110
|
-
indexed_at=excluded.indexed_at`)
|
|
111
|
-
.run(fileId, sourceId, path, status, detail, chunkCount, now);
|
|
329
|
+
.prepare('UPDATE knowledge_files SET status = ?, error = ?, indexed_at = ? WHERE id = ?')
|
|
330
|
+
.run(status, detail, Date.now(), fileId);
|
|
112
331
|
}
|
|
113
332
|
/**
|
|
114
|
-
*
|
|
333
|
+
* 清零认领的 path: 降级 File(错误路径与成功路径 identity 分裂残留)。
|
|
334
|
+
*
|
|
335
|
+
* @returns 删除的 File 行数
|
|
115
336
|
*/
|
|
337
|
+
purgeOrphanPathIdentityFiles() {
|
|
338
|
+
const res = this.db.raw
|
|
339
|
+
.prepare(`DELETE FROM knowledge_files
|
|
340
|
+
WHERE identity_key LIKE 'path:%'
|
|
341
|
+
AND NOT EXISTS (
|
|
342
|
+
SELECT 1 FROM knowledge_memberships m WHERE m.file_id = knowledge_files.id
|
|
343
|
+
)`)
|
|
344
|
+
.run();
|
|
345
|
+
return Number(res.changes ?? 0);
|
|
346
|
+
}
|
|
116
347
|
isFresh(sourceId, path, contentHash) {
|
|
348
|
+
const logical = path.replace(/\\/g, '/');
|
|
349
|
+
const row = this.db.raw
|
|
350
|
+
.prepare(`SELECT f.content_hash AS content_hash, f.chunk_count AS chunk_count
|
|
351
|
+
FROM knowledge_memberships m
|
|
352
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
353
|
+
WHERE m.source_id = ? AND m.logical_path = ? AND f.status <> 'indexing'`)
|
|
354
|
+
.get(String(sourceId), logical);
|
|
355
|
+
return row?.content_hash === contentHash && (row?.chunk_count ?? 0) > 0;
|
|
356
|
+
}
|
|
357
|
+
invalidateFileForReparse(sourceId, path) {
|
|
358
|
+
const logical = path.replace(/\\/g, '/');
|
|
117
359
|
const row = this.db.raw
|
|
118
|
-
.prepare(`SELECT
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
360
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
361
|
+
.get(String(sourceId), logical);
|
|
362
|
+
if (!row)
|
|
363
|
+
return;
|
|
364
|
+
this.db.raw
|
|
365
|
+
.prepare(`UPDATE knowledge_files SET content_hash = '' WHERE id = ?`)
|
|
366
|
+
.run(row.file_id);
|
|
123
367
|
}
|
|
124
368
|
getFile(sourceId, path) {
|
|
369
|
+
const logical = path.replace(/\\/g, '/');
|
|
125
370
|
const row = this.db.raw
|
|
126
|
-
.prepare(
|
|
127
|
-
|
|
371
|
+
.prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
|
|
372
|
+
FROM knowledge_memberships m
|
|
373
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
374
|
+
WHERE m.source_id = ? AND m.logical_path = ?`)
|
|
375
|
+
.get(String(sourceId), logical);
|
|
128
376
|
return row ? rowToFile(row) : null;
|
|
129
377
|
}
|
|
130
378
|
listFiles(sourceId) {
|
|
131
379
|
const rows = this.db.raw
|
|
132
|
-
.prepare(
|
|
133
|
-
|
|
380
|
+
.prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
|
|
381
|
+
FROM knowledge_memberships m
|
|
382
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
383
|
+
WHERE m.source_id = ? ORDER BY m.logical_path`)
|
|
384
|
+
.all(String(sourceId));
|
|
134
385
|
return rows.map(rowToFile);
|
|
135
386
|
}
|
|
387
|
+
listFilesPaged(sourceId, opts) {
|
|
388
|
+
const pageSize = Math.max(1, Math.min(200, opts?.pageSize ?? 50));
|
|
389
|
+
const page = Math.max(1, opts?.page ?? 1);
|
|
390
|
+
const status = opts?.status && opts.status !== 'all' ? opts.status : null;
|
|
391
|
+
const ext = opts?.ext && opts.ext !== 'all'
|
|
392
|
+
? opts.ext.replace(/^\./, '').toLowerCase()
|
|
393
|
+
: null;
|
|
394
|
+
const q = opts?.q?.trim().toLowerCase() || null;
|
|
395
|
+
// SQL 过滤 + LIMIT/OFFSET:禁止全量拉表再在内存分页(1k+ 文件时 UI 轮询会拖垮 Engine/回包)
|
|
396
|
+
const where = ['m.source_id = ?'];
|
|
397
|
+
const params = [String(sourceId)];
|
|
398
|
+
if (status) {
|
|
399
|
+
where.push('f.status = ?');
|
|
400
|
+
params.push(status);
|
|
401
|
+
}
|
|
402
|
+
if (ext) {
|
|
403
|
+
where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
|
|
404
|
+
params.push(`%.${ext.replace(/[\\%_]/g, (c) => `\\${c}`)}`);
|
|
405
|
+
}
|
|
406
|
+
if (q) {
|
|
407
|
+
where.push("LOWER(m.logical_path) LIKE ? ESCAPE '\\'");
|
|
408
|
+
params.push(`%${q.replace(/[\\%_]/g, (c) => `\\${c}`)}%`);
|
|
409
|
+
}
|
|
410
|
+
const whereSql = where.join(' AND ');
|
|
411
|
+
const totalRow = this.db.raw
|
|
412
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_memberships m
|
|
413
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
414
|
+
WHERE ${whereSql}`)
|
|
415
|
+
.get(...params);
|
|
416
|
+
const rows = this.db.raw
|
|
417
|
+
.prepare(`SELECT f.*, m.source_id AS membership_source_id, m.logical_path AS logical_path
|
|
418
|
+
FROM knowledge_memberships m
|
|
419
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
420
|
+
WHERE ${whereSql}
|
|
421
|
+
ORDER BY m.logical_path
|
|
422
|
+
LIMIT ? OFFSET ?`)
|
|
423
|
+
.all(...params, pageSize, (page - 1) * pageSize);
|
|
424
|
+
// counts 走 SQL 聚合:禁止把整源 logical_path 拉进 JS 再计数(UI 每页轮询会打满 Engine)
|
|
425
|
+
const statusCounts = { indexed: 0, skipped: 0, error: 0 };
|
|
426
|
+
const statusRows = this.db.raw
|
|
427
|
+
.prepare(`SELECT f.status AS status, COUNT(*) AS n
|
|
428
|
+
FROM knowledge_memberships m
|
|
429
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
430
|
+
WHERE m.source_id = ?
|
|
431
|
+
GROUP BY f.status`)
|
|
432
|
+
.all(String(sourceId));
|
|
433
|
+
for (const r of statusRows) {
|
|
434
|
+
if (r.status === 'indexed' || r.status === 'skipped' || r.status === 'error') {
|
|
435
|
+
statusCounts[r.status] = Number(r.n ?? 0);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
// ext:basename → 后缀(目录名带点不影响);JSON split 规避 SQLite 无 rsplit
|
|
439
|
+
const extRows = this.db.raw
|
|
440
|
+
.prepare(`SELECT lower(ext) AS ext, COUNT(*) AS n FROM (
|
|
441
|
+
SELECT
|
|
442
|
+
CASE
|
|
443
|
+
WHEN instr(name, '.') = 0 OR name IS NULL THEN ''
|
|
444
|
+
ELSE json_extract('["' || replace(name, '.', '","') || '"]', '$[#-1]')
|
|
445
|
+
END AS ext
|
|
446
|
+
FROM (
|
|
447
|
+
SELECT
|
|
448
|
+
json_extract(
|
|
449
|
+
'["' || replace(replace(COALESCE(m.logical_path, ''), char(92), '","'), '/', '","') || '"]',
|
|
450
|
+
'$[#-1]'
|
|
451
|
+
) AS name
|
|
452
|
+
FROM knowledge_memberships m
|
|
453
|
+
WHERE m.source_id = ?
|
|
454
|
+
)
|
|
455
|
+
)
|
|
456
|
+
WHERE ext IS NOT NULL AND ext != ''
|
|
457
|
+
GROUP BY ext`)
|
|
458
|
+
.all(String(sourceId));
|
|
459
|
+
const extCounts = extRows
|
|
460
|
+
.filter((r) => r.ext)
|
|
461
|
+
.map((r) => ({ ext: r.ext, n: Number(r.n ?? 0) }));
|
|
462
|
+
const items = rows.map((row) => {
|
|
463
|
+
const f = rowToFile(row);
|
|
464
|
+
const extName = f.path.includes('.')
|
|
465
|
+
? (f.path.split('.').pop() ?? '').toLowerCase()
|
|
466
|
+
: '';
|
|
467
|
+
return { ...f, ext: extName };
|
|
468
|
+
});
|
|
469
|
+
return {
|
|
470
|
+
items,
|
|
471
|
+
total: Number(totalRow?.n ?? 0),
|
|
472
|
+
page,
|
|
473
|
+
pageSize,
|
|
474
|
+
statusCounts,
|
|
475
|
+
extCounts,
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
listFilePathsFiltered(sourceId, filter) {
|
|
479
|
+
const rows = this.db.raw
|
|
480
|
+
.prepare(`SELECT m.logical_path AS path, f.status AS status
|
|
481
|
+
FROM knowledge_memberships m
|
|
482
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
483
|
+
WHERE m.source_id = ? ORDER BY m.logical_path`)
|
|
484
|
+
.all(String(sourceId));
|
|
485
|
+
let out = rows.map((r) => r.path);
|
|
486
|
+
if (filter?.status) {
|
|
487
|
+
out = rows.filter((r) => r.status === filter.status).map((r) => r.path);
|
|
488
|
+
}
|
|
489
|
+
if (filter?.ext && filter.ext !== 'all') {
|
|
490
|
+
const ext = filter.ext.replace(/^\./, '').toLowerCase();
|
|
491
|
+
out = out.filter((p) => p.toLowerCase().endsWith(`.${ext}`));
|
|
492
|
+
}
|
|
493
|
+
if (filter?.max != null)
|
|
494
|
+
out = out.slice(0, filter.max);
|
|
495
|
+
return out;
|
|
496
|
+
}
|
|
497
|
+
/** 解绑 path;File 仅在零认领时 purge(单文件同步语义,供测试/轻量路径) */
|
|
498
|
+
removeFile(sourceId, path) {
|
|
499
|
+
const logical = path.replace(/\\/g, '/');
|
|
500
|
+
const row = this.db.raw
|
|
501
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
502
|
+
.get(String(sourceId), logical);
|
|
503
|
+
this.db.raw
|
|
504
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
|
|
505
|
+
.run(String(sourceId), logical);
|
|
506
|
+
if (!row)
|
|
507
|
+
return;
|
|
508
|
+
const n = this.db.raw
|
|
509
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
510
|
+
.get(row.file_id).n;
|
|
511
|
+
if (n === 0)
|
|
512
|
+
this.purgeFileRow(row.file_id);
|
|
513
|
+
}
|
|
136
514
|
/**
|
|
137
|
-
*
|
|
515
|
+
* 解绑 path + 可能的 purge(异步让出)。
|
|
516
|
+
* 单文件 purge 也可能删数千 chunk——必须让出,否则 Engine 调度不了 HTTP 回包。
|
|
138
517
|
*/
|
|
139
|
-
|
|
140
|
-
|
|
518
|
+
async removeFileAsync(sourceId, path, opts) {
|
|
519
|
+
const logical = path.replace(/\\/g, '/');
|
|
520
|
+
const row = this.db.raw
|
|
521
|
+
.prepare(`SELECT file_id FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?`)
|
|
522
|
+
.get(String(sourceId), logical);
|
|
523
|
+
this.db.raw
|
|
524
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ? AND logical_path = ?')
|
|
525
|
+
.run(String(sourceId), logical);
|
|
526
|
+
if (!row)
|
|
527
|
+
return;
|
|
528
|
+
const n = this.db.raw
|
|
529
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
530
|
+
.get(row.file_id).n;
|
|
531
|
+
if (n === 0)
|
|
532
|
+
await this.purgeFileRowAsync(row.file_id, opts);
|
|
141
533
|
}
|
|
142
534
|
/**
|
|
143
|
-
*
|
|
535
|
+
* 批量解绑:每 `yieldEvery` 条让出事件循环。
|
|
536
|
+
* 删大目录(数百文件)时同步连环 DELETE 会堵死 Engine,HTTP/Query 回包都无法调度。
|
|
144
537
|
*/
|
|
145
|
-
|
|
146
|
-
|
|
538
|
+
async removeFilesAsync(sourceId, paths, opts) {
|
|
539
|
+
const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
|
|
540
|
+
let n = 0;
|
|
541
|
+
for (let i = 0; i < paths.length; i++) {
|
|
542
|
+
await this.removeFileAsync(sourceId, paths[i], opts);
|
|
543
|
+
n += 1;
|
|
544
|
+
if ((i + 1) % yieldEvery === 0) {
|
|
545
|
+
await new Promise((r) => setImmediate(r));
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return n;
|
|
549
|
+
}
|
|
550
|
+
removeFiles(sourceId, paths) {
|
|
551
|
+
for (const p of paths)
|
|
552
|
+
this.removeFile(sourceId, p);
|
|
147
553
|
}
|
|
148
554
|
/**
|
|
149
|
-
*
|
|
555
|
+
* 删除 path 及子树:先收集 logical_path,再分批异步解绑 + purge。
|
|
150
556
|
*/
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
this.
|
|
557
|
+
async removePathTreeAsync(sourceId, path, opts) {
|
|
558
|
+
const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
|
|
559
|
+
const rows = this.db.raw
|
|
560
|
+
.prepare(`SELECT logical_path FROM knowledge_memberships
|
|
561
|
+
WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
|
|
562
|
+
.all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
|
|
563
|
+
return this.removeFilesAsync(sourceId, rows.map((r) => r.logical_path), opts);
|
|
158
564
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
.prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?')
|
|
168
|
-
.run(sourceId, path);
|
|
169
|
-
this.db.raw
|
|
170
|
-
.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?')
|
|
171
|
-
.run(sourceId, path);
|
|
565
|
+
removePathTree(sourceId, path) {
|
|
566
|
+
const prefix = path.replace(/\\/g, '/').replace(/\/+$/, '');
|
|
567
|
+
const rows = this.db.raw
|
|
568
|
+
.prepare(`SELECT logical_path FROM knowledge_memberships
|
|
569
|
+
WHERE source_id = ? AND (logical_path = ? OR logical_path LIKE ? ESCAPE '\\')`)
|
|
570
|
+
.all(String(sourceId), prefix, `${prefix.replace(/[\\%_]/g, (c) => `\\${c}`)}/%`);
|
|
571
|
+
for (const r of rows)
|
|
572
|
+
this.removeFile(sourceId, r.logical_path);
|
|
172
573
|
}
|
|
173
574
|
/**
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
* @param sourceId - 知识源 id
|
|
177
|
-
* @param paths - 精确 path 列表
|
|
575
|
+
* 清空源认领:分批 purge,避免一次清数千 File 堵死事件循环。
|
|
178
576
|
*/
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
577
|
+
async clearSourceAsync(sourceId, opts) {
|
|
578
|
+
const yieldEvery = Math.max(1, opts?.yieldEvery ?? 5);
|
|
579
|
+
const rows = this.db.raw
|
|
580
|
+
.prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
|
|
581
|
+
.all(String(sourceId));
|
|
582
|
+
this.db.raw
|
|
583
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
|
|
584
|
+
.run(String(sourceId));
|
|
585
|
+
let purged = 0;
|
|
586
|
+
for (let i = 0; i < rows.length; i++) {
|
|
587
|
+
const fileId = rows[i].file_id;
|
|
588
|
+
const n = this.db.raw
|
|
589
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
590
|
+
.get(fileId).n;
|
|
591
|
+
if (n === 0) {
|
|
592
|
+
await this.purgeFileRowAsync(fileId, opts);
|
|
593
|
+
purged += 1;
|
|
195
594
|
}
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
for (const p of paths) {
|
|
199
|
-
delChunks.run(sourceId, p);
|
|
200
|
-
delFiles.run(sourceId, p);
|
|
595
|
+
if ((i + 1) % yieldEvery === 0) {
|
|
596
|
+
await new Promise((r) => setImmediate(r));
|
|
201
597
|
}
|
|
202
|
-
this.db.raw.exec('COMMIT');
|
|
203
598
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
599
|
+
return purged;
|
|
600
|
+
}
|
|
601
|
+
clearSource(sourceId) {
|
|
602
|
+
const rows = this.db.raw
|
|
603
|
+
.prepare('SELECT file_id FROM knowledge_memberships WHERE source_id = ?')
|
|
604
|
+
.all(String(sourceId));
|
|
605
|
+
this.db.raw
|
|
606
|
+
.prepare('DELETE FROM knowledge_memberships WHERE source_id = ?')
|
|
607
|
+
.run(String(sourceId));
|
|
608
|
+
for (const r of rows) {
|
|
609
|
+
const n = this.db.raw
|
|
610
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_memberships WHERE file_id = ?')
|
|
611
|
+
.get(r.file_id).n;
|
|
612
|
+
if (n === 0)
|
|
613
|
+
this.purgeFileRow(r.file_id);
|
|
207
614
|
}
|
|
208
615
|
}
|
|
616
|
+
purgePath(sourceId, path) {
|
|
617
|
+
this.removeFile(sourceId, path);
|
|
618
|
+
}
|
|
619
|
+
/** 按 fileId 物理删除(membership 已清) */
|
|
620
|
+
purgeFile(fileId) {
|
|
621
|
+
this.purgeFileRow(fileId);
|
|
622
|
+
}
|
|
623
|
+
async purgeFileAsync(fileId, opts) {
|
|
624
|
+
await this.purgeFileRowAsync(fileId, opts);
|
|
625
|
+
}
|
|
626
|
+
purgeSource(sourceId) {
|
|
627
|
+
this.clearSource(sourceId);
|
|
628
|
+
}
|
|
629
|
+
purgeFileRow(fileId) {
|
|
630
|
+
const oldIds = this.db.raw
|
|
631
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
632
|
+
.all(fileId).map((r) => r.id);
|
|
633
|
+
this.dropChunks(oldIds);
|
|
634
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
635
|
+
this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
|
|
636
|
+
this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
|
|
637
|
+
}
|
|
209
638
|
/**
|
|
210
|
-
*
|
|
211
|
-
*
|
|
212
|
-
* @param sourceId - 知识源 id
|
|
213
|
-
* @param path - 文件或目录绝对路径
|
|
639
|
+
* purge + 分批让出。大文件数千 chunk 时 dropChunks 同步段仍可能秒级——批间 setImmediate。
|
|
640
|
+
* 与 upsertFile 共用 File 写锁,避免让出窗口被 parse 插入后残留孤儿 FTS。
|
|
214
641
|
*/
|
|
215
|
-
|
|
216
|
-
this.db.raw
|
|
642
|
+
async purgeFileRowAsync(fileId, opts) {
|
|
643
|
+
const keyRow = this.db.raw
|
|
644
|
+
.prepare('SELECT identity_key FROM knowledge_files WHERE id = ?')
|
|
645
|
+
.get(fileId);
|
|
646
|
+
const lockKey = keyRow?.identity_key ?? `id:${fileId}`;
|
|
647
|
+
await this.withFileWriteLock(lockKey, async () => {
|
|
648
|
+
const oldIds = this.db.raw
|
|
649
|
+
.prepare('SELECT id FROM knowledge_chunks WHERE file_id = ?')
|
|
650
|
+
.all(fileId).map((r) => r.id);
|
|
651
|
+
await this.dropChunksAsync(oldIds);
|
|
652
|
+
this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE file_id = ?').run(fileId);
|
|
653
|
+
this.db.raw.prepare('DELETE FROM knowledge_jobs WHERE file_id = ?').run(fileId);
|
|
654
|
+
this.db.raw.prepare('DELETE FROM knowledge_files WHERE id = ?').run(fileId);
|
|
655
|
+
await new Promise((r) => setImmediate(r));
|
|
656
|
+
});
|
|
657
|
+
void opts;
|
|
658
|
+
}
|
|
659
|
+
dropChunks(chunkIds) {
|
|
660
|
+
if (chunkIds.length === 0)
|
|
661
|
+
return;
|
|
662
|
+
// 分批:大文件重解析时数千 id 一次 IN 会长时间占住 Engine 事件循环
|
|
663
|
+
for (let i = 0; i < chunkIds.length; i += 400) {
|
|
664
|
+
this.dropChunksBatch(chunkIds.slice(i, i + 400));
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
async dropChunksAsync(chunkIds) {
|
|
668
|
+
if (chunkIds.length === 0)
|
|
669
|
+
return;
|
|
670
|
+
for (let i = 0; i < chunkIds.length; i += 400) {
|
|
671
|
+
this.dropChunksBatch(chunkIds.slice(i, i + 400));
|
|
672
|
+
// 批间必须让出:否则单文件数千 chunk 仍会冻住 Engine 数秒
|
|
673
|
+
await new Promise((r) => setImmediate(r));
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
dropChunksBatch(batch) {
|
|
677
|
+
if (batch.length === 0)
|
|
678
|
+
return;
|
|
679
|
+
const ph = batch.map(() => '?').join(',');
|
|
680
|
+
this.db.raw
|
|
681
|
+
.prepare(`DELETE FROM knowledge_chunk_embeddings WHERE chunk_id IN (${ph})`)
|
|
682
|
+
.run(...batch);
|
|
683
|
+
this.deleteKnowledgeVec(batch);
|
|
684
|
+
this.fts.removeMany(batch);
|
|
685
|
+
this.invalidateStatsCache();
|
|
686
|
+
}
|
|
687
|
+
deleteKnowledgeVec(chunkIds) {
|
|
688
|
+
if (!this.db.sqliteVecEnabled || chunkIds.length === 0)
|
|
689
|
+
return;
|
|
217
690
|
try {
|
|
218
|
-
this.
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
.prepare(`DELETE FROM knowledge_chunk_embeddings
|
|
223
|
-
WHERE chunk_id IN (
|
|
224
|
-
SELECT id FROM knowledge_chunks
|
|
225
|
-
WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)
|
|
226
|
-
)`)
|
|
227
|
-
.run(sourceId, likeSlash, likeBack);
|
|
228
|
-
this.db.raw
|
|
229
|
-
.prepare(`DELETE FROM knowledge_chunks WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
|
|
230
|
-
.run(sourceId, likeSlash, likeBack);
|
|
231
|
-
this.db.raw
|
|
232
|
-
.prepare(`DELETE FROM knowledge_files WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
|
|
233
|
-
.run(sourceId, likeSlash, likeBack);
|
|
234
|
-
this.db.raw.exec('COMMIT');
|
|
691
|
+
const del = this.db.raw.prepare(`DELETE FROM knowledge_vec WHERE chunk_id IN (${chunkIds.map(() => '?').join(',')})`);
|
|
692
|
+
for (let i = 0; i < chunkIds.length; i += 400) {
|
|
693
|
+
del.run(...chunkIds.slice(i, i + 400));
|
|
694
|
+
}
|
|
235
695
|
}
|
|
236
|
-
catch
|
|
237
|
-
|
|
238
|
-
throw e;
|
|
696
|
+
catch {
|
|
697
|
+
// 无 vec 表时忽略
|
|
239
698
|
}
|
|
240
699
|
}
|
|
241
700
|
/**
|
|
242
|
-
*
|
|
243
|
-
* 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
|
|
701
|
+
* 对账差量删除:批量路径,内部让出事件循环。
|
|
244
702
|
*
|
|
245
|
-
* @
|
|
246
|
-
* @param opts.allowEmptyKeep - walk 成功且目录为空时允许清空(默认 false)
|
|
247
|
-
* @returns 删除的文件数
|
|
703
|
+
* @returns 删除条数
|
|
248
704
|
*/
|
|
705
|
+
async pruneMissingAsync(sourceId, keepPaths, opts) {
|
|
706
|
+
// keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
|
|
707
|
+
const normalize = (p) => p.replace(/\\/g, '/');
|
|
708
|
+
const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
|
|
709
|
+
// discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
|
|
710
|
+
if (keep.size === 0 && !opts?.allowEmptyKeep)
|
|
711
|
+
return 0;
|
|
712
|
+
const files = this.listFiles(sourceId);
|
|
713
|
+
const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
|
|
714
|
+
if (gone.length === 0)
|
|
715
|
+
return 0;
|
|
716
|
+
await this.removeFilesAsync(sourceId, gone, opts);
|
|
717
|
+
return gone.length;
|
|
718
|
+
}
|
|
249
719
|
pruneMissing(sourceId, keepPaths, opts) {
|
|
250
|
-
|
|
720
|
+
// keep 可能来自 discover(Windows 反斜杠);logical_path 存的是正斜杠。两边都归一再比。
|
|
721
|
+
const normalize = (p) => p.replace(/\\/g, '/');
|
|
722
|
+
const keep = new Set(Array.from(keepPaths instanceof Set ? keepPaths : new Set(keepPaths), normalize));
|
|
723
|
+
// discover 成功且目录为空时允许清空(契约 §6.2);空 keep 默认 no-op 防误清
|
|
251
724
|
if (keep.size === 0 && !opts?.allowEmptyKeep)
|
|
252
725
|
return 0;
|
|
253
726
|
const files = this.listFiles(sourceId);
|
|
254
|
-
const gone = files.filter((f) => !keep.has(f.path)).map((f) => f.path);
|
|
727
|
+
const gone = files.filter((f) => !keep.has(normalize(f.path))).map((f) => f.path);
|
|
255
728
|
if (gone.length === 0)
|
|
256
729
|
return 0;
|
|
257
730
|
this.removeFiles(sourceId, gone);
|
|
258
731
|
return gone.length;
|
|
259
732
|
}
|
|
260
733
|
sourceStats(sourceId) {
|
|
734
|
+
return this.cached(`src-stats:${sourceId}`, () => this.computeSourceStats(sourceId));
|
|
735
|
+
}
|
|
736
|
+
computeSourceStats(sourceId) {
|
|
261
737
|
const f = this.db.raw
|
|
262
738
|
.prepare(`SELECT
|
|
263
739
|
COUNT(*) AS total,
|
|
264
|
-
SUM(CASE WHEN status = 'error' THEN 1 ELSE 0 END) AS errors,
|
|
265
|
-
SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) AS skipped
|
|
266
|
-
FROM
|
|
267
|
-
|
|
740
|
+
SUM(CASE WHEN f.status = 'error' THEN 1 ELSE 0 END) AS errors,
|
|
741
|
+
SUM(CASE WHEN f.status = 'skipped' THEN 1 ELSE 0 END) AS skipped
|
|
742
|
+
FROM knowledge_memberships m
|
|
743
|
+
JOIN knowledge_files f ON f.id = m.file_id
|
|
744
|
+
WHERE m.source_id = ?`)
|
|
745
|
+
.get(String(sourceId));
|
|
268
746
|
const c = this.db.raw
|
|
269
|
-
.prepare(
|
|
270
|
-
|
|
747
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
748
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
749
|
+
WHERE m.source_id = ?`)
|
|
750
|
+
.get(String(sourceId));
|
|
271
751
|
const ec = this.db.raw
|
|
272
752
|
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
753
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
754
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
755
|
+
WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
756
|
+
.get(String(sourceId));
|
|
276
757
|
const e = this.db.raw
|
|
277
758
|
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
278
759
|
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
279
|
-
|
|
280
|
-
|
|
760
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
761
|
+
WHERE m.source_id = ?`)
|
|
762
|
+
.get(String(sourceId));
|
|
281
763
|
return {
|
|
282
764
|
files: f?.total ?? 0,
|
|
283
765
|
chunks: c?.n ?? 0,
|
|
@@ -288,315 +770,374 @@ export class KnowledgeIndexStore {
|
|
|
288
770
|
};
|
|
289
771
|
}
|
|
290
772
|
/**
|
|
291
|
-
*
|
|
773
|
+
* 关键词检索。sourceIds = 可见 Membership;chunk 按 file 去重。
|
|
292
774
|
*/
|
|
293
775
|
search(query, opts) {
|
|
294
776
|
const tokens = tokenizeKeywordQuery(query);
|
|
777
|
+
const tokenDetail = tokenizeKeywordDetail(query);
|
|
295
778
|
if (tokens.length === 0 || opts.sourceIds.length === 0)
|
|
296
779
|
return [];
|
|
297
780
|
const limit = opts.limit ?? 8;
|
|
298
781
|
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
782
|
+
let rows = null;
|
|
783
|
+
const ftsIds = this.fts.search(query, opts.sourceIds, Math.max(limit * 4, 32));
|
|
784
|
+
if (ftsIds && ftsIds.length > 0) {
|
|
785
|
+
const idPh = ftsIds.map(() => '?').join(',');
|
|
786
|
+
rows = this.db.raw
|
|
787
|
+
.prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
|
|
788
|
+
m.source_id, m.logical_path
|
|
789
|
+
FROM knowledge_chunks c
|
|
790
|
+
JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
|
|
791
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
792
|
+
WHERE c.id IN (${idPh}) AND m.source_id IN (${placeholders})`)
|
|
793
|
+
.all(...ftsIds, ...opts.sourceIds);
|
|
794
|
+
}
|
|
795
|
+
if (!rows || rows.length === 0) {
|
|
796
|
+
const likeClauses = [];
|
|
797
|
+
const likeParams = [];
|
|
798
|
+
for (const token of tokens) {
|
|
799
|
+
const like = `%${token.toLowerCase().replace(/[\\%_]/g, (c) => `\\${c}`)}%`;
|
|
800
|
+
likeClauses.push(`LOWER(c.text) LIKE ? ESCAPE '\\'`);
|
|
801
|
+
likeParams.push(like);
|
|
802
|
+
likeClauses.push(`LOWER(m.logical_path) LIKE ? ESCAPE '\\'`);
|
|
803
|
+
likeParams.push(like);
|
|
804
|
+
}
|
|
805
|
+
rows = this.db.raw
|
|
806
|
+
.prepare(`SELECT c.id, c.file_id, c.ordinal, c.text, c.start_line, c.end_line,
|
|
807
|
+
m.source_id, m.logical_path
|
|
808
|
+
FROM knowledge_chunks c
|
|
809
|
+
JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
|
|
810
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
811
|
+
WHERE m.source_id IN (${placeholders})
|
|
812
|
+
AND (${likeClauses.join(' OR ')})
|
|
813
|
+
LIMIT ?`)
|
|
814
|
+
.all(...opts.sourceIds, ...likeParams, Math.max(limit * 8, 64));
|
|
815
|
+
}
|
|
816
|
+
const byChunk = new Map();
|
|
817
|
+
for (const r of rows) {
|
|
818
|
+
let h = byChunk.get(r.id);
|
|
819
|
+
if (!h) {
|
|
820
|
+
if (byChunk.size >= limit * 3)
|
|
821
|
+
continue;
|
|
822
|
+
const score = scoreKeywordCoverage({ content: r.text, tags: r.logical_path }, tokenDetail);
|
|
823
|
+
h = {
|
|
824
|
+
chunkId: r.id,
|
|
825
|
+
fileId: r.file_id,
|
|
826
|
+
sourceIds: [],
|
|
827
|
+
logicalPaths: {},
|
|
828
|
+
path: r.logical_path,
|
|
829
|
+
ordinal: r.ordinal,
|
|
830
|
+
text: r.text,
|
|
831
|
+
startLine: r.start_line,
|
|
832
|
+
endLine: r.end_line,
|
|
833
|
+
score: score ?? 0,
|
|
834
|
+
};
|
|
835
|
+
byChunk.set(r.id, h);
|
|
836
|
+
}
|
|
837
|
+
if (!h.sourceIds.includes(r.source_id))
|
|
838
|
+
h.sourceIds.push(r.source_id);
|
|
839
|
+
h.logicalPaths[r.source_id] = r.logical_path;
|
|
840
|
+
}
|
|
841
|
+
return [...byChunk.values()]
|
|
842
|
+
.sort((a, b) => b.score - a.score)
|
|
843
|
+
.slice(0, limit)
|
|
844
|
+
.map((h) => ({
|
|
845
|
+
chunkId: asChunkId(h.chunkId),
|
|
846
|
+
sourceId: asSourceId(h.sourceIds[0] ?? ''),
|
|
847
|
+
path: h.path,
|
|
848
|
+
ordinal: h.ordinal,
|
|
849
|
+
text: h.text,
|
|
850
|
+
startLine: h.startLine,
|
|
851
|
+
endLine: h.endLine,
|
|
852
|
+
score: h.score,
|
|
853
|
+
// 契约 §8.4:共享 File 多绑定溯源
|
|
854
|
+
sourceIds: h.sourceIds,
|
|
855
|
+
logicalPaths: h.logicalPaths,
|
|
856
|
+
}));
|
|
857
|
+
}
|
|
858
|
+
vectorBackend() {
|
|
859
|
+
if (this.countEmbeddings() === 0)
|
|
860
|
+
return 'disabled';
|
|
861
|
+
return this.db.sqliteVecEnabled ? 'sqlite-vec' : 'js-bucket';
|
|
862
|
+
}
|
|
863
|
+
vectorSearch(embedding, opts) {
|
|
864
|
+
const sourceIds = opts.sourceIds.map((s) => String(s));
|
|
865
|
+
if (sourceIds.length === 0)
|
|
866
|
+
return [];
|
|
867
|
+
const limit = opts.limit ?? 8;
|
|
868
|
+
const emb = embedding instanceof Float32Array ? embedding : Float32Array.from(embedding);
|
|
869
|
+
const placeholders = sourceIds.map(() => '?').join(',');
|
|
870
|
+
const joinChunks = `JOIN knowledge_chunks c ON c.id = %s
|
|
871
|
+
JOIN knowledge_files f ON f.id = c.file_id AND f.status <> 'indexing'
|
|
872
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id`;
|
|
873
|
+
const where = `WHERE m.source_id IN (${placeholders})`;
|
|
874
|
+
const load = (ids, scores) => {
|
|
875
|
+
if (ids.length === 0)
|
|
876
|
+
return [];
|
|
877
|
+
const idPh = ids.map(() => '?').join(',');
|
|
878
|
+
const rows = this.db.raw
|
|
879
|
+
.prepare(`SELECT c.id, c.ordinal, c.text, c.start_line, c.end_line,
|
|
880
|
+
m.source_id, m.logical_path
|
|
881
|
+
FROM knowledge_chunks c
|
|
882
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
883
|
+
WHERE c.id IN (${idPh})`)
|
|
884
|
+
.all(...ids);
|
|
885
|
+
// 保持 scores 顺序(SQL IN 不保证序)
|
|
886
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
887
|
+
const ordered = (scores ? ids : ids).map((id) => byId.get(id)).filter(Boolean);
|
|
888
|
+
return ordered.map((r) => ({
|
|
889
|
+
chunkId: asChunkId(r.id),
|
|
890
|
+
sourceId: asSourceId(r.source_id),
|
|
891
|
+
path: r.logical_path,
|
|
892
|
+
ordinal: r.ordinal,
|
|
893
|
+
text: r.text,
|
|
894
|
+
startLine: r.start_line,
|
|
895
|
+
endLine: r.end_line,
|
|
896
|
+
score: scores?.get(r.id) ?? 0,
|
|
897
|
+
}));
|
|
898
|
+
};
|
|
899
|
+
if (this.db.sqliteVecEnabled) {
|
|
900
|
+
// 必须走 vec0 ANN(MATCH + k)。禁止 ORDER BY vec_distance_cosine —— 那是全表暴力扫,
|
|
901
|
+
// 14 万向量会跑到 50s+,直接打穿 query timeout。
|
|
902
|
+
try {
|
|
903
|
+
const ph = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
|
|
904
|
+
// 先 ANN 过采,再按 Membership 裁剪,最后截断
|
|
905
|
+
const k = Math.min(Math.max(limit * 4, 64), 1024);
|
|
906
|
+
const knn = this.db.raw
|
|
907
|
+
.prepare(`SELECT c.id AS id, v.distance AS dist
|
|
908
|
+
FROM (
|
|
909
|
+
SELECT chunk_id, distance FROM knowledge_vec
|
|
910
|
+
WHERE embedding MATCH ? AND k = ?
|
|
911
|
+
) v
|
|
912
|
+
${joinChunks.replace('%s', 'v.chunk_id')} ${where}
|
|
913
|
+
ORDER BY v.distance LIMIT ?`)
|
|
914
|
+
.all(ph, k, ...sourceIds, limit);
|
|
915
|
+
// ANN 完成即权威结果(含 0 命中);不再退 JS 全表
|
|
916
|
+
const scores = new Map(knn.map((r) => [r.id, 1 - Number(r.dist ?? 1)]));
|
|
917
|
+
return load(knn.map((r) => r.id), scores);
|
|
918
|
+
}
|
|
919
|
+
catch {
|
|
920
|
+
// 扩展查询失败才落到有界 JS 路径
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
// 无 sqlite-vec 的有界 JS 路径:只取 id/embedding,不拉 text
|
|
299
924
|
const rows = this.db.raw
|
|
300
|
-
.prepare(`SELECT
|
|
301
|
-
FROM
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
925
|
+
.prepare(`SELECT e.chunk_id AS chunk_id, e.embedding AS embedding
|
|
926
|
+
FROM knowledge_chunk_embeddings e
|
|
927
|
+
${joinChunks.replace('%s', 'e.chunk_id')} ${where}
|
|
928
|
+
LIMIT 2000`)
|
|
929
|
+
.all(...sourceIds);
|
|
930
|
+
if (rows.length === 0)
|
|
931
|
+
return [];
|
|
932
|
+
const qArr = emb instanceof Float32Array ? emb : Float32Array.from(emb);
|
|
933
|
+
const scored = [];
|
|
934
|
+
for (const r of rows) {
|
|
935
|
+
const raw = r.embedding;
|
|
936
|
+
const buf = raw instanceof Uint8Array ? raw : new Uint8Array(raw);
|
|
937
|
+
const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
938
|
+
if (vec.length !== qArr.length)
|
|
310
939
|
continue;
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
const row = this.db.raw
|
|
331
|
-
.prepare('SELECT id, source_id, path, text, start_line, end_line FROM knowledge_chunks WHERE id = ?')
|
|
332
|
-
.get(chunkId);
|
|
333
|
-
if (!row)
|
|
334
|
-
return null;
|
|
335
|
-
return {
|
|
336
|
-
id: asChunkId(row.id),
|
|
337
|
-
sourceId: asSourceId(row.source_id),
|
|
338
|
-
path: row.path,
|
|
339
|
-
text: row.text,
|
|
340
|
-
startLine: row.start_line,
|
|
341
|
-
endLine: row.end_line,
|
|
342
|
-
};
|
|
940
|
+
let dot = 0;
|
|
941
|
+
let nA = 0;
|
|
942
|
+
let nB = 0;
|
|
943
|
+
for (let d = 0; d < qArr.length; d++) {
|
|
944
|
+
const a = qArr[d];
|
|
945
|
+
const b = vec[d];
|
|
946
|
+
dot += a * b;
|
|
947
|
+
nA += a * a;
|
|
948
|
+
nB += b * b;
|
|
949
|
+
}
|
|
950
|
+
const den = Math.sqrt(nA) * Math.sqrt(nB);
|
|
951
|
+
const score = den === 0 ? 0 : dot / den;
|
|
952
|
+
if (score > 0)
|
|
953
|
+
scored.push({ id: r.chunk_id, score });
|
|
954
|
+
}
|
|
955
|
+
scored.sort((a, b) => b.score - a.score);
|
|
956
|
+
const top = scored.slice(0, limit);
|
|
957
|
+
const scores = new Map(top.map((s) => [s.id, s.score]));
|
|
958
|
+
return load(top.map((s) => s.id), scores);
|
|
343
959
|
}
|
|
344
|
-
|
|
345
|
-
* 按路径列出 chunks(可选行窗)
|
|
346
|
-
*/
|
|
347
|
-
listChunksByPath(sourceId, path, opts) {
|
|
960
|
+
listChunksMissingEmbedding(sourceId, limit = 50) {
|
|
348
961
|
const rows = this.db.raw
|
|
349
|
-
.prepare(`SELECT id
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
endLine: r.end_line,
|
|
359
|
-
}));
|
|
360
|
-
if (opts?.startLine != null) {
|
|
361
|
-
mapped = mapped.filter((c) => c.endLine >= opts.startLine);
|
|
362
|
-
}
|
|
363
|
-
if (opts?.endLine != null) {
|
|
364
|
-
mapped = mapped.filter((c) => c.startLine <= opts.endLine);
|
|
365
|
-
}
|
|
366
|
-
return mapped;
|
|
962
|
+
.prepare(`SELECT c.id AS id, c.text AS text
|
|
963
|
+
FROM knowledge_chunks c
|
|
964
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
965
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
966
|
+
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
967
|
+
WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
|
|
968
|
+
LIMIT ?`)
|
|
969
|
+
.all(String(sourceId), limit);
|
|
970
|
+
return rows.map((r) => ({ chunkId: r.id, text: r.text }));
|
|
367
971
|
}
|
|
368
|
-
/**
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
972
|
+
/** 是否仍有缺向量 chunk(EXISTS,不拉正文) */
|
|
973
|
+
hasChunksMissingEmbedding(sourceId) {
|
|
974
|
+
return this.cached(`emb-miss:${sourceId}`, () => {
|
|
975
|
+
const row = this.db.raw
|
|
976
|
+
.prepare(`SELECT 1 AS ok
|
|
977
|
+
FROM knowledge_chunks c
|
|
978
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
979
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
980
|
+
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
981
|
+
WHERE m.source_id = ? AND e.chunk_id IS NULL AND ${EMBEDDABLE_WHERE}
|
|
982
|
+
LIMIT 1`)
|
|
983
|
+
.get(String(sourceId));
|
|
984
|
+
return Boolean(row);
|
|
985
|
+
});
|
|
373
986
|
}
|
|
374
987
|
/**
|
|
375
|
-
*
|
|
376
|
-
*
|
|
377
|
-
* @param rows - [chunkId, embedding] 列表
|
|
988
|
+
* 批量写入嵌入。pairs: `[chunkId, number[]]`;空向量 = secret-skip 墓碑。
|
|
378
989
|
*/
|
|
379
|
-
setChunkEmbeddings(
|
|
380
|
-
if (
|
|
990
|
+
setChunkEmbeddings(pairs) {
|
|
991
|
+
if (!pairs || pairs.length === 0)
|
|
381
992
|
return;
|
|
382
|
-
|
|
383
|
-
|
|
993
|
+
this.db.ensureEmbeddingSchema();
|
|
994
|
+
const now = Date.now();
|
|
995
|
+
const insert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, bucket, created_at)
|
|
996
|
+
VALUES (?, ?, ?, ?, ?)
|
|
384
997
|
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
385
|
-
dimensions=excluded.dimensions,
|
|
386
|
-
embedding=excluded.embedding,
|
|
387
|
-
|
|
998
|
+
dimensions = excluded.dimensions,
|
|
999
|
+
embedding = excluded.embedding,
|
|
1000
|
+
bucket = excluded.bucket`);
|
|
388
1001
|
this.db.raw.exec('BEGIN');
|
|
389
1002
|
try {
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
1003
|
+
for (const [chunkId, embedding] of pairs) {
|
|
1004
|
+
const vec = Float32Array.from(embedding ?? []);
|
|
1005
|
+
const dims = vec.length;
|
|
1006
|
+
const buf = Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
1007
|
+
insert.run(chunkId, dims, buf, dims > 0 ? vectorBucket(Array.from(vec)) : -1, now);
|
|
1008
|
+
this.writeKnowledgeVec(chunkId, embedding ?? []);
|
|
394
1009
|
}
|
|
395
1010
|
this.db.raw.exec('COMMIT');
|
|
1011
|
+
this.invalidateStatsCache();
|
|
396
1012
|
}
|
|
397
|
-
catch (
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
/** sqlite-vec 表名 */
|
|
403
|
-
static VEC_TABLE = 'knowledge_vec';
|
|
404
|
-
/** 确保 vec0 虚拟表(维度首次写入时确定) */
|
|
405
|
-
ensureVecTable(dimensions) {
|
|
406
|
-
if (!this.db.sqliteVecEnabled)
|
|
407
|
-
return false;
|
|
408
|
-
try {
|
|
409
|
-
const row = this.db.raw
|
|
410
|
-
.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
|
|
411
|
-
.get(KnowledgeIndexStore.VEC_TABLE);
|
|
412
|
-
if (row?.sql) {
|
|
413
|
-
const m = /float\[(\d+)\]/i.exec(row.sql);
|
|
414
|
-
return !m || Number(m[1]) === dimensions;
|
|
1013
|
+
catch (err) {
|
|
1014
|
+
try {
|
|
1015
|
+
this.db.raw.exec('ROLLBACK');
|
|
415
1016
|
}
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
);
|
|
421
|
-
`);
|
|
422
|
-
return true;
|
|
423
|
-
}
|
|
424
|
-
catch {
|
|
425
|
-
return false;
|
|
1017
|
+
catch {
|
|
1018
|
+
// 以原异常为准
|
|
1019
|
+
}
|
|
1020
|
+
throw err;
|
|
426
1021
|
}
|
|
427
1022
|
}
|
|
428
|
-
|
|
429
|
-
|
|
1023
|
+
setChunkEmbedding(chunkId, embedding) {
|
|
1024
|
+
this.setChunkEmbeddings([[chunkId, embedding]]);
|
|
1025
|
+
}
|
|
1026
|
+
writeKnowledgeVec(chunkId, embedding) {
|
|
1027
|
+
if (!this.db.sqliteVecEnabled || !embedding?.length)
|
|
430
1028
|
return;
|
|
431
1029
|
try {
|
|
1030
|
+
const dims = embedding.length;
|
|
1031
|
+
if (!this.vecTableReady) {
|
|
1032
|
+
this.db.raw.exec(`
|
|
1033
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_vec USING vec0(
|
|
1034
|
+
chunk_id TEXT PRIMARY KEY,
|
|
1035
|
+
embedding float[${dims}] distance_metric=cosine
|
|
1036
|
+
);
|
|
1037
|
+
`);
|
|
1038
|
+
this.vecTableReady = true;
|
|
1039
|
+
}
|
|
1040
|
+
this.db.raw.prepare('DELETE FROM knowledge_vec WHERE chunk_id = ?').run(chunkId);
|
|
432
1041
|
this.db.raw
|
|
433
|
-
.prepare(
|
|
434
|
-
|
|
435
|
-
ON CONFLICT(chunk_id) DO UPDATE SET embedding = excluded.embedding`)
|
|
436
|
-
.run(String(chunkId), toF32Blob(embedding));
|
|
1042
|
+
.prepare('INSERT INTO knowledge_vec (chunk_id, embedding) VALUES (?, ?)')
|
|
1043
|
+
.run(chunkId, toVecBlob(embedding));
|
|
437
1044
|
}
|
|
438
1045
|
catch {
|
|
439
|
-
// vec
|
|
1046
|
+
// 无 sqlite-vec 时忽略
|
|
440
1047
|
}
|
|
441
1048
|
}
|
|
442
|
-
/**
|
|
443
|
-
* 列出待嵌入 chunk(无向量行)
|
|
444
|
-
*
|
|
445
|
-
* **代码不进向量**:原文件已可 file_search;片段易失真。
|
|
446
|
-
* 仅文档/正文类(text / markdown / document:* 等)参与 embedding。
|
|
447
|
-
*/
|
|
448
|
-
listChunksMissingEmbedding(sourceId, limit = 100) {
|
|
449
|
-
const rows = this.db.raw
|
|
450
|
-
.prepare(`SELECT c.id, c.text, c.path
|
|
451
|
-
FROM knowledge_chunks c
|
|
452
|
-
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
453
|
-
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
454
|
-
WHERE c.source_id = ? AND e.chunk_id IS NULL
|
|
455
|
-
AND ${EMBEDDABLE_WHERE}
|
|
456
|
-
ORDER BY c.path, c.ordinal
|
|
457
|
-
LIMIT ?`)
|
|
458
|
-
.all(sourceId, limit);
|
|
459
|
-
return rows.map((r) => ({ id: asChunkId(r.id), text: r.text, path: r.path }));
|
|
460
|
-
}
|
|
461
|
-
/**
|
|
462
|
-
* 向量覆盖度(0–1;无可嵌入 chunk 时为 1)
|
|
463
|
-
*/
|
|
464
1049
|
embeddingCoverage(sourceId) {
|
|
465
|
-
|
|
1050
|
+
return this.cached(`emb-cov:${sourceId ?? 'global'}`, () => this.computeEmbeddingCoverage(sourceId));
|
|
1051
|
+
}
|
|
1052
|
+
computeEmbeddingCoverage(sourceId) {
|
|
1053
|
+
if (sourceId) {
|
|
1054
|
+
const t = this.db.raw
|
|
1055
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
1056
|
+
JOIN knowledge_files f ON f.id = c.file_id
|
|
1057
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1058
|
+
WHERE m.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
1059
|
+
.get(String(sourceId)).n;
|
|
1060
|
+
if (t === 0)
|
|
1061
|
+
return 1;
|
|
1062
|
+
const e = this.db.raw
|
|
1063
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
1064
|
+
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
1065
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1066
|
+
WHERE m.source_id = ?`)
|
|
1067
|
+
.get(String(sourceId)).n;
|
|
1068
|
+
return Math.min(1, e / t);
|
|
1069
|
+
}
|
|
1070
|
+
const t = this.db.raw
|
|
466
1071
|
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
if (!total?.n)
|
|
1072
|
+
JOIN knowledge_files f ON f.id = c.file_id WHERE ${EMBEDDABLE_WHERE}`)
|
|
1073
|
+
.get().n;
|
|
1074
|
+
if (t === 0)
|
|
471
1075
|
return 1;
|
|
472
|
-
const
|
|
473
|
-
.prepare(
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
477
|
-
.get(sourceId);
|
|
478
|
-
return Math.min(1, (embedded?.n ?? 0) / total.n);
|
|
1076
|
+
const e = this.db.raw
|
|
1077
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
|
|
1078
|
+
.get().n;
|
|
1079
|
+
return Math.min(1, e / t);
|
|
479
1080
|
}
|
|
480
|
-
|
|
481
|
-
* 向量检索:优先 sqlite-vec KNN;否则 JS cosine(BLOB)
|
|
482
|
-
*/
|
|
483
|
-
vectorSearch(queryEmbedding, opts) {
|
|
484
|
-
if (opts.sourceIds.length === 0 || !queryEmbedding?.length)
|
|
485
|
-
return [];
|
|
486
|
-
const limit = opts.limit ?? 8;
|
|
487
|
-
// vec 表可能尚未回填;0 命中时退 JS,避免空结果短路
|
|
488
|
-
const viaVec = this.vectorSearchVec(queryEmbedding, opts, limit);
|
|
489
|
-
if (viaVec && viaVec.length > 0)
|
|
490
|
-
return viaVec;
|
|
491
|
-
return this.vectorSearchJs(queryEmbedding, opts, limit);
|
|
492
|
-
}
|
|
493
|
-
/** sqlite-vec KNN(不可用时返回 null) */
|
|
494
|
-
vectorSearchVec(queryEmbedding, opts, limit) {
|
|
1081
|
+
backfillVecFromEmbeddings(batch = 100) {
|
|
495
1082
|
if (!this.db.sqliteVecEnabled)
|
|
496
|
-
return
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
JOIN knowledge_chunks c ON c.id = v.chunk_id
|
|
510
|
-
WHERE c.source_id IN (${placeholders})
|
|
511
|
-
ORDER BY v.distance ASC`)
|
|
512
|
-
.all(toF32Blob(queryEmbedding), Math.max(limit * 2, 16), ...opts.sourceIds);
|
|
513
|
-
return rows.slice(0, limit).map((row) => ({
|
|
514
|
-
chunkId: asChunkId(row.id),
|
|
515
|
-
sourceId: asSourceId(row.source_id),
|
|
516
|
-
path: row.path,
|
|
517
|
-
ordinal: row.ordinal,
|
|
518
|
-
text: row.text,
|
|
519
|
-
startLine: row.start_line,
|
|
520
|
-
endLine: row.end_line,
|
|
521
|
-
score: 1 - Number(row.distance),
|
|
522
|
-
}));
|
|
1083
|
+
return 0;
|
|
1084
|
+
const rows = this.db.raw
|
|
1085
|
+
.prepare(`SELECT chunk_id, embedding, dimensions FROM knowledge_chunk_embeddings
|
|
1086
|
+
WHERE dimensions > 0 LIMIT ?`)
|
|
1087
|
+
.all(batch);
|
|
1088
|
+
let n = 0;
|
|
1089
|
+
for (const r of rows) {
|
|
1090
|
+
const buf = r.embedding instanceof Uint8Array
|
|
1091
|
+
? r.embedding
|
|
1092
|
+
: new Uint8Array(r.embedding);
|
|
1093
|
+
const vec = Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
|
|
1094
|
+
this.writeKnowledgeVec(r.chunk_id, vec);
|
|
1095
|
+
n += 1;
|
|
523
1096
|
}
|
|
524
|
-
|
|
1097
|
+
return n;
|
|
1098
|
+
}
|
|
1099
|
+
getChunk(chunkId) {
|
|
1100
|
+
const r = this.db.raw
|
|
1101
|
+
.prepare(`SELECT c.id, c.text, c.start_line, c.end_line, m.source_id, m.logical_path
|
|
1102
|
+
FROM knowledge_chunks c
|
|
1103
|
+
LEFT JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1104
|
+
WHERE c.id = ? LIMIT 1`)
|
|
1105
|
+
.get(chunkId);
|
|
1106
|
+
if (!r)
|
|
525
1107
|
return null;
|
|
526
|
-
|
|
1108
|
+
return {
|
|
1109
|
+
id: r.id,
|
|
1110
|
+
sourceId: r.source_id ?? '',
|
|
1111
|
+
path: r.logical_path ?? '',
|
|
1112
|
+
text: r.text,
|
|
1113
|
+
startLine: r.start_line,
|
|
1114
|
+
endLine: r.end_line,
|
|
1115
|
+
};
|
|
527
1116
|
}
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
1117
|
+
listChunksByPath(sourceId, path) {
|
|
1118
|
+
const logical = path.replace(/\\/g, '/');
|
|
531
1119
|
const rows = this.db.raw
|
|
532
|
-
.prepare(`SELECT c.id, c.
|
|
1120
|
+
.prepare(`SELECT c.id, c.text, c.start_line, c.end_line, c.ordinal
|
|
533
1121
|
FROM knowledge_chunks c
|
|
534
|
-
JOIN
|
|
535
|
-
WHERE
|
|
536
|
-
|
|
537
|
-
.all(
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
text: row.text,
|
|
552
|
-
startLine: row.start_line,
|
|
553
|
-
endLine: row.end_line,
|
|
554
|
-
score: sim,
|
|
555
|
-
});
|
|
556
|
-
}
|
|
557
|
-
hits.sort((a, b) => b.score - a.score);
|
|
558
|
-
return hits.slice(0, limit);
|
|
1122
|
+
JOIN knowledge_memberships m ON m.file_id = c.file_id
|
|
1123
|
+
WHERE m.source_id = ? AND m.logical_path = ?
|
|
1124
|
+
ORDER BY c.ordinal`)
|
|
1125
|
+
.all(String(sourceId), logical);
|
|
1126
|
+
return rows.map((r) => ({
|
|
1127
|
+
id: r.id,
|
|
1128
|
+
text: r.text,
|
|
1129
|
+
startLine: r.start_line,
|
|
1130
|
+
endLine: r.end_line,
|
|
1131
|
+
ordinal: r.ordinal,
|
|
1132
|
+
path: logical,
|
|
1133
|
+
}));
|
|
1134
|
+
}
|
|
1135
|
+
countEmbeddings() {
|
|
1136
|
+
return this.db.raw
|
|
1137
|
+
.prepare('SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings')
|
|
1138
|
+
.get().n;
|
|
559
1139
|
}
|
|
560
1140
|
}
|
|
561
|
-
/** number[] → float32 blob */
|
|
562
|
-
function toF32Blob(embedding) {
|
|
563
|
-
const f32 = Float32Array.from(embedding);
|
|
564
|
-
return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
|
|
565
|
-
}
|
|
566
|
-
/** float32 blob → number[] */
|
|
567
|
-
function fromF32Blob(buf) {
|
|
568
|
-
if (!buf || buf.byteLength === 0)
|
|
569
|
-
return null;
|
|
570
|
-
if (buf.byteLength % 4 !== 0)
|
|
571
|
-
return null;
|
|
572
|
-
const u8 = buf instanceof Buffer ? new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) : buf;
|
|
573
|
-
const f32 = new Float32Array(u8.buffer, u8.byteOffset, u8.byteLength / 4);
|
|
574
|
-
return Array.from(f32);
|
|
575
|
-
}
|
|
576
|
-
/**
|
|
577
|
-
* 内容指纹(新鲜度判断)
|
|
578
|
-
*
|
|
579
|
-
* @param content - UTF-8 文本或原始字节(二进制文档用 bytes)
|
|
580
|
-
* @returns sha256 hex
|
|
581
|
-
*/
|
|
582
1141
|
export function hashContent(content) {
|
|
583
1142
|
return createHash('sha256').update(content).digest('hex');
|
|
584
1143
|
}
|
|
585
|
-
function rowToFile(row) {
|
|
586
|
-
return {
|
|
587
|
-
id: String(row.id),
|
|
588
|
-
sourceId: asSourceId(String(row.source_id)),
|
|
589
|
-
path: String(row.path),
|
|
590
|
-
contentHash: String(row.content_hash ?? ''),
|
|
591
|
-
size: Number(row.size ?? 0),
|
|
592
|
-
mtime: Number(row.mtime ?? 0),
|
|
593
|
-
adapterId: row.adapter_id == null ? null : String(row.adapter_id),
|
|
594
|
-
status: String(row.status),
|
|
595
|
-
error: row.error == null ? undefined : String(row.error),
|
|
596
|
-
chunkCount: Number(row.chunk_count ?? 0),
|
|
597
|
-
indexedAt: Number(row.indexed_at ?? 0),
|
|
598
|
-
externalUrl: row.external_url == null ? undefined : String(row.external_url),
|
|
599
|
-
etag: row.etag == null ? undefined : String(row.etag),
|
|
600
|
-
lastModified: row.last_modified == null ? undefined : String(row.last_modified),
|
|
601
|
-
};
|
|
602
|
-
}
|