@octopi-agent/engine 0.56.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/engine.d.ts +5 -0
- package/dist/config-schema/substrate.d.ts +5 -0
- package/dist/config-schema/substrate.js +11 -0
- package/dist/config.d.ts +9 -0
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +1 -1
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/db.d.ts +4 -0
- package/dist/harness/knowledge/db.js +58 -3
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +25 -0
- package/dist/harness/knowledge/extract-document.d.ts +17 -0
- package/dist/harness/knowledge/extract-document.js +62 -0
- package/dist/harness/knowledge/index-store.d.ts +42 -4
- package/dist/harness/knowledge/index-store.js +234 -32
- package/dist/harness/knowledge/index.d.ts +2 -0
- package/dist/harness/knowledge/index.js +1 -0
- package/dist/harness/knowledge/ingest.d.ts +100 -2
- package/dist/harness/knowledge/ingest.js +582 -58
- package/dist/harness/knowledge/retriever.d.ts +8 -0
- package/dist/harness/knowledge/retriever.js +33 -1
- package/dist/harness/knowledge/source-store.d.ts +4 -0
- package/dist/harness/knowledge/source-store.js +62 -6
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
- package/dist/harness/memory/sqlite/keyword-search.js +40 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/package.json +2 -2
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ExtractResult } from '../context/capabilities/document/types.js';
|
|
2
|
+
export interface DocumentExtractWorkerOptions {
|
|
3
|
+
/** 抽取超时(默认 60s;到点 terminate worker) */
|
|
4
|
+
timeoutMs?: number;
|
|
5
|
+
maxFileBytes?: number;
|
|
6
|
+
/** 中止信号:abort 时立刻 terminate worker */
|
|
7
|
+
signal?: AbortSignal;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* 在 worker 线程抽取文档为 Markdown。
|
|
11
|
+
*
|
|
12
|
+
* @param filePath - 本地文件路径
|
|
13
|
+
* @param options - 超时 / 大小 / 中止信号
|
|
14
|
+
* @returns ExtractResult
|
|
15
|
+
* @throws Error 超时 / abort / worker 失败 / 抽取错误
|
|
16
|
+
*/
|
|
17
|
+
export declare function extractDocumentInWorker(filePath: string, options?: DocumentExtractWorkerOptions): Promise<ExtractResult>;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* extractDocumentInWorker — 线程内文档抽取 + 可终止超时/中止
|
|
3
|
+
*/
|
|
4
|
+
import { Worker } from 'node:worker_threads';
|
|
5
|
+
/**
|
|
6
|
+
* 在 worker 线程抽取文档为 Markdown。
|
|
7
|
+
*
|
|
8
|
+
* @param filePath - 本地文件路径
|
|
9
|
+
* @param options - 超时 / 大小 / 中止信号
|
|
10
|
+
* @returns ExtractResult
|
|
11
|
+
* @throws Error 超时 / abort / worker 失败 / 抽取错误
|
|
12
|
+
*/
|
|
13
|
+
export function extractDocumentInWorker(filePath, options = {}) {
|
|
14
|
+
const timeoutMs = options.timeoutMs ?? 60_000;
|
|
15
|
+
return new Promise((resolve, reject) => {
|
|
16
|
+
if (options.signal?.aborted) {
|
|
17
|
+
reject(new Error('extract_aborted'));
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
let settled = false;
|
|
21
|
+
const worker = new Worker(new URL('./document-extract-worker.js', import.meta.url), {
|
|
22
|
+
workerData: {
|
|
23
|
+
path: filePath,
|
|
24
|
+
name: filePath,
|
|
25
|
+
timeoutMs,
|
|
26
|
+
maxFileBytes: options.maxFileBytes,
|
|
27
|
+
},
|
|
28
|
+
});
|
|
29
|
+
const finish = (fn) => {
|
|
30
|
+
if (settled)
|
|
31
|
+
return;
|
|
32
|
+
settled = true;
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
35
|
+
void worker.terminate();
|
|
36
|
+
fn();
|
|
37
|
+
};
|
|
38
|
+
const onAbort = () => {
|
|
39
|
+
finish(() => reject(new Error('extract_aborted')));
|
|
40
|
+
};
|
|
41
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
42
|
+
const timer = setTimeout(() => {
|
|
43
|
+
finish(() => reject(new Error(`extract_worker_timeout after ${timeoutMs}ms`)));
|
|
44
|
+
}, timeoutMs);
|
|
45
|
+
worker.on('message', (msg) => {
|
|
46
|
+
if (msg?.ok && msg.result) {
|
|
47
|
+
finish(() => resolve(msg.result));
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
finish(() => reject(new Error(msg?.error || 'extract_worker_failed')));
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
worker.on('error', (err) => {
|
|
54
|
+
finish(() => reject(err));
|
|
55
|
+
});
|
|
56
|
+
worker.on('exit', (code) => {
|
|
57
|
+
if (code !== 0) {
|
|
58
|
+
finish(() => reject(new Error(`extract_worker_exit ${code}`)));
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
}
|
|
@@ -79,16 +79,36 @@ export declare class KnowledgeIndexStore {
|
|
|
79
79
|
*/
|
|
80
80
|
clearSource(sourceId: KnowledgeSourceId | string): void;
|
|
81
81
|
removeFile(sourceId: KnowledgeSourceId | string, path: string): void;
|
|
82
|
+
/**
|
|
83
|
+
* 批量删除 path(同一事务,减少 sync 往返)
|
|
84
|
+
*
|
|
85
|
+
* @param sourceId - 知识源 id
|
|
86
|
+
* @param paths - 精确 path 列表
|
|
87
|
+
*/
|
|
88
|
+
removeFiles(sourceId: KnowledgeSourceId | string, paths: string[]): void;
|
|
89
|
+
/**
|
|
90
|
+
* 删除路径及子树(目录被移出 / 删除)
|
|
91
|
+
*
|
|
92
|
+
* @param sourceId - 知识源 id
|
|
93
|
+
* @param path - 文件或目录绝对路径
|
|
94
|
+
*/
|
|
95
|
+
removePathTree(sourceId: KnowledgeSourceId | string, path: string): void;
|
|
82
96
|
/**
|
|
83
97
|
* 差量 prune:删除本轮 discover **未出现** 的 path(含 chunks)。
|
|
84
98
|
* 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
|
|
85
99
|
*
|
|
100
|
+
* @param keepPaths - 应保留的 path 集
|
|
101
|
+
* @param opts.allowEmptyKeep - walk 成功且目录为空时允许清空(默认 false)
|
|
86
102
|
* @returns 删除的文件数
|
|
87
103
|
*/
|
|
88
|
-
pruneMissing(sourceId: KnowledgeSourceId | string, keepPaths: ReadonlySet<string> | Iterable<string
|
|
104
|
+
pruneMissing(sourceId: KnowledgeSourceId | string, keepPaths: ReadonlySet<string> | Iterable<string>, opts?: {
|
|
105
|
+
allowEmptyKeep?: boolean;
|
|
106
|
+
}): number;
|
|
89
107
|
sourceStats(sourceId: KnowledgeSourceId | string): {
|
|
90
108
|
files: number;
|
|
91
109
|
chunks: number;
|
|
110
|
+
embeddableChunks: number;
|
|
111
|
+
embeddings: number;
|
|
92
112
|
errors: number;
|
|
93
113
|
skipped: number;
|
|
94
114
|
};
|
|
@@ -124,11 +144,25 @@ export declare class KnowledgeIndexStore {
|
|
|
124
144
|
endLine: number;
|
|
125
145
|
}>;
|
|
126
146
|
/**
|
|
127
|
-
* 写入 chunk 向量(
|
|
147
|
+
* 写入 chunk 向量(Float32 BLOB;sqlite-vec 可用时同步进 vec 表)
|
|
128
148
|
*/
|
|
129
149
|
setChunkEmbedding(chunkId: KnowledgeChunkId | string, embedding: number[]): void;
|
|
150
|
+
/**
|
|
151
|
+
* 批量写向量(单事务,减少与 parse 写路径的锁竞争)
|
|
152
|
+
*
|
|
153
|
+
* @param rows - [chunkId, embedding] 列表
|
|
154
|
+
*/
|
|
155
|
+
setChunkEmbeddings(rows: Array<[string, number[]]>): void;
|
|
156
|
+
/** sqlite-vec 表名 */
|
|
157
|
+
private static readonly VEC_TABLE;
|
|
158
|
+
/** 确保 vec0 虚拟表(维度首次写入时确定) */
|
|
159
|
+
private ensureVecTable;
|
|
160
|
+
private upsertKnowledgeVec;
|
|
130
161
|
/**
|
|
131
162
|
* 列出待嵌入 chunk(无向量行)
|
|
163
|
+
*
|
|
164
|
+
* **代码不进向量**:原文件已可 file_search;片段易失真。
|
|
165
|
+
* 仅文档/正文类(text / markdown / document:* 等)参与 embedding。
|
|
132
166
|
*/
|
|
133
167
|
listChunksMissingEmbedding(sourceId: KnowledgeSourceId | string, limit?: number): Array<{
|
|
134
168
|
id: KnowledgeChunkId;
|
|
@@ -136,16 +170,20 @@ export declare class KnowledgeIndexStore {
|
|
|
136
170
|
path: string;
|
|
137
171
|
}>;
|
|
138
172
|
/**
|
|
139
|
-
* 向量覆盖度(0–1
|
|
173
|
+
* 向量覆盖度(0–1;无可嵌入 chunk 时为 1)
|
|
140
174
|
*/
|
|
141
175
|
embeddingCoverage(sourceId: KnowledgeSourceId | string): number;
|
|
142
176
|
/**
|
|
143
|
-
*
|
|
177
|
+
* 向量检索:优先 sqlite-vec KNN;否则 JS cosine(BLOB)
|
|
144
178
|
*/
|
|
145
179
|
vectorSearch(queryEmbedding: number[], opts: {
|
|
146
180
|
sourceIds: KnowledgeSourceId[];
|
|
147
181
|
limit?: number;
|
|
148
182
|
}): ChunkHit[];
|
|
183
|
+
/** sqlite-vec KNN(不可用时返回 null) */
|
|
184
|
+
private vectorSearchVec;
|
|
185
|
+
/** JS 余弦(Float32 BLOB;无 JSON.parse) */
|
|
186
|
+
private vectorSearchJs;
|
|
149
187
|
}
|
|
150
188
|
/**
|
|
151
189
|
* 内容指纹(新鲜度判断)
|
|
@@ -7,6 +7,8 @@ import { randomUUID, createHash } from 'node:crypto';
|
|
|
7
7
|
import { tokenizeKeywordQuery, scoreKeywordFields } from '../memory/sqlite/keyword-search.js';
|
|
8
8
|
import { cosineSimilarity } from '../memory/sqlite/vector-search.js';
|
|
9
9
|
import { asChunkId, asSourceId } from './types.js';
|
|
10
|
+
/** 可嵌入向量的 chunk 条件:非 code-tree(代码走 file_search / 原文) */
|
|
11
|
+
const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
|
|
10
12
|
export class KnowledgeIndexStore {
|
|
11
13
|
db;
|
|
12
14
|
constructor(db) {
|
|
@@ -168,25 +170,92 @@ export class KnowledgeIndexStore {
|
|
|
168
170
|
.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?')
|
|
169
171
|
.run(sourceId, path);
|
|
170
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* 批量删除 path(同一事务,减少 sync 往返)
|
|
175
|
+
*
|
|
176
|
+
* @param sourceId - 知识源 id
|
|
177
|
+
* @param paths - 精确 path 列表
|
|
178
|
+
*/
|
|
179
|
+
removeFiles(sourceId, paths) {
|
|
180
|
+
if (paths.length === 0)
|
|
181
|
+
return;
|
|
182
|
+
const chunkIds = [];
|
|
183
|
+
const select = this.db.raw.prepare(`SELECT id FROM knowledge_chunks WHERE source_id = ? AND path = ?`);
|
|
184
|
+
for (const p of paths) {
|
|
185
|
+
const rows = select.all(sourceId, p);
|
|
186
|
+
for (const r of rows)
|
|
187
|
+
chunkIds.push(r.id);
|
|
188
|
+
}
|
|
189
|
+
this.db.raw.exec('BEGIN');
|
|
190
|
+
try {
|
|
191
|
+
if (chunkIds.length > 0) {
|
|
192
|
+
const delEmb = this.db.raw.prepare('DELETE FROM knowledge_chunk_embeddings WHERE chunk_id = ?');
|
|
193
|
+
for (const id of chunkIds)
|
|
194
|
+
delEmb.run(id);
|
|
195
|
+
}
|
|
196
|
+
const delChunks = this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?');
|
|
197
|
+
const delFiles = this.db.raw.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?');
|
|
198
|
+
for (const p of paths) {
|
|
199
|
+
delChunks.run(sourceId, p);
|
|
200
|
+
delFiles.run(sourceId, p);
|
|
201
|
+
}
|
|
202
|
+
this.db.raw.exec('COMMIT');
|
|
203
|
+
}
|
|
204
|
+
catch (e) {
|
|
205
|
+
this.db.raw.exec('ROLLBACK');
|
|
206
|
+
throw e;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* 删除路径及子树(目录被移出 / 删除)
|
|
211
|
+
*
|
|
212
|
+
* @param sourceId - 知识源 id
|
|
213
|
+
* @param path - 文件或目录绝对路径
|
|
214
|
+
*/
|
|
215
|
+
removePathTree(sourceId, path) {
|
|
216
|
+
this.db.raw.exec('BEGIN');
|
|
217
|
+
try {
|
|
218
|
+
this.removeFile(sourceId, path);
|
|
219
|
+
const likeSlash = `${path}/%`;
|
|
220
|
+
const likeBack = `${path}\\%`;
|
|
221
|
+
this.db.raw
|
|
222
|
+
.prepare(`DELETE FROM knowledge_chunk_embeddings
|
|
223
|
+
WHERE chunk_id IN (
|
|
224
|
+
SELECT id FROM knowledge_chunks
|
|
225
|
+
WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)
|
|
226
|
+
)`)
|
|
227
|
+
.run(sourceId, likeSlash, likeBack);
|
|
228
|
+
this.db.raw
|
|
229
|
+
.prepare(`DELETE FROM knowledge_chunks WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
|
|
230
|
+
.run(sourceId, likeSlash, likeBack);
|
|
231
|
+
this.db.raw
|
|
232
|
+
.prepare(`DELETE FROM knowledge_files WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
|
|
233
|
+
.run(sourceId, likeSlash, likeBack);
|
|
234
|
+
this.db.raw.exec('COMMIT');
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
this.db.raw.exec('ROLLBACK');
|
|
238
|
+
throw e;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
171
241
|
/**
|
|
172
242
|
* 差量 prune:删除本轮 discover **未出现** 的 path(含 chunks)。
|
|
173
243
|
* 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
|
|
174
244
|
*
|
|
245
|
+
* @param keepPaths - 应保留的 path 集
|
|
246
|
+
* @param opts.allowEmptyKeep - walk 成功且目录为空时允许清空(默认 false)
|
|
175
247
|
* @returns 删除的文件数
|
|
176
248
|
*/
|
|
177
|
-
pruneMissing(sourceId, keepPaths) {
|
|
249
|
+
pruneMissing(sourceId, keepPaths, opts) {
|
|
178
250
|
const keep = keepPaths instanceof Set ? keepPaths : new Set(keepPaths);
|
|
179
|
-
if (keep.size === 0)
|
|
251
|
+
if (keep.size === 0 && !opts?.allowEmptyKeep)
|
|
180
252
|
return 0;
|
|
181
253
|
const files = this.listFiles(sourceId);
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
return removed;
|
|
254
|
+
const gone = files.filter((f) => !keep.has(f.path)).map((f) => f.path);
|
|
255
|
+
if (gone.length === 0)
|
|
256
|
+
return 0;
|
|
257
|
+
this.removeFiles(sourceId, gone);
|
|
258
|
+
return gone.length;
|
|
190
259
|
}
|
|
191
260
|
sourceStats(sourceId) {
|
|
192
261
|
const f = this.db.raw
|
|
@@ -199,9 +268,21 @@ export class KnowledgeIndexStore {
|
|
|
199
268
|
const c = this.db.raw
|
|
200
269
|
.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
|
|
201
270
|
.get(sourceId);
|
|
271
|
+
const ec = this.db.raw
|
|
272
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
273
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
274
|
+
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
275
|
+
.get(sourceId);
|
|
276
|
+
const e = this.db.raw
|
|
277
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
278
|
+
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
279
|
+
WHERE c.source_id = ?`)
|
|
280
|
+
.get(sourceId);
|
|
202
281
|
return {
|
|
203
282
|
files: f?.total ?? 0,
|
|
204
283
|
chunks: c?.n ?? 0,
|
|
284
|
+
embeddableChunks: ec?.n ?? 0,
|
|
285
|
+
embeddings: e?.n ?? 0,
|
|
205
286
|
errors: f?.errors ?? 0,
|
|
206
287
|
skipped: f?.skipped ?? 0,
|
|
207
288
|
};
|
|
@@ -285,72 +366,178 @@ export class KnowledgeIndexStore {
|
|
|
285
366
|
return mapped;
|
|
286
367
|
}
|
|
287
368
|
/**
|
|
288
|
-
* 写入 chunk 向量(
|
|
369
|
+
* 写入 chunk 向量(Float32 BLOB;sqlite-vec 可用时同步进 vec 表)
|
|
289
370
|
*/
|
|
290
371
|
setChunkEmbedding(chunkId, embedding) {
|
|
291
|
-
this.
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
372
|
+
this.setChunkEmbeddings([[chunkId, embedding]]);
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* 批量写向量(单事务,减少与 parse 写路径的锁竞争)
|
|
376
|
+
*
|
|
377
|
+
* @param rows - [chunkId, embedding] 列表
|
|
378
|
+
*/
|
|
379
|
+
setChunkEmbeddings(rows) {
|
|
380
|
+
if (rows.length === 0)
|
|
381
|
+
return;
|
|
382
|
+
const upsert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, created_at)
|
|
383
|
+
VALUES (?, ?, ?, ?)
|
|
384
|
+
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
385
|
+
dimensions=excluded.dimensions,
|
|
386
|
+
embedding=excluded.embedding,
|
|
387
|
+
created_at=excluded.created_at`);
|
|
388
|
+
this.db.raw.exec('BEGIN');
|
|
389
|
+
try {
|
|
390
|
+
const now = Date.now();
|
|
391
|
+
for (const [chunkId, embedding] of rows) {
|
|
392
|
+
upsert.run(chunkId, embedding.length, toF32Blob(embedding), now);
|
|
393
|
+
this.upsertKnowledgeVec(chunkId, embedding);
|
|
394
|
+
}
|
|
395
|
+
this.db.raw.exec('COMMIT');
|
|
396
|
+
}
|
|
397
|
+
catch (e) {
|
|
398
|
+
this.db.raw.exec('ROLLBACK');
|
|
399
|
+
throw e;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
/** sqlite-vec 表名 */
|
|
403
|
+
static VEC_TABLE = 'knowledge_vec';
|
|
404
|
+
/** 确保 vec0 虚拟表(维度首次写入时确定) */
|
|
405
|
+
ensureVecTable(dimensions) {
|
|
406
|
+
if (!this.db.sqliteVecEnabled)
|
|
407
|
+
return false;
|
|
408
|
+
try {
|
|
409
|
+
const row = this.db.raw
|
|
410
|
+
.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
|
|
411
|
+
.get(KnowledgeIndexStore.VEC_TABLE);
|
|
412
|
+
if (row?.sql) {
|
|
413
|
+
const m = /float\[(\d+)\]/i.exec(row.sql);
|
|
414
|
+
return !m || Number(m[1]) === dimensions;
|
|
415
|
+
}
|
|
416
|
+
this.db.raw.exec(`
|
|
417
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${KnowledgeIndexStore.VEC_TABLE} USING vec0(
|
|
418
|
+
chunk_id TEXT PRIMARY KEY,
|
|
419
|
+
embedding float[${dimensions}] distance_metric=cosine
|
|
420
|
+
);
|
|
421
|
+
`);
|
|
422
|
+
return true;
|
|
423
|
+
}
|
|
424
|
+
catch {
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
upsertKnowledgeVec(chunkId, embedding) {
|
|
429
|
+
if (!this.ensureVecTable(embedding.length))
|
|
430
|
+
return;
|
|
431
|
+
try {
|
|
432
|
+
this.db.raw
|
|
433
|
+
.prepare(`INSERT INTO ${KnowledgeIndexStore.VEC_TABLE} (chunk_id, embedding)
|
|
434
|
+
VALUES (?, ?)
|
|
435
|
+
ON CONFLICT(chunk_id) DO UPDATE SET embedding = excluded.embedding`)
|
|
436
|
+
.run(String(chunkId), toF32Blob(embedding));
|
|
437
|
+
}
|
|
438
|
+
catch {
|
|
439
|
+
// vec 表不可用时忽略,检索走 JS 路径
|
|
440
|
+
}
|
|
299
441
|
}
|
|
300
442
|
/**
|
|
301
443
|
* 列出待嵌入 chunk(无向量行)
|
|
444
|
+
*
|
|
445
|
+
* **代码不进向量**:原文件已可 file_search;片段易失真。
|
|
446
|
+
* 仅文档/正文类(text / markdown / document:* 等)参与 embedding。
|
|
302
447
|
*/
|
|
303
448
|
listChunksMissingEmbedding(sourceId, limit = 100) {
|
|
304
449
|
const rows = this.db.raw
|
|
305
450
|
.prepare(`SELECT c.id, c.text, c.path
|
|
306
451
|
FROM knowledge_chunks c
|
|
307
452
|
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
453
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
308
454
|
WHERE c.source_id = ? AND e.chunk_id IS NULL
|
|
455
|
+
AND ${EMBEDDABLE_WHERE}
|
|
309
456
|
ORDER BY c.path, c.ordinal
|
|
310
457
|
LIMIT ?`)
|
|
311
458
|
.all(sourceId, limit);
|
|
312
459
|
return rows.map((r) => ({ id: asChunkId(r.id), text: r.text, path: r.path }));
|
|
313
460
|
}
|
|
314
461
|
/**
|
|
315
|
-
* 向量覆盖度(0–1
|
|
462
|
+
* 向量覆盖度(0–1;无可嵌入 chunk 时为 1)
|
|
316
463
|
*/
|
|
317
464
|
embeddingCoverage(sourceId) {
|
|
318
465
|
const total = this.db.raw
|
|
319
|
-
.prepare(
|
|
466
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
467
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
468
|
+
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
320
469
|
.get(sourceId);
|
|
321
470
|
if (!total?.n)
|
|
322
471
|
return 1;
|
|
323
472
|
const embedded = this.db.raw
|
|
324
473
|
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
325
474
|
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
326
|
-
|
|
475
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
476
|
+
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
327
477
|
.get(sourceId);
|
|
328
478
|
return Math.min(1, (embedded?.n ?? 0) / total.n);
|
|
329
479
|
}
|
|
330
480
|
/**
|
|
331
|
-
*
|
|
481
|
+
* 向量检索:优先 sqlite-vec KNN;否则 JS cosine(BLOB)
|
|
332
482
|
*/
|
|
333
483
|
vectorSearch(queryEmbedding, opts) {
|
|
334
484
|
if (opts.sourceIds.length === 0 || !queryEmbedding?.length)
|
|
335
485
|
return [];
|
|
336
486
|
const limit = opts.limit ?? 8;
|
|
487
|
+
// vec 表可能尚未回填;0 命中时退 JS,避免空结果短路
|
|
488
|
+
const viaVec = this.vectorSearchVec(queryEmbedding, opts, limit);
|
|
489
|
+
if (viaVec && viaVec.length > 0)
|
|
490
|
+
return viaVec;
|
|
491
|
+
return this.vectorSearchJs(queryEmbedding, opts, limit);
|
|
492
|
+
}
|
|
493
|
+
/** sqlite-vec KNN(不可用时返回 null) */
|
|
494
|
+
vectorSearchVec(queryEmbedding, opts, limit) {
|
|
495
|
+
if (!this.db.sqliteVecEnabled)
|
|
496
|
+
return null;
|
|
497
|
+
if (!this.ensureVecTable(queryEmbedding.length))
|
|
498
|
+
return null;
|
|
499
|
+
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
500
|
+
try {
|
|
501
|
+
const rows = this.db.raw
|
|
502
|
+
.prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, v.distance AS distance
|
|
503
|
+
FROM (
|
|
504
|
+
SELECT chunk_id, distance
|
|
505
|
+
FROM ${KnowledgeIndexStore.VEC_TABLE}
|
|
506
|
+
WHERE embedding MATCH ?
|
|
507
|
+
AND k = ?
|
|
508
|
+
) v
|
|
509
|
+
JOIN knowledge_chunks c ON c.id = v.chunk_id
|
|
510
|
+
WHERE c.source_id IN (${placeholders})
|
|
511
|
+
ORDER BY v.distance ASC`)
|
|
512
|
+
.all(toF32Blob(queryEmbedding), Math.max(limit * 2, 16), ...opts.sourceIds);
|
|
513
|
+
return rows.slice(0, limit).map((row) => ({
|
|
514
|
+
chunkId: asChunkId(row.id),
|
|
515
|
+
sourceId: asSourceId(row.source_id),
|
|
516
|
+
path: row.path,
|
|
517
|
+
ordinal: row.ordinal,
|
|
518
|
+
text: row.text,
|
|
519
|
+
startLine: row.start_line,
|
|
520
|
+
endLine: row.end_line,
|
|
521
|
+
score: 1 - Number(row.distance),
|
|
522
|
+
}));
|
|
523
|
+
}
|
|
524
|
+
catch {
|
|
525
|
+
return null;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
/** JS 余弦(Float32 BLOB;无 JSON.parse) */
|
|
529
|
+
vectorSearchJs(queryEmbedding, opts, limit) {
|
|
337
530
|
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
338
531
|
const rows = this.db.raw
|
|
339
|
-
.prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.
|
|
532
|
+
.prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.embedding
|
|
340
533
|
FROM knowledge_chunks c
|
|
341
534
|
JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
342
535
|
WHERE c.source_id IN (${placeholders})
|
|
343
|
-
LIMIT
|
|
536
|
+
LIMIT 200000`)
|
|
344
537
|
.all(...opts.sourceIds);
|
|
345
538
|
const hits = [];
|
|
346
539
|
for (const row of rows) {
|
|
347
|
-
|
|
348
|
-
try {
|
|
349
|
-
emb = JSON.parse(row.embedding_json);
|
|
350
|
-
}
|
|
351
|
-
catch {
|
|
352
|
-
emb = null;
|
|
353
|
-
}
|
|
540
|
+
const emb = fromF32Blob(row.embedding);
|
|
354
541
|
if (!emb?.length || emb.length !== queryEmbedding.length)
|
|
355
542
|
continue;
|
|
356
543
|
const sim = cosineSimilarity(queryEmbedding, emb);
|
|
@@ -371,6 +558,21 @@ export class KnowledgeIndexStore {
|
|
|
371
558
|
return hits.slice(0, limit);
|
|
372
559
|
}
|
|
373
560
|
}
|
|
561
|
+
/** number[] → float32 blob */
|
|
562
|
+
function toF32Blob(embedding) {
|
|
563
|
+
const f32 = Float32Array.from(embedding);
|
|
564
|
+
return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
|
|
565
|
+
}
|
|
566
|
+
/** float32 blob → number[] */
|
|
567
|
+
function fromF32Blob(buf) {
|
|
568
|
+
if (!buf || buf.byteLength === 0)
|
|
569
|
+
return null;
|
|
570
|
+
if (buf.byteLength % 4 !== 0)
|
|
571
|
+
return null;
|
|
572
|
+
const u8 = buf instanceof Buffer ? new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) : buf;
|
|
573
|
+
const f32 = new Float32Array(u8.buffer, u8.byteOffset, u8.byteLength / 4);
|
|
574
|
+
return Array.from(f32);
|
|
575
|
+
}
|
|
374
576
|
/**
|
|
375
577
|
* 内容指纹(新鲜度判断)
|
|
376
578
|
*
|
|
@@ -31,6 +31,8 @@ export { KnowledgePurger } from './purge.js';
|
|
|
31
31
|
export type { PurgeResult } from './purge.js';
|
|
32
32
|
export { resolveKnowledgePaths } from './paths.js';
|
|
33
33
|
export type { KnowledgePaths } from './paths.js';
|
|
34
|
+
export { deriveTopicsFromPaths } from './topics.js';
|
|
35
|
+
export type { KnowledgeCatalogItem, KnowledgeCatalogProvider } from './catalog-types.js';
|
|
34
36
|
export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
|
|
35
37
|
export type { KnowledgeDescribePort, KnowledgeDescribeOptions, KnowledgeDescribeResult, } from './describe.js';
|
|
36
38
|
export { scanSecretShapes } from './secret-scan.js';
|
|
@@ -19,6 +19,7 @@ export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQu
|
|
|
19
19
|
export { KnowledgeHitLog } from './hit-log.js';
|
|
20
20
|
export { KnowledgePurger } from './purge.js';
|
|
21
21
|
export { resolveKnowledgePaths } from './paths.js';
|
|
22
|
+
export { deriveTopicsFromPaths } from './topics.js';
|
|
22
23
|
export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
|
|
23
24
|
export { scanSecretShapes } from './secret-scan.js';
|
|
24
25
|
export { asSourceId, asChunkId } from './types.js';
|