@octopi-agent/engine 0.55.0 → 0.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/context.d.ts +38 -0
- package/dist/config-schema/context.js +39 -0
- package/dist/config-schema/engine.d.ts +42 -0
- package/dist/config-schema/engine.js +3 -1
- package/dist/config-schema/substrate.d.ts +5 -0
- package/dist/config-schema/substrate.js +11 -0
- package/dist/config.d.ts +44 -0
- package/dist/harness/agent/config-bridge.js +3 -0
- package/dist/harness/context/capabilities/document/backends/docx.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/docx.js +67 -0
- package/dist/harness/context/capabilities/document/backends/office-parser.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/office-parser.js +76 -0
- package/dist/harness/context/capabilities/document/backends/pdf.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/pdf.js +86 -0
- package/dist/harness/context/capabilities/document/backends/plain.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/plain.js +42 -0
- package/dist/harness/context/capabilities/document/backends/sheet.d.ts +7 -0
- package/dist/harness/context/capabilities/document/backends/sheet.js +90 -0
- package/dist/harness/context/capabilities/document/backends/xmind.d.ts +20 -0
- package/dist/harness/context/capabilities/document/backends/xmind.js +198 -0
- package/dist/harness/context/capabilities/document/errors.d.ts +27 -0
- package/dist/harness/context/capabilities/document/errors.js +33 -0
- package/dist/harness/context/capabilities/document/format.d.ts +38 -0
- package/dist/harness/context/capabilities/document/format.js +113 -0
- package/dist/harness/context/capabilities/document/html-to-md.d.ts +14 -0
- package/dist/harness/context/capabilities/document/html-to-md.js +97 -0
- package/dist/harness/context/capabilities/document/index.d.ts +21 -0
- package/dist/harness/context/capabilities/document/index.js +20 -0
- package/dist/harness/context/capabilities/document/legacy/factory.d.ts +13 -0
- package/dist/harness/context/capabilities/document/legacy/factory.js +23 -0
- package/dist/harness/context/capabilities/document/legacy/soffice.d.ts +32 -0
- package/dist/harness/context/capabilities/document/legacy/soffice.js +186 -0
- package/dist/harness/context/capabilities/document/port.d.ts +19 -0
- package/dist/harness/context/capabilities/document/port.js +400 -0
- package/dist/harness/context/capabilities/document/types.d.ts +149 -0
- package/dist/harness/context/capabilities/document/types.js +8 -0
- package/dist/harness/context/capabilities/index.d.ts +1 -0
- package/dist/harness/context/capabilities/index.js +1 -0
- package/dist/harness/context/constitution/default-agents.md +1 -0
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +26 -2
- package/dist/harness/context/layer-types.js +59 -11
- package/dist/harness/context/layers.d.ts +3 -0
- package/dist/harness/context/layers.js +38 -12
- package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +10 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.d.ts +64 -0
- package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +155 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
- package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +2 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +4 -1
- package/dist/harness/index.d.ts +3 -3
- package/dist/harness/index.js +2 -2
- package/dist/harness/knowledge/catalog-types.d.ts +4 -0
- package/dist/harness/knowledge/db.d.ts +4 -0
- package/dist/harness/knowledge/db.js +58 -3
- package/dist/harness/knowledge/describe.d.ts +2 -0
- package/dist/harness/knowledge/describe.js +5 -1
- package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
- package/dist/harness/knowledge/document-extract-worker.js +25 -0
- package/dist/harness/knowledge/extract-document.d.ts +17 -0
- package/dist/harness/knowledge/extract-document.js +62 -0
- package/dist/harness/knowledge/index-store.d.ts +49 -5
- package/dist/harness/knowledge/index-store.js +240 -32
- package/dist/harness/knowledge/index.d.ts +2 -0
- package/dist/harness/knowledge/index.js +1 -0
- package/dist/harness/knowledge/ingest.d.ts +123 -2
- package/dist/harness/knowledge/ingest.js +661 -59
- package/dist/harness/knowledge/retriever.d.ts +8 -0
- package/dist/harness/knowledge/retriever.js +33 -1
- package/dist/harness/knowledge/source-store.d.ts +4 -0
- package/dist/harness/knowledge/source-store.js +62 -6
- package/dist/harness/knowledge/topics.d.ts +13 -0
- package/dist/harness/knowledge/topics.js +48 -0
- package/dist/harness/memory/index.d.ts +3 -2
- package/dist/harness/memory/index.js +3 -1
- package/dist/harness/memory/retrieval-rank.d.ts +51 -0
- package/dist/harness/memory/retrieval-rank.js +60 -0
- package/dist/harness/memory/sqlite/index.d.ts +2 -2
- package/dist/harness/memory/sqlite/index.js +1 -1
- package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
- package/dist/harness/memory/sqlite/keyword-search.js +40 -17
- package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
- package/dist/harness/memory/sqlite/memory-store.js +89 -39
- package/dist/harness/memory/store.js +39 -21
- package/dist/harness/memory/types.d.ts +7 -0
- package/dist/harness/run/runner.d.ts +1 -0
- package/dist/harness/run/runner.js +4 -2
- package/dist/harness/session/attachments/extract.d.ts +3 -2
- package/dist/harness/session/attachments/extract.js +19 -6
- package/dist/harness/session/attachments/service.d.ts +5 -1
- package/dist/harness/session/attachments/service.js +44 -2
- package/dist/harness/session/tasks/index.d.ts +1 -1
- package/dist/harness/session/tasks/index.js +1 -1
- package/dist/harness/session/tasks/render.d.ts +10 -0
- package/dist/harness/session/tasks/render.js +27 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/package.json +8 -3
|
@@ -7,6 +7,8 @@ import { randomUUID, createHash } from 'node:crypto';
|
|
|
7
7
|
import { tokenizeKeywordQuery, scoreKeywordFields } from '../memory/sqlite/keyword-search.js';
|
|
8
8
|
import { cosineSimilarity } from '../memory/sqlite/vector-search.js';
|
|
9
9
|
import { asChunkId, asSourceId } from './types.js';
|
|
10
|
+
/** 可嵌入向量的 chunk 条件:非 code-tree(代码走 file_search / 原文) */
|
|
11
|
+
const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
|
|
10
12
|
export class KnowledgeIndexStore {
|
|
11
13
|
db;
|
|
12
14
|
constructor(db) {
|
|
@@ -168,25 +170,92 @@ export class KnowledgeIndexStore {
|
|
|
168
170
|
.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?')
|
|
169
171
|
.run(sourceId, path);
|
|
170
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* 批量删除 path(同一事务,减少 sync 往返)
|
|
175
|
+
*
|
|
176
|
+
* @param sourceId - 知识源 id
|
|
177
|
+
* @param paths - 精确 path 列表
|
|
178
|
+
*/
|
|
179
|
+
removeFiles(sourceId, paths) {
|
|
180
|
+
if (paths.length === 0)
|
|
181
|
+
return;
|
|
182
|
+
const chunkIds = [];
|
|
183
|
+
const select = this.db.raw.prepare(`SELECT id FROM knowledge_chunks WHERE source_id = ? AND path = ?`);
|
|
184
|
+
for (const p of paths) {
|
|
185
|
+
const rows = select.all(sourceId, p);
|
|
186
|
+
for (const r of rows)
|
|
187
|
+
chunkIds.push(r.id);
|
|
188
|
+
}
|
|
189
|
+
this.db.raw.exec('BEGIN');
|
|
190
|
+
try {
|
|
191
|
+
if (chunkIds.length > 0) {
|
|
192
|
+
const delEmb = this.db.raw.prepare('DELETE FROM knowledge_chunk_embeddings WHERE chunk_id = ?');
|
|
193
|
+
for (const id of chunkIds)
|
|
194
|
+
delEmb.run(id);
|
|
195
|
+
}
|
|
196
|
+
const delChunks = this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?');
|
|
197
|
+
const delFiles = this.db.raw.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?');
|
|
198
|
+
for (const p of paths) {
|
|
199
|
+
delChunks.run(sourceId, p);
|
|
200
|
+
delFiles.run(sourceId, p);
|
|
201
|
+
}
|
|
202
|
+
this.db.raw.exec('COMMIT');
|
|
203
|
+
}
|
|
204
|
+
catch (e) {
|
|
205
|
+
this.db.raw.exec('ROLLBACK');
|
|
206
|
+
throw e;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* 删除路径及子树(目录被移出 / 删除)
|
|
211
|
+
*
|
|
212
|
+
* @param sourceId - 知识源 id
|
|
213
|
+
* @param path - 文件或目录绝对路径
|
|
214
|
+
*/
|
|
215
|
+
removePathTree(sourceId, path) {
|
|
216
|
+
this.db.raw.exec('BEGIN');
|
|
217
|
+
try {
|
|
218
|
+
this.removeFile(sourceId, path);
|
|
219
|
+
const likeSlash = `${path}/%`;
|
|
220
|
+
const likeBack = `${path}\\%`;
|
|
221
|
+
this.db.raw
|
|
222
|
+
.prepare(`DELETE FROM knowledge_chunk_embeddings
|
|
223
|
+
WHERE chunk_id IN (
|
|
224
|
+
SELECT id FROM knowledge_chunks
|
|
225
|
+
WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)
|
|
226
|
+
)`)
|
|
227
|
+
.run(sourceId, likeSlash, likeBack);
|
|
228
|
+
this.db.raw
|
|
229
|
+
.prepare(`DELETE FROM knowledge_chunks WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
|
|
230
|
+
.run(sourceId, likeSlash, likeBack);
|
|
231
|
+
this.db.raw
|
|
232
|
+
.prepare(`DELETE FROM knowledge_files WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
|
|
233
|
+
.run(sourceId, likeSlash, likeBack);
|
|
234
|
+
this.db.raw.exec('COMMIT');
|
|
235
|
+
}
|
|
236
|
+
catch (e) {
|
|
237
|
+
this.db.raw.exec('ROLLBACK');
|
|
238
|
+
throw e;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
171
241
|
/**
|
|
172
242
|
* 差量 prune:删除本轮 discover **未出现** 的 path(含 chunks)。
|
|
173
243
|
* 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
|
|
174
244
|
*
|
|
245
|
+
* @param keepPaths - 应保留的 path 集
|
|
246
|
+
* @param opts.allowEmptyKeep - walk 成功且目录为空时允许清空(默认 false)
|
|
175
247
|
* @returns 删除的文件数
|
|
176
248
|
*/
|
|
177
|
-
pruneMissing(sourceId, keepPaths) {
|
|
249
|
+
pruneMissing(sourceId, keepPaths, opts) {
|
|
178
250
|
const keep = keepPaths instanceof Set ? keepPaths : new Set(keepPaths);
|
|
179
|
-
if (keep.size === 0)
|
|
251
|
+
if (keep.size === 0 && !opts?.allowEmptyKeep)
|
|
180
252
|
return 0;
|
|
181
253
|
const files = this.listFiles(sourceId);
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
return removed;
|
|
254
|
+
const gone = files.filter((f) => !keep.has(f.path)).map((f) => f.path);
|
|
255
|
+
if (gone.length === 0)
|
|
256
|
+
return 0;
|
|
257
|
+
this.removeFiles(sourceId, gone);
|
|
258
|
+
return gone.length;
|
|
190
259
|
}
|
|
191
260
|
sourceStats(sourceId) {
|
|
192
261
|
const f = this.db.raw
|
|
@@ -199,9 +268,21 @@ export class KnowledgeIndexStore {
|
|
|
199
268
|
const c = this.db.raw
|
|
200
269
|
.prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
|
|
201
270
|
.get(sourceId);
|
|
271
|
+
const ec = this.db.raw
|
|
272
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
273
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
274
|
+
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
275
|
+
.get(sourceId);
|
|
276
|
+
const e = this.db.raw
|
|
277
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
278
|
+
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
279
|
+
WHERE c.source_id = ?`)
|
|
280
|
+
.get(sourceId);
|
|
202
281
|
return {
|
|
203
282
|
files: f?.total ?? 0,
|
|
204
283
|
chunks: c?.n ?? 0,
|
|
284
|
+
embeddableChunks: ec?.n ?? 0,
|
|
285
|
+
embeddings: e?.n ?? 0,
|
|
205
286
|
errors: f?.errors ?? 0,
|
|
206
287
|
skipped: f?.skipped ?? 0,
|
|
207
288
|
};
|
|
@@ -285,72 +366,178 @@ export class KnowledgeIndexStore {
|
|
|
285
366
|
return mapped;
|
|
286
367
|
}
|
|
287
368
|
/**
|
|
288
|
-
* 写入 chunk 向量(
|
|
369
|
+
* 写入 chunk 向量(Float32 BLOB;sqlite-vec 可用时同步进 vec 表)
|
|
289
370
|
*/
|
|
290
371
|
setChunkEmbedding(chunkId, embedding) {
|
|
291
|
-
this.
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
372
|
+
this.setChunkEmbeddings([[chunkId, embedding]]);
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* 批量写向量(单事务,减少与 parse 写路径的锁竞争)
|
|
376
|
+
*
|
|
377
|
+
* @param rows - [chunkId, embedding] 列表
|
|
378
|
+
*/
|
|
379
|
+
setChunkEmbeddings(rows) {
|
|
380
|
+
if (rows.length === 0)
|
|
381
|
+
return;
|
|
382
|
+
const upsert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, created_at)
|
|
383
|
+
VALUES (?, ?, ?, ?)
|
|
384
|
+
ON CONFLICT(chunk_id) DO UPDATE SET
|
|
385
|
+
dimensions=excluded.dimensions,
|
|
386
|
+
embedding=excluded.embedding,
|
|
387
|
+
created_at=excluded.created_at`);
|
|
388
|
+
this.db.raw.exec('BEGIN');
|
|
389
|
+
try {
|
|
390
|
+
const now = Date.now();
|
|
391
|
+
for (const [chunkId, embedding] of rows) {
|
|
392
|
+
upsert.run(chunkId, embedding.length, toF32Blob(embedding), now);
|
|
393
|
+
this.upsertKnowledgeVec(chunkId, embedding);
|
|
394
|
+
}
|
|
395
|
+
this.db.raw.exec('COMMIT');
|
|
396
|
+
}
|
|
397
|
+
catch (e) {
|
|
398
|
+
this.db.raw.exec('ROLLBACK');
|
|
399
|
+
throw e;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
/** sqlite-vec 表名 */
|
|
403
|
+
static VEC_TABLE = 'knowledge_vec';
|
|
404
|
+
/** 确保 vec0 虚拟表(维度首次写入时确定) */
|
|
405
|
+
ensureVecTable(dimensions) {
|
|
406
|
+
if (!this.db.sqliteVecEnabled)
|
|
407
|
+
return false;
|
|
408
|
+
try {
|
|
409
|
+
const row = this.db.raw
|
|
410
|
+
.prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
|
|
411
|
+
.get(KnowledgeIndexStore.VEC_TABLE);
|
|
412
|
+
if (row?.sql) {
|
|
413
|
+
const m = /float\[(\d+)\]/i.exec(row.sql);
|
|
414
|
+
return !m || Number(m[1]) === dimensions;
|
|
415
|
+
}
|
|
416
|
+
this.db.raw.exec(`
|
|
417
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${KnowledgeIndexStore.VEC_TABLE} USING vec0(
|
|
418
|
+
chunk_id TEXT PRIMARY KEY,
|
|
419
|
+
embedding float[${dimensions}] distance_metric=cosine
|
|
420
|
+
);
|
|
421
|
+
`);
|
|
422
|
+
return true;
|
|
423
|
+
}
|
|
424
|
+
catch {
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
upsertKnowledgeVec(chunkId, embedding) {
|
|
429
|
+
if (!this.ensureVecTable(embedding.length))
|
|
430
|
+
return;
|
|
431
|
+
try {
|
|
432
|
+
this.db.raw
|
|
433
|
+
.prepare(`INSERT INTO ${KnowledgeIndexStore.VEC_TABLE} (chunk_id, embedding)
|
|
434
|
+
VALUES (?, ?)
|
|
435
|
+
ON CONFLICT(chunk_id) DO UPDATE SET embedding = excluded.embedding`)
|
|
436
|
+
.run(String(chunkId), toF32Blob(embedding));
|
|
437
|
+
}
|
|
438
|
+
catch {
|
|
439
|
+
// vec 表不可用时忽略,检索走 JS 路径
|
|
440
|
+
}
|
|
299
441
|
}
|
|
300
442
|
/**
|
|
301
443
|
* 列出待嵌入 chunk(无向量行)
|
|
444
|
+
*
|
|
445
|
+
* **代码不进向量**:原文件已可 file_search;片段易失真。
|
|
446
|
+
* 仅文档/正文类(text / markdown / document:* 等)参与 embedding。
|
|
302
447
|
*/
|
|
303
448
|
listChunksMissingEmbedding(sourceId, limit = 100) {
|
|
304
449
|
const rows = this.db.raw
|
|
305
450
|
.prepare(`SELECT c.id, c.text, c.path
|
|
306
451
|
FROM knowledge_chunks c
|
|
307
452
|
LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
453
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
308
454
|
WHERE c.source_id = ? AND e.chunk_id IS NULL
|
|
455
|
+
AND ${EMBEDDABLE_WHERE}
|
|
309
456
|
ORDER BY c.path, c.ordinal
|
|
310
457
|
LIMIT ?`)
|
|
311
458
|
.all(sourceId, limit);
|
|
312
459
|
return rows.map((r) => ({ id: asChunkId(r.id), text: r.text, path: r.path }));
|
|
313
460
|
}
|
|
314
461
|
/**
|
|
315
|
-
* 向量覆盖度(0–1
|
|
462
|
+
* 向量覆盖度(0–1;无可嵌入 chunk 时为 1)
|
|
316
463
|
*/
|
|
317
464
|
embeddingCoverage(sourceId) {
|
|
318
465
|
const total = this.db.raw
|
|
319
|
-
.prepare(
|
|
466
|
+
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
|
|
467
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
468
|
+
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
320
469
|
.get(sourceId);
|
|
321
470
|
if (!total?.n)
|
|
322
471
|
return 1;
|
|
323
472
|
const embedded = this.db.raw
|
|
324
473
|
.prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
|
|
325
474
|
JOIN knowledge_chunks c ON c.id = e.chunk_id
|
|
326
|
-
|
|
475
|
+
LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
|
|
476
|
+
WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
|
|
327
477
|
.get(sourceId);
|
|
328
478
|
return Math.min(1, (embedded?.n ?? 0) / total.n);
|
|
329
479
|
}
|
|
330
480
|
/**
|
|
331
|
-
*
|
|
481
|
+
* 向量检索:优先 sqlite-vec KNN;否则 JS cosine(BLOB)
|
|
332
482
|
*/
|
|
333
483
|
vectorSearch(queryEmbedding, opts) {
|
|
334
484
|
if (opts.sourceIds.length === 0 || !queryEmbedding?.length)
|
|
335
485
|
return [];
|
|
336
486
|
const limit = opts.limit ?? 8;
|
|
487
|
+
// vec 表可能尚未回填;0 命中时退 JS,避免空结果短路
|
|
488
|
+
const viaVec = this.vectorSearchVec(queryEmbedding, opts, limit);
|
|
489
|
+
if (viaVec && viaVec.length > 0)
|
|
490
|
+
return viaVec;
|
|
491
|
+
return this.vectorSearchJs(queryEmbedding, opts, limit);
|
|
492
|
+
}
|
|
493
|
+
/** sqlite-vec KNN(不可用时返回 null) */
|
|
494
|
+
vectorSearchVec(queryEmbedding, opts, limit) {
|
|
495
|
+
if (!this.db.sqliteVecEnabled)
|
|
496
|
+
return null;
|
|
497
|
+
if (!this.ensureVecTable(queryEmbedding.length))
|
|
498
|
+
return null;
|
|
499
|
+
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
500
|
+
try {
|
|
501
|
+
const rows = this.db.raw
|
|
502
|
+
.prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, v.distance AS distance
|
|
503
|
+
FROM (
|
|
504
|
+
SELECT chunk_id, distance
|
|
505
|
+
FROM ${KnowledgeIndexStore.VEC_TABLE}
|
|
506
|
+
WHERE embedding MATCH ?
|
|
507
|
+
AND k = ?
|
|
508
|
+
) v
|
|
509
|
+
JOIN knowledge_chunks c ON c.id = v.chunk_id
|
|
510
|
+
WHERE c.source_id IN (${placeholders})
|
|
511
|
+
ORDER BY v.distance ASC`)
|
|
512
|
+
.all(toF32Blob(queryEmbedding), Math.max(limit * 2, 16), ...opts.sourceIds);
|
|
513
|
+
return rows.slice(0, limit).map((row) => ({
|
|
514
|
+
chunkId: asChunkId(row.id),
|
|
515
|
+
sourceId: asSourceId(row.source_id),
|
|
516
|
+
path: row.path,
|
|
517
|
+
ordinal: row.ordinal,
|
|
518
|
+
text: row.text,
|
|
519
|
+
startLine: row.start_line,
|
|
520
|
+
endLine: row.end_line,
|
|
521
|
+
score: 1 - Number(row.distance),
|
|
522
|
+
}));
|
|
523
|
+
}
|
|
524
|
+
catch {
|
|
525
|
+
return null;
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
/** JS 余弦(Float32 BLOB;无 JSON.parse) */
|
|
529
|
+
vectorSearchJs(queryEmbedding, opts, limit) {
|
|
337
530
|
const placeholders = opts.sourceIds.map(() => '?').join(',');
|
|
338
531
|
const rows = this.db.raw
|
|
339
|
-
.prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.
|
|
532
|
+
.prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.embedding
|
|
340
533
|
FROM knowledge_chunks c
|
|
341
534
|
JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
|
|
342
535
|
WHERE c.source_id IN (${placeholders})
|
|
343
|
-
LIMIT
|
|
536
|
+
LIMIT 200000`)
|
|
344
537
|
.all(...opts.sourceIds);
|
|
345
538
|
const hits = [];
|
|
346
539
|
for (const row of rows) {
|
|
347
|
-
|
|
348
|
-
try {
|
|
349
|
-
emb = JSON.parse(row.embedding_json);
|
|
350
|
-
}
|
|
351
|
-
catch {
|
|
352
|
-
emb = null;
|
|
353
|
-
}
|
|
540
|
+
const emb = fromF32Blob(row.embedding);
|
|
354
541
|
if (!emb?.length || emb.length !== queryEmbedding.length)
|
|
355
542
|
continue;
|
|
356
543
|
const sim = cosineSimilarity(queryEmbedding, emb);
|
|
@@ -371,6 +558,27 @@ export class KnowledgeIndexStore {
|
|
|
371
558
|
return hits.slice(0, limit);
|
|
372
559
|
}
|
|
373
560
|
}
|
|
561
|
+
/** number[] → float32 blob */
|
|
562
|
+
function toF32Blob(embedding) {
|
|
563
|
+
const f32 = Float32Array.from(embedding);
|
|
564
|
+
return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
|
|
565
|
+
}
|
|
566
|
+
/** float32 blob → number[] */
|
|
567
|
+
function fromF32Blob(buf) {
|
|
568
|
+
if (!buf || buf.byteLength === 0)
|
|
569
|
+
return null;
|
|
570
|
+
if (buf.byteLength % 4 !== 0)
|
|
571
|
+
return null;
|
|
572
|
+
const u8 = buf instanceof Buffer ? new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) : buf;
|
|
573
|
+
const f32 = new Float32Array(u8.buffer, u8.byteOffset, u8.byteLength / 4);
|
|
574
|
+
return Array.from(f32);
|
|
575
|
+
}
|
|
576
|
+
/**
|
|
577
|
+
* 内容指纹(新鲜度判断)
|
|
578
|
+
*
|
|
579
|
+
* @param content - UTF-8 文本或原始字节(二进制文档用 bytes)
|
|
580
|
+
* @returns sha256 hex
|
|
581
|
+
*/
|
|
374
582
|
export function hashContent(content) {
|
|
375
583
|
return createHash('sha256').update(content).digest('hex');
|
|
376
584
|
}
|
|
@@ -31,6 +31,8 @@ export { KnowledgePurger } from './purge.js';
|
|
|
31
31
|
export type { PurgeResult } from './purge.js';
|
|
32
32
|
export { resolveKnowledgePaths } from './paths.js';
|
|
33
33
|
export type { KnowledgePaths } from './paths.js';
|
|
34
|
+
export { deriveTopicsFromPaths } from './topics.js';
|
|
35
|
+
export type { KnowledgeCatalogItem, KnowledgeCatalogProvider } from './catalog-types.js';
|
|
34
36
|
export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
|
|
35
37
|
export type { KnowledgeDescribePort, KnowledgeDescribeOptions, KnowledgeDescribeResult, } from './describe.js';
|
|
36
38
|
export { scanSecretShapes } from './secret-scan.js';
|
|
@@ -19,6 +19,7 @@ export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQu
|
|
|
19
19
|
export { KnowledgeHitLog } from './hit-log.js';
|
|
20
20
|
export { KnowledgePurger } from './purge.js';
|
|
21
21
|
export { resolveKnowledgePaths } from './paths.js';
|
|
22
|
+
export { deriveTopicsFromPaths } from './topics.js';
|
|
22
23
|
export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
|
|
23
24
|
export { scanSecretShapes } from './secret-scan.js';
|
|
24
25
|
export { asSourceId, asChunkId } from './types.js';
|
|
@@ -11,6 +11,7 @@ import { type SourceFetcher, type VirtualDocument } from './fetchers.js';
|
|
|
11
11
|
import { ConnectorRegistry } from './connectors.js';
|
|
12
12
|
import type { CredentialStore } from '../governance/credentials/store.js';
|
|
13
13
|
import type { EmbeddingProvider } from '../memory/sqlite/embedding.js';
|
|
14
|
+
import type { DocumentPort } from '../context/capabilities/document/types.js';
|
|
14
15
|
export type IngestJobKind = 'parse_file' | 'walk_source' | 'drop_file' | 'embed_source' | 'fetch_doc';
|
|
15
16
|
export interface IngestProgressEvent {
|
|
16
17
|
type: 'job' | 'source' | 'error' | 'embed';
|
|
@@ -31,6 +32,8 @@ export interface KnowledgeIngestOptions {
|
|
|
31
32
|
debounceMs?: number;
|
|
32
33
|
/** 单文件大小上限(默认 5MB;超出 skip) */
|
|
33
34
|
maxFileBytes?: number;
|
|
35
|
+
/** 单文件 parse 超时 ms(默认 45000) */
|
|
36
|
+
parseTimeoutMs?: number;
|
|
34
37
|
/** Phase B embedding(未配则纯关键词) */
|
|
35
38
|
embeddingProvider?: EmbeddingProvider | null;
|
|
36
39
|
/** embed 批大小(默认 32) */
|
|
@@ -53,6 +56,11 @@ export interface KnowledgeIngestOptions {
|
|
|
53
56
|
pollMinIntervalMs?: number;
|
|
54
57
|
/** 每轮 poll 最多处理源数(成本上限,默认 8) */
|
|
55
58
|
maxPollPerTick?: number;
|
|
59
|
+
/**
|
|
60
|
+
* DocumentPort:启用后 PDF/Office 等走抽取 → Markdown 切块
|
|
61
|
+
* (缺省 null = 维持 BINARY skip 语义)
|
|
62
|
+
*/
|
|
63
|
+
documentPort?: DocumentPort | null;
|
|
56
64
|
}
|
|
57
65
|
export declare class KnowledgeIngest extends EventEmitter {
|
|
58
66
|
private readonly sources;
|
|
@@ -62,6 +70,8 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
62
70
|
private readonly maxQueueDepth;
|
|
63
71
|
private readonly debounceMs;
|
|
64
72
|
private readonly maxFileBytes;
|
|
73
|
+
/** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
|
|
74
|
+
private readonly parseTimeoutMs;
|
|
65
75
|
private readonly embedding;
|
|
66
76
|
private readonly embedBatch;
|
|
67
77
|
private readonly embedMinIntervalMs;
|
|
@@ -74,11 +84,20 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
74
84
|
private readonly pollTickMs;
|
|
75
85
|
private readonly pollMinIntervalMs;
|
|
76
86
|
private readonly maxPollPerTick;
|
|
87
|
+
private readonly documentPort;
|
|
77
88
|
private lastEmbedAt;
|
|
78
89
|
private pollTimer;
|
|
79
|
-
private
|
|
90
|
+
private reconcileTimer;
|
|
91
|
+
/** 源级中止控制器(手动 abort / 重建前清场) */
|
|
92
|
+
private abortBySource;
|
|
93
|
+
/** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
|
|
94
|
+
onSourceSettled?: (sourceId: string) => void;
|
|
95
|
+
/** 上次 parse 缺口扫描时间 */
|
|
96
|
+
private lastParseGapScanAt;
|
|
97
|
+
private runningParse;
|
|
80
98
|
private runningEmbed;
|
|
81
99
|
private draining;
|
|
100
|
+
private kickAgain;
|
|
82
101
|
/** source 级锁:同一 source 不并行 walk/parse 冲突写 */
|
|
83
102
|
private sourceLocks;
|
|
84
103
|
/** walk 发现的文件数(Phase A coverage 分母) */
|
|
@@ -87,7 +106,19 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
87
106
|
private debounceTimers;
|
|
88
107
|
private disposed;
|
|
89
108
|
constructor(options: KnowledgeIngestOptions);
|
|
109
|
+
/**
|
|
110
|
+
* 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
|
|
111
|
+
* 仅构造时调用——运行中同进程的 running 是合法的。
|
|
112
|
+
*/
|
|
113
|
+
private reclaimOrphanRunningJobs;
|
|
90
114
|
get indexStore(): KnowledgeIndexStore;
|
|
115
|
+
/**
|
|
116
|
+
* 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
|
|
117
|
+
*
|
|
118
|
+
* @param p - 文件路径
|
|
119
|
+
* @returns 是否跳过
|
|
120
|
+
*/
|
|
121
|
+
private shouldSkipFile;
|
|
91
122
|
/**
|
|
92
123
|
* 全量/增量索引某源(Phase A)
|
|
93
124
|
*/
|
|
@@ -129,6 +160,75 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
129
160
|
*/
|
|
130
161
|
idle(timeoutMs?: number): Promise<void>;
|
|
131
162
|
dispose(): void;
|
|
163
|
+
/**
|
|
164
|
+
* 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
|
|
165
|
+
*
|
|
166
|
+
* @param opts.sourceId - 仅中止该源;省略则全部源
|
|
167
|
+
* @returns 取消统计
|
|
168
|
+
*/
|
|
169
|
+
abortJobs(opts?: {
|
|
170
|
+
sourceId?: string;
|
|
171
|
+
}): {
|
|
172
|
+
cancelledQueued: number;
|
|
173
|
+
abortedRunning: number;
|
|
174
|
+
};
|
|
175
|
+
/**
|
|
176
|
+
* 恢复/继续索引(相对「重建」更轻)
|
|
177
|
+
*
|
|
178
|
+
* - 清除该源中止态
|
|
179
|
+
* - 把 cancelled 任务放回 queued
|
|
180
|
+
* - 若仍缺向量则排队 embed
|
|
181
|
+
* - 不重新 walk 全量;需要全量对齐请 reindex
|
|
182
|
+
*
|
|
183
|
+
* @param opts.sourceId - 仅恢复该源;省略则全部源
|
|
184
|
+
* @returns 恢复统计
|
|
185
|
+
*/
|
|
186
|
+
resumeJobs(opts?: {
|
|
187
|
+
sourceId?: string;
|
|
188
|
+
}): {
|
|
189
|
+
restoredCancelled: number;
|
|
190
|
+
embedQueued: number;
|
|
191
|
+
};
|
|
192
|
+
/** 开启新的中止纪元(abort 后新任务不受旧信号影响) */
|
|
193
|
+
private beginAbortEpoch;
|
|
194
|
+
/**
|
|
195
|
+
* 重建前作废本源未完成任务(supersede)
|
|
196
|
+
*
|
|
197
|
+
* - queued → cancelled(superseded_by_reindex)
|
|
198
|
+
* - running 打断(worker/循环退出)
|
|
199
|
+
* - 随后 beginAbortEpoch,本轮重建可继续写
|
|
200
|
+
*/
|
|
201
|
+
private supersedeSourceWork;
|
|
202
|
+
private ensureParseCoverage;
|
|
203
|
+
/**
|
|
204
|
+
* 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
|
|
205
|
+
*
|
|
206
|
+
* 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
|
|
207
|
+
* embed 前调用,避免给已移出文件的残 chunks 上向量。
|
|
208
|
+
*
|
|
209
|
+
* @param sourceId - 知识源 id
|
|
210
|
+
* @returns 删除的文件数
|
|
211
|
+
*/
|
|
212
|
+
pruneMissingOnDisk(sourceId: string): Promise<number>;
|
|
213
|
+
/** 逐文件 stat(file 源 / walk 失败回退);并行限流 */
|
|
214
|
+
private pruneByStatScan;
|
|
215
|
+
private abortSignalFor;
|
|
216
|
+
private isAborted;
|
|
217
|
+
/**
|
|
218
|
+
* 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
|
|
219
|
+
*
|
|
220
|
+
* 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
|
|
221
|
+
* 1. 回收超时 running(崩溃/挂死孤儿)
|
|
222
|
+
* 2. 有缺向量且无 active embed → 自动排队
|
|
223
|
+
* 3. 无队列、无缺口 → 刷 coverage/status
|
|
224
|
+
*/
|
|
225
|
+
startReconciler(intervalMs?: number): void;
|
|
226
|
+
stopReconciler(): void;
|
|
227
|
+
/** 对账入口(也可手工调用) */
|
|
228
|
+
reconcileJobs(): Promise<void>;
|
|
229
|
+
/** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
|
|
230
|
+
private reclaimStaleRunningJobs;
|
|
231
|
+
private countActiveJobs;
|
|
132
232
|
/**
|
|
133
233
|
* 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
|
|
134
234
|
*/
|
|
@@ -144,17 +244,38 @@ export declare class KnowledgeIngest extends EventEmitter {
|
|
|
144
244
|
private enqueue;
|
|
145
245
|
private kick;
|
|
146
246
|
private drain;
|
|
247
|
+
private countQueuedKinds;
|
|
147
248
|
private claimJob;
|
|
148
249
|
private runJob;
|
|
250
|
+
private withTimeout;
|
|
149
251
|
/**
|
|
150
252
|
* 外源单文档 job:按逻辑键 fetch + 索引
|
|
151
253
|
*/
|
|
152
254
|
private fetchDocJob;
|
|
153
255
|
private parseOne;
|
|
154
256
|
/**
|
|
155
|
-
*
|
|
257
|
+
* PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
|
|
258
|
+
* 新鲜度以原件字节 hash 为准。
|
|
259
|
+
*
|
|
260
|
+
* @param sourceId - 知识源 id
|
|
261
|
+
* @param filePath - 本地文件路径
|
|
262
|
+
* @returns 是否成功入索引
|
|
263
|
+
*/
|
|
264
|
+
private parseDocumentFile;
|
|
265
|
+
/** 优先 worker 抽取;worker 不可用时退回进程内 port */
|
|
266
|
+
private extractDocument;
|
|
267
|
+
/**
|
|
268
|
+
* Phase B:嵌入待处理 chunk
|
|
269
|
+
*
|
|
270
|
+
* 单 job 内按 embedConcurrency **并行**多批调用 API;
|
|
271
|
+
* 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
|
|
156
272
|
*/
|
|
157
273
|
private embedPending;
|
|
274
|
+
/** 单片 embedding;返回成功写入条数 */
|
|
275
|
+
private embedOneSlice;
|
|
276
|
+
/** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
|
|
277
|
+
private ensureEmbedJob;
|
|
278
|
+
private heartbeatJob;
|
|
158
279
|
private refreshCoverage;
|
|
159
280
|
private emitProgress;
|
|
160
281
|
/** 磁盘水位告警(仅告警,不丢任务) */
|