@octopi-agent/engine 0.55.0 → 0.58.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/config-schema/context.d.ts +38 -0
  2. package/dist/config-schema/context.js +39 -0
  3. package/dist/config-schema/engine.d.ts +42 -0
  4. package/dist/config-schema/engine.js +3 -1
  5. package/dist/config-schema/substrate.d.ts +5 -0
  6. package/dist/config-schema/substrate.js +11 -0
  7. package/dist/config.d.ts +44 -0
  8. package/dist/harness/agent/config-bridge.js +3 -0
  9. package/dist/harness/context/capabilities/document/backends/docx.d.ts +7 -0
  10. package/dist/harness/context/capabilities/document/backends/docx.js +67 -0
  11. package/dist/harness/context/capabilities/document/backends/office-parser.d.ts +7 -0
  12. package/dist/harness/context/capabilities/document/backends/office-parser.js +76 -0
  13. package/dist/harness/context/capabilities/document/backends/pdf.d.ts +7 -0
  14. package/dist/harness/context/capabilities/document/backends/pdf.js +86 -0
  15. package/dist/harness/context/capabilities/document/backends/plain.d.ts +7 -0
  16. package/dist/harness/context/capabilities/document/backends/plain.js +42 -0
  17. package/dist/harness/context/capabilities/document/backends/sheet.d.ts +7 -0
  18. package/dist/harness/context/capabilities/document/backends/sheet.js +90 -0
  19. package/dist/harness/context/capabilities/document/backends/xmind.d.ts +20 -0
  20. package/dist/harness/context/capabilities/document/backends/xmind.js +198 -0
  21. package/dist/harness/context/capabilities/document/errors.d.ts +27 -0
  22. package/dist/harness/context/capabilities/document/errors.js +33 -0
  23. package/dist/harness/context/capabilities/document/format.d.ts +38 -0
  24. package/dist/harness/context/capabilities/document/format.js +113 -0
  25. package/dist/harness/context/capabilities/document/html-to-md.d.ts +14 -0
  26. package/dist/harness/context/capabilities/document/html-to-md.js +97 -0
  27. package/dist/harness/context/capabilities/document/index.d.ts +21 -0
  28. package/dist/harness/context/capabilities/document/index.js +20 -0
  29. package/dist/harness/context/capabilities/document/legacy/factory.d.ts +13 -0
  30. package/dist/harness/context/capabilities/document/legacy/factory.js +23 -0
  31. package/dist/harness/context/capabilities/document/legacy/soffice.d.ts +32 -0
  32. package/dist/harness/context/capabilities/document/legacy/soffice.js +186 -0
  33. package/dist/harness/context/capabilities/document/port.d.ts +19 -0
  34. package/dist/harness/context/capabilities/document/port.js +400 -0
  35. package/dist/harness/context/capabilities/document/types.d.ts +149 -0
  36. package/dist/harness/context/capabilities/document/types.js +8 -0
  37. package/dist/harness/context/capabilities/index.d.ts +1 -0
  38. package/dist/harness/context/capabilities/index.js +1 -0
  39. package/dist/harness/context/constitution/default-agents.md +1 -0
  40. package/dist/harness/context/index.d.ts +2 -2
  41. package/dist/harness/context/index.js +1 -1
  42. package/dist/harness/context/layer-types.d.ts +26 -2
  43. package/dist/harness/context/layer-types.js +59 -11
  44. package/dist/harness/context/layers.d.ts +3 -0
  45. package/dist/harness/context/layers.js +38 -12
  46. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  47. package/dist/harness/context/system-prompt-assembler.js +2 -0
  48. package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +1 -0
  49. package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +10 -0
  50. package/dist/harness/extension/plugin-ecosystem/tools/document-tools.d.ts +64 -0
  51. package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +155 -0
  52. package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +1 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/index.js +1 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +2 -0
  57. package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +4 -1
  58. package/dist/harness/index.d.ts +3 -3
  59. package/dist/harness/index.js +2 -2
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/db.d.ts +4 -0
  62. package/dist/harness/knowledge/db.js +58 -3
  63. package/dist/harness/knowledge/describe.d.ts +2 -0
  64. package/dist/harness/knowledge/describe.js +5 -1
  65. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  66. package/dist/harness/knowledge/document-extract-worker.js +25 -0
  67. package/dist/harness/knowledge/extract-document.d.ts +17 -0
  68. package/dist/harness/knowledge/extract-document.js +62 -0
  69. package/dist/harness/knowledge/index-store.d.ts +49 -5
  70. package/dist/harness/knowledge/index-store.js +240 -32
  71. package/dist/harness/knowledge/index.d.ts +2 -0
  72. package/dist/harness/knowledge/index.js +1 -0
  73. package/dist/harness/knowledge/ingest.d.ts +123 -2
  74. package/dist/harness/knowledge/ingest.js +661 -59
  75. package/dist/harness/knowledge/retriever.d.ts +8 -0
  76. package/dist/harness/knowledge/retriever.js +33 -1
  77. package/dist/harness/knowledge/source-store.d.ts +4 -0
  78. package/dist/harness/knowledge/source-store.js +62 -6
  79. package/dist/harness/knowledge/topics.d.ts +13 -0
  80. package/dist/harness/knowledge/topics.js +48 -0
  81. package/dist/harness/memory/index.d.ts +3 -2
  82. package/dist/harness/memory/index.js +3 -1
  83. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  84. package/dist/harness/memory/retrieval-rank.js +60 -0
  85. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  86. package/dist/harness/memory/sqlite/index.js +1 -1
  87. package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
  88. package/dist/harness/memory/sqlite/keyword-search.js +40 -17
  89. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  90. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  91. package/dist/harness/memory/store.js +39 -21
  92. package/dist/harness/memory/types.d.ts +7 -0
  93. package/dist/harness/run/runner.d.ts +1 -0
  94. package/dist/harness/run/runner.js +4 -2
  95. package/dist/harness/session/attachments/extract.d.ts +3 -2
  96. package/dist/harness/session/attachments/extract.js +19 -6
  97. package/dist/harness/session/attachments/service.d.ts +5 -1
  98. package/dist/harness/session/attachments/service.js +44 -2
  99. package/dist/harness/session/tasks/index.d.ts +1 -1
  100. package/dist/harness/session/tasks/index.js +1 -1
  101. package/dist/harness/session/tasks/render.d.ts +10 -0
  102. package/dist/harness/session/tasks/render.js +27 -0
  103. package/dist/index.d.ts +2 -2
  104. package/dist/index.js +1 -1
  105. package/package.json +8 -3
@@ -7,6 +7,8 @@ import { randomUUID, createHash } from 'node:crypto';
7
7
  import { tokenizeKeywordQuery, scoreKeywordFields } from '../memory/sqlite/keyword-search.js';
8
8
  import { cosineSimilarity } from '../memory/sqlite/vector-search.js';
9
9
  import { asChunkId, asSourceId } from './types.js';
10
+ /** 可嵌入向量的 chunk 条件:非 code-tree(代码走 file_search / 原文) */
11
+ const EMBEDDABLE_WHERE = "(f.adapter_id IS NULL OR f.adapter_id NOT IN ('code-tree'))";
10
12
  export class KnowledgeIndexStore {
11
13
  db;
12
14
  constructor(db) {
@@ -168,25 +170,92 @@ export class KnowledgeIndexStore {
168
170
  .prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?')
169
171
  .run(sourceId, path);
170
172
  }
173
+ /**
174
+ * 批量删除 path(同一事务,减少 sync 往返)
175
+ *
176
+ * @param sourceId - 知识源 id
177
+ * @param paths - 精确 path 列表
178
+ */
179
+ removeFiles(sourceId, paths) {
180
+ if (paths.length === 0)
181
+ return;
182
+ const chunkIds = [];
183
+ const select = this.db.raw.prepare(`SELECT id FROM knowledge_chunks WHERE source_id = ? AND path = ?`);
184
+ for (const p of paths) {
185
+ const rows = select.all(sourceId, p);
186
+ for (const r of rows)
187
+ chunkIds.push(r.id);
188
+ }
189
+ this.db.raw.exec('BEGIN');
190
+ try {
191
+ if (chunkIds.length > 0) {
192
+ const delEmb = this.db.raw.prepare('DELETE FROM knowledge_chunk_embeddings WHERE chunk_id = ?');
193
+ for (const id of chunkIds)
194
+ delEmb.run(id);
195
+ }
196
+ const delChunks = this.db.raw.prepare('DELETE FROM knowledge_chunks WHERE source_id = ? AND path = ?');
197
+ const delFiles = this.db.raw.prepare('DELETE FROM knowledge_files WHERE source_id = ? AND path = ?');
198
+ for (const p of paths) {
199
+ delChunks.run(sourceId, p);
200
+ delFiles.run(sourceId, p);
201
+ }
202
+ this.db.raw.exec('COMMIT');
203
+ }
204
+ catch (e) {
205
+ this.db.raw.exec('ROLLBACK');
206
+ throw e;
207
+ }
208
+ }
209
+ /**
210
+ * 删除路径及子树(目录被移出 / 删除)
211
+ *
212
+ * @param sourceId - 知识源 id
213
+ * @param path - 文件或目录绝对路径
214
+ */
215
+ removePathTree(sourceId, path) {
216
+ this.db.raw.exec('BEGIN');
217
+ try {
218
+ this.removeFile(sourceId, path);
219
+ const likeSlash = `${path}/%`;
220
+ const likeBack = `${path}\\%`;
221
+ this.db.raw
222
+ .prepare(`DELETE FROM knowledge_chunk_embeddings
223
+ WHERE chunk_id IN (
224
+ SELECT id FROM knowledge_chunks
225
+ WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)
226
+ )`)
227
+ .run(sourceId, likeSlash, likeBack);
228
+ this.db.raw
229
+ .prepare(`DELETE FROM knowledge_chunks WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
230
+ .run(sourceId, likeSlash, likeBack);
231
+ this.db.raw
232
+ .prepare(`DELETE FROM knowledge_files WHERE source_id = ? AND (path LIKE ? OR path LIKE ?)`)
233
+ .run(sourceId, likeSlash, likeBack);
234
+ this.db.raw.exec('COMMIT');
235
+ }
236
+ catch (e) {
237
+ this.db.raw.exec('ROLLBACK');
238
+ throw e;
239
+ }
240
+ }
171
241
  /**
172
242
  * 差量 prune:删除本轮 discover **未出现** 的 path(含 chunks)。
173
243
  * 仅在 discover 成功后调用;keepPaths 为空集时 no-op(防误清全库)。
174
244
  *
245
+ * @param keepPaths - 应保留的 path 集
246
+ * @param opts.allowEmptyKeep - walk 成功且目录为空时允许清空(默认 false)
175
247
  * @returns 删除的文件数
176
248
  */
177
- pruneMissing(sourceId, keepPaths) {
249
+ pruneMissing(sourceId, keepPaths, opts) {
178
250
  const keep = keepPaths instanceof Set ? keepPaths : new Set(keepPaths);
179
- if (keep.size === 0)
251
+ if (keep.size === 0 && !opts?.allowEmptyKeep)
180
252
  return 0;
181
253
  const files = this.listFiles(sourceId);
182
- let removed = 0;
183
- for (const f of files) {
184
- if (!keep.has(f.path)) {
185
- this.removeFile(sourceId, f.path);
186
- removed += 1;
187
- }
188
- }
189
- return removed;
254
+ const gone = files.filter((f) => !keep.has(f.path)).map((f) => f.path);
255
+ if (gone.length === 0)
256
+ return 0;
257
+ this.removeFiles(sourceId, gone);
258
+ return gone.length;
190
259
  }
191
260
  sourceStats(sourceId) {
192
261
  const f = this.db.raw
@@ -199,9 +268,21 @@ export class KnowledgeIndexStore {
199
268
  const c = this.db.raw
200
269
  .prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
201
270
  .get(sourceId);
271
+ const ec = this.db.raw
272
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
273
+ LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
274
+ WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
275
+ .get(sourceId);
276
+ const e = this.db.raw
277
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
278
+ JOIN knowledge_chunks c ON c.id = e.chunk_id
279
+ WHERE c.source_id = ?`)
280
+ .get(sourceId);
202
281
  return {
203
282
  files: f?.total ?? 0,
204
283
  chunks: c?.n ?? 0,
284
+ embeddableChunks: ec?.n ?? 0,
285
+ embeddings: e?.n ?? 0,
205
286
  errors: f?.errors ?? 0,
206
287
  skipped: f?.skipped ?? 0,
207
288
  };
@@ -285,72 +366,178 @@ export class KnowledgeIndexStore {
285
366
  return mapped;
286
367
  }
287
368
  /**
288
- * 写入 chunk 向量(JSON 冗余;P3 用 JS cosine)
369
+ * 写入 chunk 向量(Float32 BLOB;sqlite-vec 可用时同步进 vec 表)
289
370
  */
290
371
  setChunkEmbedding(chunkId, embedding) {
291
- this.db.raw
292
- .prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding_json, created_at)
293
- VALUES (?, ?, ?, ?)
294
- ON CONFLICT(chunk_id) DO UPDATE SET
295
- dimensions=excluded.dimensions,
296
- embedding_json=excluded.embedding_json,
297
- created_at=excluded.created_at`)
298
- .run(chunkId, embedding.length, JSON.stringify(embedding), Date.now());
372
+ this.setChunkEmbeddings([[chunkId, embedding]]);
373
+ }
374
+ /**
375
+ * 批量写向量(单事务,减少与 parse 写路径的锁竞争)
376
+ *
377
+ * @param rows - [chunkId, embedding] 列表
378
+ */
379
+ setChunkEmbeddings(rows) {
380
+ if (rows.length === 0)
381
+ return;
382
+ const upsert = this.db.raw.prepare(`INSERT INTO knowledge_chunk_embeddings (chunk_id, dimensions, embedding, created_at)
383
+ VALUES (?, ?, ?, ?)
384
+ ON CONFLICT(chunk_id) DO UPDATE SET
385
+ dimensions=excluded.dimensions,
386
+ embedding=excluded.embedding,
387
+ created_at=excluded.created_at`);
388
+ this.db.raw.exec('BEGIN');
389
+ try {
390
+ const now = Date.now();
391
+ for (const [chunkId, embedding] of rows) {
392
+ upsert.run(chunkId, embedding.length, toF32Blob(embedding), now);
393
+ this.upsertKnowledgeVec(chunkId, embedding);
394
+ }
395
+ this.db.raw.exec('COMMIT');
396
+ }
397
+ catch (e) {
398
+ this.db.raw.exec('ROLLBACK');
399
+ throw e;
400
+ }
401
+ }
402
+ /** sqlite-vec 表名 */
403
+ static VEC_TABLE = 'knowledge_vec';
404
+ /** 确保 vec0 虚拟表(维度首次写入时确定) */
405
+ ensureVecTable(dimensions) {
406
+ if (!this.db.sqliteVecEnabled)
407
+ return false;
408
+ try {
409
+ const row = this.db.raw
410
+ .prepare(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
411
+ .get(KnowledgeIndexStore.VEC_TABLE);
412
+ if (row?.sql) {
413
+ const m = /float\[(\d+)\]/i.exec(row.sql);
414
+ return !m || Number(m[1]) === dimensions;
415
+ }
416
+ this.db.raw.exec(`
417
+ CREATE VIRTUAL TABLE IF NOT EXISTS ${KnowledgeIndexStore.VEC_TABLE} USING vec0(
418
+ chunk_id TEXT PRIMARY KEY,
419
+ embedding float[${dimensions}] distance_metric=cosine
420
+ );
421
+ `);
422
+ return true;
423
+ }
424
+ catch {
425
+ return false;
426
+ }
427
+ }
428
+ upsertKnowledgeVec(chunkId, embedding) {
429
+ if (!this.ensureVecTable(embedding.length))
430
+ return;
431
+ try {
432
+ this.db.raw
433
+ .prepare(`INSERT INTO ${KnowledgeIndexStore.VEC_TABLE} (chunk_id, embedding)
434
+ VALUES (?, ?)
435
+ ON CONFLICT(chunk_id) DO UPDATE SET embedding = excluded.embedding`)
436
+ .run(String(chunkId), toF32Blob(embedding));
437
+ }
438
+ catch {
439
+ // vec 表不可用时忽略,检索走 JS 路径
440
+ }
299
441
  }
300
442
  /**
301
443
  * 列出待嵌入 chunk(无向量行)
444
+ *
445
+ * **代码不进向量**:原文件已可 file_search;片段易失真。
446
+ * 仅文档/正文类(text / markdown / document:* 等)参与 embedding。
302
447
  */
303
448
  listChunksMissingEmbedding(sourceId, limit = 100) {
304
449
  const rows = this.db.raw
305
450
  .prepare(`SELECT c.id, c.text, c.path
306
451
  FROM knowledge_chunks c
307
452
  LEFT JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
453
+ LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
308
454
  WHERE c.source_id = ? AND e.chunk_id IS NULL
455
+ AND ${EMBEDDABLE_WHERE}
309
456
  ORDER BY c.path, c.ordinal
310
457
  LIMIT ?`)
311
458
  .all(sourceId, limit);
312
459
  return rows.map((r) => ({ id: asChunkId(r.id), text: r.text, path: r.path }));
313
460
  }
314
461
  /**
315
- * 向量覆盖度(0–1;无 chunk 时为 1)
462
+ * 向量覆盖度(0–1;无可嵌入 chunk 时为 1)
316
463
  */
317
464
  embeddingCoverage(sourceId) {
318
465
  const total = this.db.raw
319
- .prepare('SELECT COUNT(*) AS n FROM knowledge_chunks WHERE source_id = ?')
466
+ .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunks c
467
+ LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
468
+ WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
320
469
  .get(sourceId);
321
470
  if (!total?.n)
322
471
  return 1;
323
472
  const embedded = this.db.raw
324
473
  .prepare(`SELECT COUNT(*) AS n FROM knowledge_chunk_embeddings e
325
474
  JOIN knowledge_chunks c ON c.id = e.chunk_id
326
- WHERE c.source_id = ?`)
475
+ LEFT JOIN knowledge_files f ON f.source_id = c.source_id AND f.path = c.path
476
+ WHERE c.source_id = ? AND ${EMBEDDABLE_WHERE}`)
327
477
  .get(sourceId);
328
478
  return Math.min(1, (embedded?.n ?? 0) / total.n);
329
479
  }
330
480
  /**
331
- * 向量检索(JS cosine);embeddingJson 不可用的 chunk 跳过
481
+ * 向量检索:优先 sqlite-vec KNN;否则 JS cosine(BLOB)
332
482
  */
333
483
  vectorSearch(queryEmbedding, opts) {
334
484
  if (opts.sourceIds.length === 0 || !queryEmbedding?.length)
335
485
  return [];
336
486
  const limit = opts.limit ?? 8;
487
+ // vec 表可能尚未回填;0 命中时退 JS,避免空结果短路
488
+ const viaVec = this.vectorSearchVec(queryEmbedding, opts, limit);
489
+ if (viaVec && viaVec.length > 0)
490
+ return viaVec;
491
+ return this.vectorSearchJs(queryEmbedding, opts, limit);
492
+ }
493
+ /** sqlite-vec KNN(不可用时返回 null) */
494
+ vectorSearchVec(queryEmbedding, opts, limit) {
495
+ if (!this.db.sqliteVecEnabled)
496
+ return null;
497
+ if (!this.ensureVecTable(queryEmbedding.length))
498
+ return null;
499
+ const placeholders = opts.sourceIds.map(() => '?').join(',');
500
+ try {
501
+ const rows = this.db.raw
502
+ .prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, v.distance AS distance
503
+ FROM (
504
+ SELECT chunk_id, distance
505
+ FROM ${KnowledgeIndexStore.VEC_TABLE}
506
+ WHERE embedding MATCH ?
507
+ AND k = ?
508
+ ) v
509
+ JOIN knowledge_chunks c ON c.id = v.chunk_id
510
+ WHERE c.source_id IN (${placeholders})
511
+ ORDER BY v.distance ASC`)
512
+ .all(toF32Blob(queryEmbedding), Math.max(limit * 2, 16), ...opts.sourceIds);
513
+ return rows.slice(0, limit).map((row) => ({
514
+ chunkId: asChunkId(row.id),
515
+ sourceId: asSourceId(row.source_id),
516
+ path: row.path,
517
+ ordinal: row.ordinal,
518
+ text: row.text,
519
+ startLine: row.start_line,
520
+ endLine: row.end_line,
521
+ score: 1 - Number(row.distance),
522
+ }));
523
+ }
524
+ catch {
525
+ return null;
526
+ }
527
+ }
528
+ /** JS 余弦(Float32 BLOB;无 JSON.parse) */
529
+ vectorSearchJs(queryEmbedding, opts, limit) {
337
530
  const placeholders = opts.sourceIds.map(() => '?').join(',');
338
531
  const rows = this.db.raw
339
- .prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.embedding_json
532
+ .prepare(`SELECT c.id, c.source_id, c.path, c.ordinal, c.text, c.start_line, c.end_line, e.embedding
340
533
  FROM knowledge_chunks c
341
534
  JOIN knowledge_chunk_embeddings e ON e.chunk_id = c.id
342
535
  WHERE c.source_id IN (${placeholders})
343
- LIMIT 20000`)
536
+ LIMIT 200000`)
344
537
  .all(...opts.sourceIds);
345
538
  const hits = [];
346
539
  for (const row of rows) {
347
- let emb = null;
348
- try {
349
- emb = JSON.parse(row.embedding_json);
350
- }
351
- catch {
352
- emb = null;
353
- }
540
+ const emb = fromF32Blob(row.embedding);
354
541
  if (!emb?.length || emb.length !== queryEmbedding.length)
355
542
  continue;
356
543
  const sim = cosineSimilarity(queryEmbedding, emb);
@@ -371,6 +558,27 @@ export class KnowledgeIndexStore {
371
558
  return hits.slice(0, limit);
372
559
  }
373
560
  }
561
+ /** number[] → float32 blob */
562
+ function toF32Blob(embedding) {
563
+ const f32 = Float32Array.from(embedding);
564
+ return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
565
+ }
566
+ /** float32 blob → number[] */
567
+ function fromF32Blob(buf) {
568
+ if (!buf || buf.byteLength === 0)
569
+ return null;
570
+ if (buf.byteLength % 4 !== 0)
571
+ return null;
572
+ const u8 = buf instanceof Buffer ? new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength) : buf;
573
+ const f32 = new Float32Array(u8.buffer, u8.byteOffset, u8.byteLength / 4);
574
+ return Array.from(f32);
575
+ }
576
+ /**
577
+ * 内容指纹(新鲜度判断)
578
+ *
579
+ * @param content - UTF-8 文本或原始字节(二进制文档用 bytes)
580
+ * @returns sha256 hex
581
+ */
374
582
  export function hashContent(content) {
375
583
  return createHash('sha256').update(content).digest('hex');
376
584
  }
@@ -31,6 +31,8 @@ export { KnowledgePurger } from './purge.js';
31
31
  export type { PurgeResult } from './purge.js';
32
32
  export { resolveKnowledgePaths } from './paths.js';
33
33
  export type { KnowledgePaths } from './paths.js';
34
+ export { deriveTopicsFromPaths } from './topics.js';
35
+ export type { KnowledgeCatalogItem, KnowledgeCatalogProvider } from './catalog-types.js';
34
36
  export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
35
37
  export type { KnowledgeDescribePort, KnowledgeDescribeOptions, KnowledgeDescribeResult, } from './describe.js';
36
38
  export { scanSecretShapes } from './secret-scan.js';
@@ -19,6 +19,7 @@ export { GroundingAssembler, formatKnowledgeGroundingMessage, resolveGroundingQu
19
19
  export { KnowledgeHitLog } from './hit-log.js';
20
20
  export { KnowledgePurger } from './purge.js';
21
21
  export { resolveKnowledgePaths } from './paths.js';
22
+ export { deriveTopicsFromPaths } from './topics.js';
22
23
  export { generateKnowledgeDescription, heuristicDescription, } from './describe.js';
23
24
  export { scanSecretShapes } from './secret-scan.js';
24
25
  export { asSourceId, asChunkId } from './types.js';
@@ -11,6 +11,7 @@ import { type SourceFetcher, type VirtualDocument } from './fetchers.js';
11
11
  import { ConnectorRegistry } from './connectors.js';
12
12
  import type { CredentialStore } from '../governance/credentials/store.js';
13
13
  import type { EmbeddingProvider } from '../memory/sqlite/embedding.js';
14
+ import type { DocumentPort } from '../context/capabilities/document/types.js';
14
15
  export type IngestJobKind = 'parse_file' | 'walk_source' | 'drop_file' | 'embed_source' | 'fetch_doc';
15
16
  export interface IngestProgressEvent {
16
17
  type: 'job' | 'source' | 'error' | 'embed';
@@ -31,6 +32,8 @@ export interface KnowledgeIngestOptions {
31
32
  debounceMs?: number;
32
33
  /** 单文件大小上限(默认 5MB;超出 skip) */
33
34
  maxFileBytes?: number;
35
+ /** 单文件 parse 超时 ms(默认 45000) */
36
+ parseTimeoutMs?: number;
34
37
  /** Phase B embedding(未配则纯关键词) */
35
38
  embeddingProvider?: EmbeddingProvider | null;
36
39
  /** embed 批大小(默认 32) */
@@ -53,6 +56,11 @@ export interface KnowledgeIngestOptions {
53
56
  pollMinIntervalMs?: number;
54
57
  /** 每轮 poll 最多处理源数(成本上限,默认 8) */
55
58
  maxPollPerTick?: number;
59
+ /**
60
+ * DocumentPort:启用后 PDF/Office 等走抽取 → Markdown 切块
61
+ * (缺省 null = 维持 BINARY skip 语义)
62
+ */
63
+ documentPort?: DocumentPort | null;
56
64
  }
57
65
  export declare class KnowledgeIngest extends EventEmitter {
58
66
  private readonly sources;
@@ -62,6 +70,8 @@ export declare class KnowledgeIngest extends EventEmitter {
62
70
  private readonly maxQueueDepth;
63
71
  private readonly debounceMs;
64
72
  private readonly maxFileBytes;
73
+ /** 单文件 parse/fetch 超时(默认 45s;大 Office 同步抽取会堵事件循环) */
74
+ private readonly parseTimeoutMs;
65
75
  private readonly embedding;
66
76
  private readonly embedBatch;
67
77
  private readonly embedMinIntervalMs;
@@ -74,11 +84,20 @@ export declare class KnowledgeIngest extends EventEmitter {
74
84
  private readonly pollTickMs;
75
85
  private readonly pollMinIntervalMs;
76
86
  private readonly maxPollPerTick;
87
+ private readonly documentPort;
77
88
  private lastEmbedAt;
78
89
  private pollTimer;
79
- private running;
90
+ private reconcileTimer;
91
+ /** 源级中止控制器(手动 abort / 重建前清场) */
92
+ private abortBySource;
93
+ /** 源任务稳定(无 queued/running)时回调 — 用于 auto-describe 等收尾 */
94
+ onSourceSettled?: (sourceId: string) => void;
95
+ /** 上次 parse 缺口扫描时间 */
96
+ private lastParseGapScanAt;
97
+ private runningParse;
80
98
  private runningEmbed;
81
99
  private draining;
100
+ private kickAgain;
82
101
  /** source 级锁:同一 source 不并行 walk/parse 冲突写 */
83
102
  private sourceLocks;
84
103
  /** walk 发现的文件数(Phase A coverage 分母) */
@@ -87,7 +106,19 @@ export declare class KnowledgeIngest extends EventEmitter {
87
106
  private debounceTimers;
88
107
  private disposed;
89
108
  constructor(options: KnowledgeIngestOptions);
109
+ /**
110
+ * 无跨进程 Lease:本进程启动时把历史 running 收回 queued。
111
+ * 仅构造时调用——运行中同进程的 running 是合法的。
112
+ */
113
+ private reclaimOrphanRunningJobs;
90
114
  get indexStore(): KnowledgeIndexStore;
115
+ /**
116
+ * 路径过滤:有 DocumentPort 时文档扩展名不再被 BINARY 短路。
117
+ *
118
+ * @param p - 文件路径
119
+ * @returns 是否跳过
120
+ */
121
+ private shouldSkipFile;
91
122
  /**
92
123
  * 全量/增量索引某源(Phase A)
93
124
  */
@@ -129,6 +160,75 @@ export declare class KnowledgeIngest extends EventEmitter {
129
160
  */
130
161
  idle(timeoutMs?: number): Promise<void>;
131
162
  dispose(): void;
163
+ /**
164
+ * 手动中止索引任务(业务完整:停收 + 清队 + 打断在跑)
165
+ *
166
+ * @param opts.sourceId - 仅中止该源;省略则全部源
167
+ * @returns 取消统计
168
+ */
169
+ abortJobs(opts?: {
170
+ sourceId?: string;
171
+ }): {
172
+ cancelledQueued: number;
173
+ abortedRunning: number;
174
+ };
175
+ /**
176
+ * 恢复/继续索引(相对「重建」更轻)
177
+ *
178
+ * - 清除该源中止态
179
+ * - 把 cancelled 任务放回 queued
180
+ * - 若仍缺向量则排队 embed
181
+ * - 不重新 walk 全量;需要全量对齐请 reindex
182
+ *
183
+ * @param opts.sourceId - 仅恢复该源;省略则全部源
184
+ * @returns 恢复统计
185
+ */
186
+ resumeJobs(opts?: {
187
+ sourceId?: string;
188
+ }): {
189
+ restoredCancelled: number;
190
+ embedQueued: number;
191
+ };
192
+ /** 开启新的中止纪元(abort 后新任务不受旧信号影响) */
193
+ private beginAbortEpoch;
194
+ /**
195
+ * 重建前作废本源未完成任务(supersede)
196
+ *
197
+ * - queued → cancelled(superseded_by_reindex)
198
+ * - running 打断(worker/循环退出)
199
+ * - 随后 beginAbortEpoch,本轮重建可继续写
200
+ */
201
+ private supersedeSourceWork;
202
+ private ensureParseCoverage;
203
+ /**
204
+ * 磁盘↔索引对齐:删掉已不存在的文件(含 chunks/embeddings)
205
+ *
206
+ * 目录源用一次 discover walk 得 keep 集(快);失败退回并行 stat。
207
+ * embed 前调用,避免给已移出文件的残 chunks 上向量。
208
+ *
209
+ * @param sourceId - 知识源 id
210
+ * @returns 删除的文件数
211
+ */
212
+ pruneMissingOnDisk(sourceId: string): Promise<number>;
213
+ /** 逐文件 stat(file 源 / walk 失败回退);并行限流 */
214
+ private pruneByStatScan;
215
+ private abortSignalFor;
216
+ private isAborted;
217
+ /**
218
+ * 任务对账(看门狗)— 不依赖单次 ensureEmbedJob
219
+ *
220
+ * 解决「续跑被去重吞掉 / 进程中断 / 状态卡 partial」等静默停摆:
221
+ * 1. 回收超时 running(崩溃/挂死孤儿)
222
+ * 2. 有缺向量且无 active embed → 自动排队
223
+ * 3. 无队列、无缺口 → 刷 coverage/status
224
+ */
225
+ startReconciler(intervalMs?: number): void;
226
+ stopReconciler(): void;
227
+ /** 对账入口(也可手工调用) */
228
+ reconcileJobs(): Promise<void>;
229
+ /** running 超时(无 heartbeat)→ 收回 queued;多次失败的直接标 failed,防毒文件死循环 */
230
+ private reclaimStaleRunningJobs;
231
+ private countActiveJobs;
132
232
  /**
133
233
  * 启动 poll 调度(幂等);仅处理 sync.strategy=poll 且 enabled 的源
134
234
  */
@@ -144,17 +244,38 @@ export declare class KnowledgeIngest extends EventEmitter {
144
244
  private enqueue;
145
245
  private kick;
146
246
  private drain;
247
+ private countQueuedKinds;
147
248
  private claimJob;
148
249
  private runJob;
250
+ private withTimeout;
149
251
  /**
150
252
  * 外源单文档 job:按逻辑键 fetch + 索引
151
253
  */
152
254
  private fetchDocJob;
153
255
  private parseOne;
154
256
  /**
155
- * Phase B:嵌入待处理 chunk(批 + 限速;失败不丢,下次再补)
257
+ * PDF/Office:DocumentPort 抽取为 Markdown 后走 markdownAdapter 切块。
258
+ * 新鲜度以原件字节 hash 为准。
259
+ *
260
+ * @param sourceId - 知识源 id
261
+ * @param filePath - 本地文件路径
262
+ * @returns 是否成功入索引
263
+ */
264
+ private parseDocumentFile;
265
+ /** 优先 worker 抽取;worker 不可用时退回进程内 port */
266
+ private extractDocument;
267
+ /**
268
+ * Phase B:嵌入待处理 chunk
269
+ *
270
+ * 单 job 内按 embedConcurrency **并行**多批调用 API;
271
+ * 失败/超时也 re-enqueue,避免 job 标 done 后 coverage 永久卡住。
156
272
  */
157
273
  private embedPending;
274
+ /** 单片 embedding;返回成功写入条数 */
275
+ private embedOneSlice;
276
+ /** 仍有缺向量则排队下一个 embed_source(须在当前 job 标 done 之后调用) */
277
+ private ensureEmbedJob;
278
+ private heartbeatJob;
158
279
  private refreshCoverage;
159
280
  private emitProgress;
160
281
  /** 磁盘水位告警(仅告警,不丢任务) */