@octopi-agent/engine 0.55.0 → 0.58.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/config-schema/context.d.ts +38 -0
  2. package/dist/config-schema/context.js +39 -0
  3. package/dist/config-schema/engine.d.ts +42 -0
  4. package/dist/config-schema/engine.js +3 -1
  5. package/dist/config-schema/substrate.d.ts +5 -0
  6. package/dist/config-schema/substrate.js +11 -0
  7. package/dist/config.d.ts +44 -0
  8. package/dist/harness/agent/config-bridge.js +3 -0
  9. package/dist/harness/context/capabilities/document/backends/docx.d.ts +7 -0
  10. package/dist/harness/context/capabilities/document/backends/docx.js +67 -0
  11. package/dist/harness/context/capabilities/document/backends/office-parser.d.ts +7 -0
  12. package/dist/harness/context/capabilities/document/backends/office-parser.js +76 -0
  13. package/dist/harness/context/capabilities/document/backends/pdf.d.ts +7 -0
  14. package/dist/harness/context/capabilities/document/backends/pdf.js +86 -0
  15. package/dist/harness/context/capabilities/document/backends/plain.d.ts +7 -0
  16. package/dist/harness/context/capabilities/document/backends/plain.js +42 -0
  17. package/dist/harness/context/capabilities/document/backends/sheet.d.ts +7 -0
  18. package/dist/harness/context/capabilities/document/backends/sheet.js +90 -0
  19. package/dist/harness/context/capabilities/document/backends/xmind.d.ts +20 -0
  20. package/dist/harness/context/capabilities/document/backends/xmind.js +198 -0
  21. package/dist/harness/context/capabilities/document/errors.d.ts +27 -0
  22. package/dist/harness/context/capabilities/document/errors.js +33 -0
  23. package/dist/harness/context/capabilities/document/format.d.ts +38 -0
  24. package/dist/harness/context/capabilities/document/format.js +113 -0
  25. package/dist/harness/context/capabilities/document/html-to-md.d.ts +14 -0
  26. package/dist/harness/context/capabilities/document/html-to-md.js +97 -0
  27. package/dist/harness/context/capabilities/document/index.d.ts +21 -0
  28. package/dist/harness/context/capabilities/document/index.js +20 -0
  29. package/dist/harness/context/capabilities/document/legacy/factory.d.ts +13 -0
  30. package/dist/harness/context/capabilities/document/legacy/factory.js +23 -0
  31. package/dist/harness/context/capabilities/document/legacy/soffice.d.ts +32 -0
  32. package/dist/harness/context/capabilities/document/legacy/soffice.js +186 -0
  33. package/dist/harness/context/capabilities/document/port.d.ts +19 -0
  34. package/dist/harness/context/capabilities/document/port.js +400 -0
  35. package/dist/harness/context/capabilities/document/types.d.ts +149 -0
  36. package/dist/harness/context/capabilities/document/types.js +8 -0
  37. package/dist/harness/context/capabilities/index.d.ts +1 -0
  38. package/dist/harness/context/capabilities/index.js +1 -0
  39. package/dist/harness/context/constitution/default-agents.md +1 -0
  40. package/dist/harness/context/index.d.ts +2 -2
  41. package/dist/harness/context/index.js +1 -1
  42. package/dist/harness/context/layer-types.d.ts +26 -2
  43. package/dist/harness/context/layer-types.js +59 -11
  44. package/dist/harness/context/layers.d.ts +3 -0
  45. package/dist/harness/context/layers.js +38 -12
  46. package/dist/harness/context/system-prompt-assembler.d.ts +5 -0
  47. package/dist/harness/context/system-prompt-assembler.js +2 -0
  48. package/dist/harness/extension/plugin-ecosystem/tools/builtin.d.ts +1 -0
  49. package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +10 -0
  50. package/dist/harness/extension/plugin-ecosystem/tools/document-tools.d.ts +64 -0
  51. package/dist/harness/extension/plugin-ecosystem/tools/document-tools.js +155 -0
  52. package/dist/harness/extension/plugin-ecosystem/tools/index.d.ts +1 -0
  53. package/dist/harness/extension/plugin-ecosystem/tools/index.js +1 -0
  54. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +13 -0
  55. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +3 -0
  56. package/dist/harness/extension/plugin-ecosystem/tools/tool-set.d.ts +2 -0
  57. package/dist/harness/extension/plugin-ecosystem/tools/tool-set.js +4 -1
  58. package/dist/harness/index.d.ts +3 -3
  59. package/dist/harness/index.js +2 -2
  60. package/dist/harness/knowledge/catalog-types.d.ts +4 -0
  61. package/dist/harness/knowledge/db.d.ts +4 -0
  62. package/dist/harness/knowledge/db.js +58 -3
  63. package/dist/harness/knowledge/describe.d.ts +2 -0
  64. package/dist/harness/knowledge/describe.js +5 -1
  65. package/dist/harness/knowledge/document-extract-worker.d.ts +1 -0
  66. package/dist/harness/knowledge/document-extract-worker.js +25 -0
  67. package/dist/harness/knowledge/extract-document.d.ts +17 -0
  68. package/dist/harness/knowledge/extract-document.js +62 -0
  69. package/dist/harness/knowledge/index-store.d.ts +49 -5
  70. package/dist/harness/knowledge/index-store.js +240 -32
  71. package/dist/harness/knowledge/index.d.ts +2 -0
  72. package/dist/harness/knowledge/index.js +1 -0
  73. package/dist/harness/knowledge/ingest.d.ts +123 -2
  74. package/dist/harness/knowledge/ingest.js +661 -59
  75. package/dist/harness/knowledge/retriever.d.ts +8 -0
  76. package/dist/harness/knowledge/retriever.js +33 -1
  77. package/dist/harness/knowledge/source-store.d.ts +4 -0
  78. package/dist/harness/knowledge/source-store.js +62 -6
  79. package/dist/harness/knowledge/topics.d.ts +13 -0
  80. package/dist/harness/knowledge/topics.js +48 -0
  81. package/dist/harness/memory/index.d.ts +3 -2
  82. package/dist/harness/memory/index.js +3 -1
  83. package/dist/harness/memory/retrieval-rank.d.ts +51 -0
  84. package/dist/harness/memory/retrieval-rank.js +60 -0
  85. package/dist/harness/memory/sqlite/index.d.ts +2 -2
  86. package/dist/harness/memory/sqlite/index.js +1 -1
  87. package/dist/harness/memory/sqlite/keyword-search.d.ts +20 -2
  88. package/dist/harness/memory/sqlite/keyword-search.js +40 -17
  89. package/dist/harness/memory/sqlite/memory-store.d.ts +18 -4
  90. package/dist/harness/memory/sqlite/memory-store.js +89 -39
  91. package/dist/harness/memory/store.js +39 -21
  92. package/dist/harness/memory/types.d.ts +7 -0
  93. package/dist/harness/run/runner.d.ts +1 -0
  94. package/dist/harness/run/runner.js +4 -2
  95. package/dist/harness/session/attachments/extract.d.ts +3 -2
  96. package/dist/harness/session/attachments/extract.js +19 -6
  97. package/dist/harness/session/attachments/service.d.ts +5 -1
  98. package/dist/harness/session/attachments/service.js +44 -2
  99. package/dist/harness/session/tasks/index.d.ts +1 -1
  100. package/dist/harness/session/tasks/index.js +1 -1
  101. package/dist/harness/session/tasks/render.d.ts +10 -0
  102. package/dist/harness/session/tasks/render.js +27 -0
  103. package/dist/index.d.ts +2 -2
  104. package/dist/index.js +1 -1
  105. package/package.json +8 -3
@@ -12,6 +12,12 @@ export interface HybridSearchOptions {
12
12
  limit?: number;
13
13
  /** 仅关键词(跳过 embedding query) */
14
14
  keywordOnly?: boolean;
15
+ /**
16
+ * 可选源过滤(与可见集求交,禁止越权)。
17
+ * `sourceIds` 精确 id;`source` 为 displayName/id 模糊(单源)。
18
+ */
19
+ sourceIds?: string[];
20
+ source?: string;
15
21
  }
16
22
  export interface HybridSearchResult {
17
23
  hits: ChunkHit[];
@@ -70,6 +76,8 @@ export declare class KnowledgeRetriever {
70
76
  * 可见 sourceId 列表
71
77
  */
72
78
  visibleSourceIds(agentId: string, sessionId?: string): import('./types.js').KnowledgeSourceId[];
79
+ /** 可见集 ∩ 请求过滤;无过滤时原样返回可见集 */
80
+ private filterSourceIds;
73
81
  /**
74
82
  * hybrid 检索
75
83
  */
@@ -33,11 +33,43 @@ export class KnowledgeRetriever {
33
33
  visibleSourceIds(agentId, sessionId) {
34
34
  return this.sources.listVisible(agentId, sessionId).map((s) => s.id);
35
35
  }
36
+ /** 可见集 ∩ 请求过滤;无过滤时原样返回可见集 */
37
+ filterSourceIds(visible, opts) {
38
+ const requested = opts.sourceIds?.map((s) => s.trim()).filter(Boolean) ?? [];
39
+ const name = opts.source?.trim();
40
+ if (requested.length === 0 && !name)
41
+ return visible;
42
+ const visibleSet = new Set(visible);
43
+ let allowed = null;
44
+ if (requested.length > 0) {
45
+ allowed = new Set(requested.filter((id) => visibleSet.has(id)));
46
+ }
47
+ if (name) {
48
+ const lower = name.toLowerCase();
49
+ const matched = visible.filter((id) => {
50
+ const src = this.sources.get(id);
51
+ if (!src)
52
+ return false;
53
+ return (src.id.toLowerCase() === lower ||
54
+ src.displayName.toLowerCase() === lower ||
55
+ src.displayName.toLowerCase().includes(lower) ||
56
+ src.id.toLowerCase().includes(lower));
57
+ });
58
+ if (allowed) {
59
+ allowed = new Set(matched.filter((id) => allowed.has(id)));
60
+ }
61
+ else {
62
+ allowed = new Set(matched);
63
+ }
64
+ }
65
+ return visible.filter((id) => allowed?.has(id) ?? true);
66
+ }
36
67
  /**
37
68
  * hybrid 检索
38
69
  */
39
70
  async search(query, opts) {
40
- const sourceIds = this.visibleSourceIds(opts.agentId, opts.sessionId);
71
+ const visible = this.visibleSourceIds(opts.agentId, opts.sessionId);
72
+ const sourceIds = this.filterSourceIds(visible, opts);
41
73
  const limit = opts.limit ?? 8;
42
74
  const keywordHits = this.hybridKeyword
43
75
  ? this.index.search(query, { sourceIds, limit: limit * 2 })
@@ -91,6 +91,10 @@ export declare class KnowledgeSourceStore {
91
91
  sessionId?: string;
92
92
  maxEntries?: number;
93
93
  }): KnowledgeCatalogItem[];
94
+ /** 每源已入库文件数(catalog 规模粗标用) */
95
+ private fileCountsBySource;
96
+ /** 每源路径派生 topics(catalog 搜索线索) */
97
+ private topicsBySource;
94
98
  /**
95
99
  * catalog 粗桶指纹(防每轮重渲)
96
100
  */
@@ -3,6 +3,7 @@
3
3
  */
4
4
  import { randomUUID, createHash } from 'node:crypto';
5
5
  import { KnowledgeDatabase } from './db.js';
6
+ import { deriveTopicsFromPaths } from './topics.js';
6
7
  import { asSourceId } from './types.js';
7
8
  const DEFAULT_SYNC = { strategy: 'watch', debounceMs: 2000, enabled: true };
8
9
  function statusBucket(status) {
@@ -16,11 +17,28 @@ function statusBucket(status) {
16
17
  return 'off';
17
18
  return 'pending';
18
19
  }
19
- function scaleLabel(source) {
20
+ /** 规模粗标:`~1.2k files` / `small`(arch/knowledge-layer.md §4.1);不输出精确 coverage */
21
+ function formatFileScale(fileCount) {
22
+ if (fileCount <= 0)
23
+ return 'empty';
24
+ if (fileCount < 30)
25
+ return 'small';
26
+ if (fileCount < 1000)
27
+ return `~${fileCount} files`;
28
+ if (fileCount < 1_000_000) {
29
+ const k = fileCount / 1000;
30
+ return `~${k >= 10 ? Math.round(k) : Math.round(k * 10) / 10}k files`;
31
+ }
32
+ const m = fileCount / 1_000_000;
33
+ return `~${m >= 10 ? Math.round(m) : Math.round(m * 10) / 10}m files`;
34
+ }
35
+ function scaleLabel(source, fileCount) {
36
+ if (fileCount != null && fileCount > 0)
37
+ return formatFileScale(fileCount);
20
38
  const cov = source.coverage;
21
39
  if (cov == null)
22
40
  return undefined;
23
- // 粗标,避免 coverage 数字抖动进 catalog fingerprint
41
+ // 尚无文件规模时用覆盖粗桶;避免 coverage 数字抖动进 catalog fingerprint
24
42
  if (cov >= 0.999)
25
43
  return 'ready';
26
44
  if (cov >= 0.5)
@@ -41,7 +59,7 @@ function deriveDisplayName(location) {
41
59
  const parts = cleaned.split(/[\\/]/).filter(Boolean);
42
60
  return parts[parts.length - 1] || location || 'source';
43
61
  }
44
- function toCatalogItem(source) {
62
+ function toCatalogItem(source, extras) {
45
63
  return {
46
64
  id: source.id,
47
65
  displayName: source.displayName,
@@ -49,7 +67,9 @@ function toCatalogItem(source) {
49
67
  status: statusBucket(source.status),
50
68
  description: source.description?.trim() || source.generatedDescription?.trim() || undefined,
51
69
  scopeLevel: source.scopeRef.level,
52
- scaleLabel: scaleLabel(source),
70
+ scaleLabel: scaleLabel(source, extras?.fileCount),
71
+ location: source.location,
72
+ topics: extras?.topics?.length ? extras.topics : undefined,
53
73
  };
54
74
  }
55
75
  function rowToSource(row) {
@@ -459,10 +479,39 @@ export class KnowledgeSourceStore {
459
479
  return pb - pa;
460
480
  return a.displayName.localeCompare(b.displayName);
461
481
  });
482
+ const fileCounts = this.fileCountsBySource();
483
+ const topicsBySource = this.topicsBySource();
462
484
  return sorted
463
485
  .filter((s) => !s.hiddenFromCatalog)
464
486
  .slice(0, max)
465
- .map((s) => toCatalogItem(s));
487
+ .map((s) => toCatalogItem(s, {
488
+ fileCount: fileCounts.get(s.id),
489
+ topics: topicsBySource.get(s.id),
490
+ }));
491
+ }
492
+ /** 每源已入库文件数(catalog 规模粗标用) */
493
+ fileCountsBySource() {
494
+ const rows = this.db.raw
495
+ .prepare('SELECT source_id AS id, COUNT(*) AS n FROM knowledge_files GROUP BY source_id')
496
+ .all();
497
+ return new Map(rows.map((r) => [r.id, r.n]));
498
+ }
499
+ /** 每源路径派生 topics(catalog 搜索线索) */
500
+ topicsBySource() {
501
+ const rows = this.db.raw
502
+ .prepare('SELECT source_id AS id, path AS path FROM knowledge_files')
503
+ .all();
504
+ const bySource = new Map();
505
+ for (const r of rows) {
506
+ const list = bySource.get(r.id) ?? [];
507
+ list.push(r.path);
508
+ bySource.set(r.id, list);
509
+ }
510
+ const out = new Map();
511
+ for (const [id, paths] of bySource) {
512
+ out.set(id, deriveTopicsFromPaths(paths));
513
+ }
514
+ return out;
466
515
  }
467
516
  /**
468
517
  * catalog 粗桶指纹(防每轮重渲)
@@ -470,7 +519,14 @@ export class KnowledgeSourceStore {
470
519
  catalogFingerprint(agentId, opts) {
471
520
  const items = this.catalogFor(agentId, { ...opts, maxEntries: 10_000 });
472
521
  const payload = items
473
- .map((i) => [i.id, i.displayName, i.status ?? '', i.description ?? '', i.scopeLevel ?? ''].join('|'))
522
+ .map((i) => [
523
+ i.id,
524
+ i.displayName,
525
+ i.status ?? '',
526
+ i.description ?? '',
527
+ i.scopeLevel ?? '',
528
+ (i.topics ?? []).join(','),
529
+ ].join('|'))
474
530
  .join('\n');
475
531
  return createHash('sha256').update(payload).digest('hex').slice(0, 16);
476
532
  }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Catalog topics — 从索引路径派生主题线索(无 LLM)
3
+ *
4
+ * 供 Tier 0 catalog 的 `topics` 字段;不依赖人工 description / auto-describe。
5
+ */
6
+ /**
7
+ * 路径词频 → 话题短词(≤ maxTopics)
8
+ *
9
+ * @param paths 已入库相对/绝对路径
10
+ * @param maxTopics 上限(默认 6)
11
+ * @returns 稳定排序的短词列表
12
+ */
13
+ export declare function deriveTopicsFromPaths(paths: string[], maxTopics?: number): string[];
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Catalog topics — 从索引路径派生主题线索(无 LLM)
3
+ *
4
+ * 供 Tier 0 catalog 的 `topics` 字段;不依赖人工 description / auto-describe。
5
+ */
6
+ const STOP_TOKENS = new Set([
7
+ 'src', 'lib', 'dist', 'build', 'out', 'bin', 'pkg', 'packages', 'node_modules',
8
+ 'test', 'tests', 'spec', 'specs', 'assets', 'public', 'static', 'vendor',
9
+ 'the', 'and', 'for', 'with', 'from', 'and', 'app', 'utils', 'common', 'helpers',
10
+ ]);
11
+ /** 从单条路径抽出候选词(目录名 + 文件名词干) */
12
+ function tokensFromPath(path) {
13
+ const parts = path.split(/[\\/]+/).filter(Boolean);
14
+ const out = [];
15
+ for (const part of parts) {
16
+ const stem = part.replace(/\.[^.]+$/, '');
17
+ for (const t of stem.split(/[-_\s.]+/)) {
18
+ if (!t)
19
+ continue;
20
+ const lower = t.toLowerCase();
21
+ if (lower.length < 2 || STOP_TOKENS.has(lower) || /^\d+$/.test(lower))
22
+ continue;
23
+ out.push(lower);
24
+ }
25
+ }
26
+ return out;
27
+ }
28
+ /**
29
+ * 路径词频 → 话题短词(≤ maxTopics)
30
+ *
31
+ * @param paths 已入库相对/绝对路径
32
+ * @param maxTopics 上限(默认 6)
33
+ * @returns 稳定排序的短词列表
34
+ */
35
+ export function deriveTopicsFromPaths(paths, maxTopics = 6) {
36
+ if (paths.length === 0)
37
+ return [];
38
+ const counts = new Map();
39
+ for (const p of paths) {
40
+ for (const t of tokensFromPath(p)) {
41
+ counts.set(t, (counts.get(t) ?? 0) + 1);
42
+ }
43
+ }
44
+ return [...counts.entries()]
45
+ .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
46
+ .slice(0, maxTopics)
47
+ .map(([t]) => t);
48
+ }
@@ -12,8 +12,9 @@ export { AgentDatabase, SqliteMemoryStore, SqliteWisdomStore, SqliteConceptGraph
12
12
  export type { AgentDatabaseOptions, SqliteMemoryStoreOptions, SqliteConceptGraphOptions, EmbeddingProvider, EmbeddingConfig } from './sqlite/index.js';
13
13
  export { createEmbeddingProviderFromModels, resolveEmbeddingRuntime, isEmbeddingEnabled, resolveEmbeddingEndpoint, } from './sqlite/index.js';
14
14
  export type { ResolvedEmbeddingRuntime } from './sqlite/index.js';
15
- export { tokenizeKeywordQuery, scoreKeywordFields, buildKeywordLikeSql, } from './sqlite/index.js';
16
- export type { KeywordFields } from './sqlite/index.js';
15
+ export { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordFields, buildKeywordLikeSql, } from './sqlite/index.js';
16
+ export type { KeywordFields, KeywordTokens } from './sqlite/index.js';
17
+ export { DEFAULT_MIN_SIMILARITY, DEFAULT_SIMILARITY_WEIGHT, DEFAULT_MIN_KEYWORD_SCORE, qualityScore, blendRank, passesSimilarityFloor, resolveRetrievalKnobs, } from './retrieval-rank.js';
17
18
  export { findDuplicate, normalizedProposition, charTrigramSimilarity } from './similarity.js';
18
19
  export type { FindDuplicateOptions, DuplicateHit } from './similarity.js';
19
20
  export { DEFAULT_DECAY_TYPE_PARAMS, resolveDecayParams, nextDecayFactor, isDecayDue, } from './decay-policy.js';
@@ -11,7 +11,9 @@ export { InMemoryConceptGraph } from './cognition.js';
11
11
  // ── SQLite 实现 ──
12
12
  export { AgentDatabase, SqliteMemoryStore, SqliteWisdomStore, SqliteConceptGraph, createEmbeddingProvider, SqliteBackfillCoverageStore } from './sqlite/index.js';
13
13
  export { createEmbeddingProviderFromModels, resolveEmbeddingRuntime, isEmbeddingEnabled, resolveEmbeddingEndpoint, } from './sqlite/index.js';
14
- export { tokenizeKeywordQuery, scoreKeywordFields, buildKeywordLikeSql, } from './sqlite/index.js';
14
+ export { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordFields, buildKeywordLikeSql, } from './sqlite/index.js';
15
+ // ── 检索排序 ──
16
+ export { DEFAULT_MIN_SIMILARITY, DEFAULT_SIMILARITY_WEIGHT, DEFAULT_MIN_KEYWORD_SCORE, qualityScore, blendRank, passesSimilarityFloor, resolveRetrievalKnobs, } from './retrieval-rank.js';
15
17
  // ── 写入去重 ──
16
18
  export { findDuplicate, normalizedProposition, charTrigramSimilarity } from './similarity.js';
17
19
  export { DEFAULT_DECAY_TYPE_PARAMS, resolveDecayParams, nextDecayFactor, isDecayDue, } from './decay-policy.js';
@@ -0,0 +1,51 @@
1
+ /**
2
+ * Memory 检索排序与相关性地板
3
+ *
4
+ * 向量路径必须先过相似度地板再按「相关性 + 质量」混合排序;
5
+ * 无命中时宁缺毋滥返回空,禁止用高 quality 无关条目填满 limit。
6
+ *
7
+ * @module harness/memory/retrieval-rank
8
+ */
9
+ import type { MemoryEntry } from './types.js';
10
+ /** 向量路径默认最低余弦相似度(低于则丢弃) */
11
+ export declare const DEFAULT_MIN_SIMILARITY = 0.35;
12
+ /** 混合排序中相似度权重(其余给 quality = importance×confidence×decay) */
13
+ export declare const DEFAULT_SIMILARITY_WEIGHT = 0.65;
14
+ /** 关键词路径最低字段命中分(strong content=3 / weak content=1.5) */
15
+ /** 默认 3:strong 单命中可通过;仅 weak 二元组(×0.5)不足以过关 */
16
+ export declare const DEFAULT_MIN_KEYWORD_SCORE = 3;
17
+ /**
18
+ * 记忆质量分(与相关性无关的「值不值得注入」)
19
+ *
20
+ * @param entry - 记忆条目
21
+ * @returns importance × confidence × decayFactor,范围约 [0,1]
22
+ */
23
+ export declare function qualityScore(entry: Pick<MemoryEntry, 'importance' | 'confidence' | 'decayFactor'>): number;
24
+ /**
25
+ * 混合排序分:相关性与质量线性加权
26
+ *
27
+ * @param similarity - 余弦相似度 [0,1]
28
+ * @param quality - qualityScore
29
+ * @param similarityWeight - 相似度权重 [0,1]
30
+ * @returns 越大越优先
31
+ */
32
+ export declare function blendRank(similarity: number, quality: number, similarityWeight?: number): number;
33
+ /**
34
+ * 是否达到向量相关性地板
35
+ *
36
+ * @param similarity - 余弦相似度
37
+ * @param minSimilarity - 地板;undefined 用默认
38
+ */
39
+ export declare function passesSimilarityFloor(similarity: number, minSimilarity?: number): boolean;
40
+ /**
41
+ * 解析检索相关性旋钮(query 覆盖 options / 默认)
42
+ *
43
+ * @param queryMin - MemoryQuery.minSimilarity
44
+ * @param queryMinKeyword - MemoryQuery.minKeywordScore
45
+ * @param optionsMin - store 选项 minSimilarity
46
+ * @param optionsMinKeyword - store 选项 minKeywordScore
47
+ */
48
+ export declare function resolveRetrievalKnobs(queryMin?: number, queryMinKeyword?: number, optionsMin?: number, optionsMinKeyword?: number): {
49
+ minSimilarity: number;
50
+ minKeywordScore: number;
51
+ };
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Memory 检索排序与相关性地板
3
+ *
4
+ * 向量路径必须先过相似度地板再按「相关性 + 质量」混合排序;
5
+ * 无命中时宁缺毋滥返回空,禁止用高 quality 无关条目填满 limit。
6
+ *
7
+ * @module harness/memory/retrieval-rank
8
+ */
9
+ /** 向量路径默认最低余弦相似度(低于则丢弃) */
10
+ export const DEFAULT_MIN_SIMILARITY = 0.35;
11
+ /** 混合排序中相似度权重(其余给 quality = importance×confidence×decay) */
12
+ export const DEFAULT_SIMILARITY_WEIGHT = 0.65;
13
+ /** 关键词路径最低字段命中分(strong content=3 / weak content=1.5) */
14
+ /** 默认 3:strong 单命中可通过;仅 weak 二元组(×0.5)不足以过关 */
15
+ export const DEFAULT_MIN_KEYWORD_SCORE = 3;
16
+ /**
17
+ * 记忆质量分(与相关性无关的「值不值得注入」)
18
+ *
19
+ * @param entry - 记忆条目
20
+ * @returns importance × confidence × decayFactor,范围约 [0,1]
21
+ */
22
+ export function qualityScore(entry) {
23
+ return entry.importance * entry.confidence * entry.decayFactor;
24
+ }
25
+ /**
26
+ * 混合排序分:相关性与质量线性加权
27
+ *
28
+ * @param similarity - 余弦相似度 [0,1]
29
+ * @param quality - qualityScore
30
+ * @param similarityWeight - 相似度权重 [0,1]
31
+ * @returns 越大越优先
32
+ */
33
+ export function blendRank(similarity, quality, similarityWeight = DEFAULT_SIMILARITY_WEIGHT) {
34
+ const w = Math.min(1, Math.max(0, similarityWeight));
35
+ return w * similarity + (1 - w) * quality;
36
+ }
37
+ /**
38
+ * 是否达到向量相关性地板
39
+ *
40
+ * @param similarity - 余弦相似度
41
+ * @param minSimilarity - 地板;undefined 用默认
42
+ */
43
+ export function passesSimilarityFloor(similarity, minSimilarity) {
44
+ const floor = minSimilarity ?? DEFAULT_MIN_SIMILARITY;
45
+ return similarity >= floor;
46
+ }
47
+ /**
48
+ * 解析检索相关性旋钮(query 覆盖 options / 默认)
49
+ *
50
+ * @param queryMin - MemoryQuery.minSimilarity
51
+ * @param queryMinKeyword - MemoryQuery.minKeywordScore
52
+ * @param optionsMin - store 选项 minSimilarity
53
+ * @param optionsMinKeyword - store 选项 minKeywordScore
54
+ */
55
+ export function resolveRetrievalKnobs(queryMin, queryMinKeyword, optionsMin, optionsMinKeyword) {
56
+ return {
57
+ minSimilarity: queryMin ?? optionsMin ?? DEFAULT_MIN_SIMILARITY,
58
+ minKeywordScore: queryMinKeyword ?? optionsMinKeyword ?? DEFAULT_MIN_KEYWORD_SCORE,
59
+ };
60
+ }
@@ -11,8 +11,8 @@ export { createEmbeddingProviderFromModels, resolveEmbeddingRuntime, isEmbedding
11
11
  export type { ResolvedEmbeddingRuntime } from './embedding-from-models.js';
12
12
  export { cosineSimilarity, cosineDistance, searchTopK, parseEmbedding, serializeEmbedding } from './vector-search.js';
13
13
  export type { VectorSearchResult } from './vector-search.js';
14
- export { tokenizeKeywordQuery, scoreKeywordFields, buildKeywordLikeSql, } from './keyword-search.js';
15
- export type { KeywordFields } from './keyword-search.js';
14
+ export { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordFields, buildKeywordLikeSql, } from './keyword-search.js';
15
+ export type { KeywordFields, KeywordTokens } from './keyword-search.js';
16
16
  export { tryLoadSqliteVec, ensureMemoryVecTable, upsertMemoryVector, deleteMemoryVector, searchMemoryVectors, MEMORY_VEC_TABLE, } from './sqlite-vec.js';
17
17
  export { SqliteMemoryStore } from './memory-store.js';
18
18
  export type { SqliteMemoryStoreOptions } from './memory-store.js';
@@ -7,7 +7,7 @@ export { AgentDatabase } from './agent-db.js';
7
7
  export { createEmbeddingProvider } from './embedding.js';
8
8
  export { createEmbeddingProviderFromModels, resolveEmbeddingRuntime, isEmbeddingEnabled, resolveEmbeddingEndpoint, } from './embedding-from-models.js';
9
9
  export { cosineSimilarity, cosineDistance, searchTopK, parseEmbedding, serializeEmbedding } from './vector-search.js';
10
- export { tokenizeKeywordQuery, scoreKeywordFields, buildKeywordLikeSql, } from './keyword-search.js';
10
+ export { tokenizeKeywordQuery, tokenizeKeywordDetail, scoreKeywordFields, buildKeywordLikeSql, } from './keyword-search.js';
11
11
  export { tryLoadSqliteVec, ensureMemoryVecTable, upsertMemoryVector, deleteMemoryVector, searchMemoryVectors, MEMORY_VEC_TABLE, } from './sqlite-vec.js';
12
12
  export { SqliteMemoryStore } from './memory-store.js';
13
13
  export { SqliteWisdomStore } from './wisdom-store.js';
@@ -15,16 +15,34 @@ export interface KeywordFields {
15
15
  /**
16
16
  * 查询分词:空白/标点切分;连续 CJK 片段额外产出二元组。
17
17
  *
18
+ * 丢弃长度 &lt; 2 的 token:单字母/单字 `includes` 会命中任意子串(`is`⊂`this`)。
19
+ * 派生二元组(长 CJK 滑窗)记入 weak:单独命中不足以过相关性地板。
20
+ *
18
21
  * @param text - 原始查询
19
- * @returns 去重后的检索词(小写)
22
+ * @returns 去重后的检索词(小写);与 {@link tokenizeKeywordDetail}.all 一致
20
23
  */
21
24
  export declare function tokenizeKeywordQuery(text: string): string[];
25
+ /** 分词明细:strong=整词/整段;weak=长 CJK 滑窗二元组 */
26
+ export interface KeywordTokens {
27
+ all: string[];
28
+ strong: string[];
29
+ weak: string[];
30
+ }
31
+ /**
32
+ * 分词并区分强弱 token。
33
+ *
34
+ * @param text - 原始查询
35
+ */
36
+ export declare function tokenizeKeywordDetail(text: string): KeywordTokens;
22
37
  /**
23
38
  * 单条记忆的关键词命中得分。
24
39
  *
40
+ * 字段权重:content > future_use/tags > evidence/anchors。
41
+ * weak token(CJK 滑窗二元组)命中权重减半,避免「什么/好处」类弱重叠单独过关。
42
+ *
25
43
  * @returns 0 表示无命中;字段权重相加
26
44
  */
27
- export declare function scoreKeywordFields(fields: KeywordFields, tokens: string[]): number;
45
+ export declare function scoreKeywordFields(fields: KeywordFields, tokens: string[] | KeywordTokens): number;
28
46
  /**
29
47
  * 构造 SQLite 多字段 LIKE 条件(OR 各 token × 各字段)。
30
48
  *
@@ -11,29 +11,45 @@ function isCjk(ch) {
11
11
  /**
12
12
  * 查询分词:空白/标点切分;连续 CJK 片段额外产出二元组。
13
13
  *
14
+ * 丢弃长度 &lt; 2 的 token:单字母/单字 `includes` 会命中任意子串(`is`⊂`this`)。
15
+ * 派生二元组(长 CJK 滑窗)记入 weak:单独命中不足以过相关性地板。
16
+ *
14
17
  * @param text - 原始查询
15
- * @returns 去重后的检索词(小写)
18
+ * @returns 去重后的检索词(小写);与 {@link tokenizeKeywordDetail}.all 一致
16
19
  */
17
20
  export function tokenizeKeywordQuery(text) {
21
+ return tokenizeKeywordDetail(text).all;
22
+ }
23
+ /**
24
+ * 分词并区分强弱 token。
25
+ *
26
+ * @param text - 原始查询
27
+ */
28
+ export function tokenizeKeywordDetail(text) {
18
29
  const raw = (text ?? '').trim().toLowerCase();
19
30
  if (!raw)
20
- return [];
21
- const tokens = new Set();
31
+ return { all: [], strong: [], weak: [] };
32
+ const strong = new Set();
33
+ const weak = new Set();
22
34
  const parts = raw.split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
23
35
  for (const part of parts) {
24
- if (part.length >= 1)
25
- tokens.add(part);
36
+ if (part.length >= 2)
37
+ strong.add(part);
26
38
  if (!isCjk(part[0] ?? ''))
27
39
  continue;
28
- // CJK 连续段:按段再切,生成二元组(保留整段词)
29
40
  let run = '';
30
41
  const flush = () => {
31
42
  if (!run)
32
43
  return;
33
44
  if (run.length >= 2)
34
- tokens.add(run);
35
- for (let i = 0; i + 1 < run.length; i++) {
36
- tokens.add(run.slice(i, i + 2));
45
+ strong.add(run);
46
+ // 仅长段的滑窗二元组算 weak(整段本身已进 strong)
47
+ if (run.length > 2) {
48
+ for (let i = 0; i + 1 < run.length; i++) {
49
+ const bi = run.slice(i, i + 2);
50
+ if (!strong.has(bi))
51
+ weak.add(bi);
52
+ }
37
53
  }
38
54
  run = '';
39
55
  };
@@ -45,7 +61,8 @@ export function tokenizeKeywordQuery(text) {
45
61
  }
46
62
  flush();
47
63
  }
48
- return [...tokens].filter((t) => t.length > 0);
64
+ const all = [...new Set([...strong, ...weak])].filter((t) => t.length >= 2);
65
+ return { all, strong: [...strong].filter((t) => t.length >= 2), weak: [...weak] };
49
66
  }
50
67
  function asText(v) {
51
68
  if (v == null)
@@ -57,30 +74,36 @@ function asText(v) {
57
74
  /**
58
75
  * 单条记忆的关键词命中得分。
59
76
  *
77
+ * 字段权重:content > future_use/tags > evidence/anchors。
78
+ * weak token(CJK 滑窗二元组)命中权重减半,避免「什么/好处」类弱重叠单独过关。
79
+ *
60
80
  * @returns 0 表示无命中;字段权重相加
61
81
  */
62
82
  export function scoreKeywordFields(fields, tokens) {
63
- if (tokens.length === 0)
83
+ const list = Array.isArray(tokens) ? tokens : tokens.all;
84
+ if (list.length === 0)
64
85
  return 0;
86
+ const weakSet = new Set(Array.isArray(tokens) ? [] : tokens.weak);
65
87
  const content = (fields.content ?? '').toLowerCase();
66
88
  const futureUse = (fields.futureUse ?? '').toLowerCase();
67
89
  const tags = asText(fields.tags).toLowerCase();
68
90
  const anchors = asText(fields.anchors).toLowerCase();
69
91
  const evidence = (fields.evidence ?? '').toLowerCase();
70
92
  let score = 0;
71
- for (const token of tokens) {
93
+ for (const token of list) {
72
94
  if (!token)
73
95
  continue;
96
+ const w = weakSet.has(token) ? 0.5 : 1;
74
97
  if (content.includes(token))
75
- score += 3;
98
+ score += 3 * w;
76
99
  if (futureUse.includes(token))
77
- score += 2;
100
+ score += 2 * w;
78
101
  if (tags.includes(token))
79
- score += 2;
102
+ score += 2 * w;
80
103
  if (anchors.includes(token))
81
- score += 1;
104
+ score += 1 * w;
82
105
  if (evidence.includes(token))
83
- score += 1;
106
+ score += 1 * w;
84
107
  }
85
108
  return score;
86
109
  }
@@ -18,6 +18,12 @@ export interface SqliteMemoryStoreOptions {
18
18
  candidateCap?: number;
19
19
  /** 向量引擎;auto=优先 sqlite-vec(需 db 已 load 扩展),否则 JS */
20
20
  vectorEngine?: VectorEngineChoice;
21
+ /** 向量路径默认相关性地板(query.minSimilarity 可覆盖) */
22
+ minSimilarity?: number;
23
+ /** 混合排序相似度权重;其余为 quality */
24
+ similarityWeight?: number;
25
+ /** 关键词路径默认最低命中分 */
26
+ minKeywordScore?: number;
21
27
  }
22
28
  export declare class SqliteMemoryStore implements MemoryStore {
23
29
  readonly name = "sqlite";
@@ -25,6 +31,9 @@ export declare class SqliteMemoryStore implements MemoryStore {
25
31
  private embedding;
26
32
  private readonly candidateCap;
27
33
  private readonly vectorEngine;
34
+ private readonly defaultMinSimilarity;
35
+ private readonly similarityWeight;
36
+ private readonly defaultMinKeywordScore;
28
37
  private vecReady;
29
38
  private embeddingBackfillScheduled;
30
39
  constructor(db: AgentDatabase, options?: SqliteMemoryStoreOptions);
@@ -36,18 +45,23 @@ export declare class SqliteMemoryStore implements MemoryStore {
36
45
  private embedText;
37
46
  retrieve(query: MemoryQuery): Promise<MemoryEntry[]>;
38
47
  private visibilityWhere;
39
- /** sqlite-vec KNN 检索(需先 embed 查询文本) */
48
+ /** sqlite-vec KNN 检索(需先 embed 查询文本);过相关性地板后按混合分排序 */
40
49
  private vecRetrieveAsync;
41
50
  /**
42
- * 混合检索:SQL 预筛候选(避免全表载入 embedding)→ JS 余弦 topK
43
- * candidateCap 默认 500,可通过 SqliteMemoryStoreOptions 覆盖。
51
+ * 混合检索:SQL 预筛候选 → JS 余弦 topK → 相关性地板。
52
+ * 命中不足 limit 时**不**用无关条目填充。
53
+ * @param options.allowKeywordFallback - 仅 embed 硬失败场景允许;默认 false(宁缺)
44
54
  */
45
55
  private hybridRetrieve;
46
56
  private structuredRetrieve;
57
+ /** 单条结构化过滤(向量路径逐条判断) */
58
+ private passesApplyFilters;
47
59
  /** 与 InMemory 对齐:tags 交集过滤 */
48
60
  private filterByTags;
49
61
  private applyFilters;
50
- private sortAndLimit;
62
+ /** 向量命中:similarity 参与排序(禁止仅按 quality 顶掉贴题条目) */
63
+ private sortLimitScored;
64
+ private limitAndUpdate;
51
65
  /**
52
66
  * 回填缺失 embedding(历史数据 / 关键词时代写入)。
53
67
  *