@octopi-agent/engine 0.58.0 → 0.70.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (226) hide show
  1. package/dist/config-schema/context.d.ts +1 -1
  2. package/dist/config-schema/context.js +1 -1
  3. package/dist/config-schema/engine.d.ts +122 -1
  4. package/dist/config-schema/engine.js +4 -1
  5. package/dist/config-schema/substrate.d.ts +123 -0
  6. package/dist/config-schema/substrate.js +161 -0
  7. package/dist/config.d.ts +101 -1
  8. package/dist/harness/agent/builder.d.ts +15 -1
  9. package/dist/harness/agent/builder.js +41 -4
  10. package/dist/harness/agent/config-bridge.js +17 -4
  11. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.d.ts +1 -1
  12. package/dist/harness/{context/capabilities → capabilities}/document/backends/docx.js +1 -1
  13. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.d.ts +1 -1
  14. package/dist/harness/{context/capabilities → capabilities}/document/backends/office-parser.js +1 -1
  15. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.d.ts +1 -1
  16. package/dist/harness/{context/capabilities → capabilities}/document/backends/pdf.js +13 -3
  17. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.d.ts +1 -1
  18. package/dist/harness/{context/capabilities → capabilities}/document/backends/plain.js +13 -4
  19. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.d.ts +1 -1
  20. package/dist/harness/{context/capabilities → capabilities}/document/backends/sheet.js +25 -5
  21. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.d.ts +1 -1
  22. package/dist/harness/{context/capabilities → capabilities}/document/backends/xmind.js +1 -1
  23. package/dist/harness/{context/capabilities → capabilities}/document/errors.d.ts +1 -1
  24. package/dist/harness/{context/capabilities → capabilities}/document/errors.js +1 -1
  25. package/dist/harness/capabilities/document/factory.d.ts +33 -0
  26. package/dist/harness/capabilities/document/factory.js +37 -0
  27. package/dist/harness/{context/capabilities → capabilities}/document/format.d.ts +1 -1
  28. package/dist/harness/{context/capabilities → capabilities}/document/format.js +1 -1
  29. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.d.ts +1 -1
  30. package/dist/harness/{context/capabilities → capabilities}/document/html-to-md.js +1 -1
  31. package/dist/harness/{context/capabilities → capabilities}/document/index.d.ts +3 -1
  32. package/dist/harness/{context/capabilities → capabilities}/document/index.js +2 -1
  33. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.d.ts +3 -1
  34. package/dist/harness/{context/capabilities → capabilities}/document/legacy/factory.js +3 -1
  35. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.d.ts +3 -1
  36. package/dist/harness/{context/capabilities → capabilities}/document/legacy/soffice.js +44 -1
  37. package/dist/harness/{context/capabilities → capabilities}/document/port.d.ts +1 -1
  38. package/dist/harness/{context/capabilities → capabilities}/document/port.js +4 -4
  39. package/dist/harness/{context/capabilities → capabilities}/document/types.d.ts +11 -1
  40. package/dist/harness/{context/capabilities → capabilities}/document/types.js +1 -1
  41. package/dist/harness/capabilities/document/worker-port.d.ts +13 -0
  42. package/dist/harness/capabilities/document/worker-port.js +78 -0
  43. package/dist/harness/context/capabilities/index.d.ts +2 -2
  44. package/dist/harness/context/capabilities/index.js +2 -2
  45. package/dist/harness/context/capabilities/summary/gate.js +2 -2
  46. package/dist/harness/context/capabilities/summary/policies.js +5 -5
  47. package/dist/harness/context/capabilities/summary/registry.js +29 -24
  48. package/dist/harness/context/layer-health.js +3 -2
  49. package/dist/harness/context/layers.d.ts +27 -13
  50. package/dist/harness/context/layers.js +76 -66
  51. package/dist/harness/context/system-prompt-assembler.d.ts +3 -0
  52. package/dist/harness/context/system-prompt-assembler.js +11 -1
  53. package/dist/harness/extension/plugin-ecosystem/client-tools/index.d.ts +4 -0
  54. package/dist/harness/extension/plugin-ecosystem/client-tools/index.js +3 -0
  55. package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.d.ts +56 -0
  56. package/dist/harness/extension/plugin-ecosystem/client-tools/invoke.js +281 -0
  57. package/dist/harness/extension/plugin-ecosystem/client-tools/registry.d.ts +98 -0
  58. package/dist/harness/extension/plugin-ecosystem/client-tools/registry.js +250 -0
  59. package/dist/harness/extension/plugin-ecosystem/client-tools/stream.d.ts +124 -0
  60. package/dist/harness/extension/plugin-ecosystem/client-tools/stream.js +372 -0
  61. package/dist/harness/extension/plugin-ecosystem/client-tools/types.d.ts +109 -0
  62. package/dist/harness/extension/plugin-ecosystem/client-tools/types.js +8 -0
  63. package/dist/harness/extension/plugin-ecosystem/index.d.ts +1 -0
  64. package/dist/harness/extension/plugin-ecosystem/index.js +1 -0
  65. package/dist/harness/extension/plugin-ecosystem/tools/builtin.js +1 -1
  66. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.d.ts +10 -0
  67. package/dist/harness/extension/plugin-ecosystem/tools/knowledge.js +102 -5
  68. package/dist/harness/extension/plugin-ecosystem/tools/memory.d.ts +11 -1
  69. package/dist/harness/extension/plugin-ecosystem/tools/memory.js +14 -0
  70. package/dist/harness/index.d.ts +8 -5
  71. package/dist/harness/index.js +4 -2
  72. package/dist/harness/knowledge/adapters.d.ts +8 -5
  73. package/dist/harness/knowledge/adapters.js +53 -13
  74. package/dist/harness/knowledge/client.d.ts +197 -0
  75. package/dist/harness/knowledge/client.js +237 -0
  76. package/dist/harness/knowledge/connector-fetcher.d.ts +2 -2
  77. package/dist/harness/knowledge/connectors.d.ts +3 -3
  78. package/dist/harness/knowledge/connectors.js +14 -8
  79. package/dist/harness/knowledge/db.d.ts +40 -5
  80. package/dist/harness/knowledge/db.js +441 -113
  81. package/dist/harness/knowledge/document-extract-worker.js +55 -12
  82. package/dist/harness/knowledge/embed-runner.d.ts +53 -0
  83. package/dist/harness/knowledge/embed-runner.js +171 -0
  84. package/dist/harness/knowledge/engine-thread.d.ts +1 -0
  85. package/dist/harness/knowledge/engine-thread.js +302 -0
  86. package/dist/harness/knowledge/extract-document.d.ts +21 -5
  87. package/dist/harness/knowledge/extract-document.js +32 -8
  88. package/dist/harness/knowledge/fetchers.d.ts +14 -3
  89. package/dist/harness/knowledge/fetchers.js +52 -16
  90. package/dist/harness/knowledge/file-identity.d.ts +53 -0
  91. package/dist/harness/knowledge/file-identity.js +172 -0
  92. package/dist/harness/knowledge/file-index-store.d.ts +50 -0
  93. package/dist/harness/knowledge/file-index-store.js +132 -0
  94. package/dist/harness/knowledge/file-limits.d.ts +86 -0
  95. package/dist/harness/knowledge/file-limits.js +175 -0
  96. package/dist/harness/knowledge/fts.d.ts +60 -0
  97. package/dist/harness/knowledge/fts.js +218 -0
  98. package/dist/harness/knowledge/grounding.d.ts +3 -2
  99. package/dist/harness/knowledge/http-app.d.ts +50 -0
  100. package/dist/harness/knowledge/http-app.js +652 -0
  101. package/dist/harness/knowledge/http-bridge.d.ts +49 -0
  102. package/dist/harness/knowledge/http-bridge.js +17 -0
  103. package/dist/harness/knowledge/index-store.d.ts +158 -93
  104. package/dist/harness/knowledge/index-store.js +982 -441
  105. package/dist/harness/knowledge/index.d.ts +45 -5
  106. package/dist/harness/knowledge/index.js +23 -2
  107. package/dist/harness/knowledge/ingest.d.ts +262 -87
  108. package/dist/harness/knowledge/ingest.js +1072 -480
  109. package/dist/harness/knowledge/job-control-state.d.ts +29 -0
  110. package/dist/harness/knowledge/job-control-state.js +44 -0
  111. package/dist/harness/knowledge/job-control.d.ts +35 -0
  112. package/dist/harness/knowledge/job-control.js +86 -0
  113. package/dist/harness/knowledge/job-queue.d.ts +62 -0
  114. package/dist/harness/knowledge/job-queue.js +173 -0
  115. package/dist/harness/knowledge/knowledge-serve-child.d.ts +1 -0
  116. package/dist/harness/knowledge/knowledge-serve-child.js +100 -0
  117. package/dist/harness/knowledge/local-stack.d.ts +27 -0
  118. package/dist/harness/knowledge/local-stack.js +63 -0
  119. package/dist/harness/knowledge/membership-store.d.ts +65 -0
  120. package/dist/harness/knowledge/membership-store.js +198 -0
  121. package/dist/harness/knowledge/network-guard.d.ts +11 -2
  122. package/dist/harness/knowledge/network-guard.js +130 -85
  123. package/dist/harness/knowledge/parse-text-in-worker.d.ts +32 -0
  124. package/dist/harness/knowledge/parse-text-in-worker.js +158 -0
  125. package/dist/harness/knowledge/query-service.d.ts +272 -0
  126. package/dist/harness/knowledge/query-service.js +326 -0
  127. package/dist/harness/knowledge/query-worker-client.d.ts +96 -0
  128. package/dist/harness/knowledge/query-worker-client.js +207 -0
  129. package/dist/harness/knowledge/query-worker.d.ts +1 -0
  130. package/dist/harness/knowledge/query-worker.js +186 -0
  131. package/dist/harness/knowledge/read-http.d.ts +35 -0
  132. package/dist/harness/knowledge/read-http.js +373 -0
  133. package/dist/harness/knowledge/retriever.d.ts +22 -5
  134. package/dist/harness/knowledge/retriever.js +66 -35
  135. package/dist/harness/knowledge/secret-scan.d.ts +13 -2
  136. package/dist/harness/knowledge/secret-scan.js +25 -2
  137. package/dist/harness/knowledge/serve.d.ts +48 -0
  138. package/dist/harness/knowledge/serve.js +336 -0
  139. package/dist/harness/knowledge/source-store.d.ts +97 -19
  140. package/dist/harness/knowledge/source-store.js +166 -94
  141. package/dist/harness/knowledge/start-service-process.d.ts +49 -0
  142. package/dist/harness/knowledge/start-service-process.js +158 -0
  143. package/dist/harness/knowledge/text-parse-worker.d.ts +1 -0
  144. package/dist/harness/knowledge/text-parse-worker.js +54 -0
  145. package/dist/harness/knowledge/types.d.ts +1 -1
  146. package/dist/harness/knowledge/types.js +1 -1
  147. package/dist/harness/knowledge/vector-ann.d.ts +42 -0
  148. package/dist/harness/knowledge/vector-ann.js +68 -0
  149. package/dist/harness/knowledge/writer-lock.d.ts +13 -0
  150. package/dist/harness/knowledge/writer-lock.js +71 -0
  151. package/dist/harness/knowledge/writer-service.d.ts +194 -0
  152. package/dist/harness/knowledge/writer-service.js +355 -0
  153. package/dist/harness/knowledge/writer-worker-client.d.ts +102 -0
  154. package/dist/harness/knowledge/writer-worker-client.js +381 -0
  155. package/dist/harness/knowledge/writer-worker.d.ts +1 -0
  156. package/dist/harness/knowledge/writer-worker.js +189 -0
  157. package/dist/harness/memory/cognition-decay.d.ts +36 -0
  158. package/dist/harness/memory/cognition-decay.js +55 -0
  159. package/dist/harness/memory/cognition-gates.d.ts +60 -0
  160. package/dist/harness/memory/cognition-gates.js +191 -0
  161. package/dist/harness/memory/cognition-trigger.d.ts +30 -0
  162. package/dist/harness/memory/cognition-trigger.js +41 -0
  163. package/dist/harness/memory/cognition-types.d.ts +1 -1
  164. package/dist/harness/memory/cognition.d.ts +40 -8
  165. package/dist/harness/memory/cognition.js +432 -79
  166. package/dist/harness/memory/conceptualizer.d.ts +66 -0
  167. package/dist/harness/memory/conceptualizer.js +246 -0
  168. package/dist/harness/memory/index.d.ts +16 -2
  169. package/dist/harness/memory/index.js +11 -1
  170. package/dist/harness/memory/sqlite/agent-db.d.ts +9 -0
  171. package/dist/harness/memory/sqlite/agent-db.js +118 -15
  172. package/dist/harness/memory/sqlite/cognition-store.d.ts +55 -16
  173. package/dist/harness/memory/sqlite/cognition-store.js +591 -144
  174. package/dist/harness/memory/sqlite/embedding.d.ts +5 -2
  175. package/dist/harness/memory/sqlite/embedding.js +109 -16
  176. package/dist/harness/memory/sqlite/keyword-search.d.ts +21 -0
  177. package/dist/harness/memory/sqlite/keyword-search.js +64 -0
  178. package/dist/harness/memory/sqlite/wisdom-store.d.ts +33 -14
  179. package/dist/harness/memory/sqlite/wisdom-store.js +257 -44
  180. package/dist/harness/memory/types.d.ts +331 -16
  181. package/dist/harness/memory/types.js +49 -0
  182. package/dist/harness/memory/wisdom-formation.d.ts +101 -0
  183. package/dist/harness/memory/wisdom-formation.js +304 -0
  184. package/dist/harness/memory/wisdom-gates.d.ts +50 -0
  185. package/dist/harness/memory/wisdom-gates.js +204 -0
  186. package/dist/harness/memory/wisdom-policy.d.ts +107 -0
  187. package/dist/harness/memory/wisdom-policy.js +220 -0
  188. package/dist/harness/memory/wisdom-types.d.ts +3 -2
  189. package/dist/harness/memory/wisdom-types.js +2 -2
  190. package/dist/harness/memory/wisdom.d.ts +27 -0
  191. package/dist/harness/memory/wisdom.js +181 -0
  192. package/dist/harness/run/agent/agent.d.ts +9 -0
  193. package/dist/harness/run/agent/agent.js +19 -0
  194. package/dist/harness/run/reliability/run-agent.js +6 -2
  195. package/dist/harness/run/runner.js +2 -1
  196. package/dist/harness/session/attachments/service.d.ts +1 -1
  197. package/dist/harness/session/attachments/service.js +1 -1
  198. package/dist/harness/session/index.d.ts +1 -0
  199. package/dist/harness/session/index.js +1 -0
  200. package/dist/harness/session/title.d.ts +86 -0
  201. package/dist/harness/session/title.js +268 -0
  202. package/dist/harness/shared/events/agent-event-map.d.ts +8 -0
  203. package/dist/harness/shared/events/agent-event-map.js +1 -0
  204. package/dist/index.d.ts +7 -3
  205. package/dist/index.js +3 -1
  206. package/dist/subsystems/cognition-steward/conceptualize/SUBSYSTEM.md +43 -0
  207. package/dist/subsystems/cognition-steward/conceptualize/config.yaml +50 -0
  208. package/dist/subsystems/cognition-steward/conceptualize/handler.d.ts +9 -0
  209. package/dist/subsystems/cognition-steward/conceptualize/handler.js +125 -0
  210. package/dist/subsystems/cognition-steward/consolidate/SUBSYSTEM.md +6 -0
  211. package/dist/subsystems/cognition-steward/consolidate/config.yaml +54 -0
  212. package/dist/subsystems/cognition-steward/consolidate/handler.d.ts +12 -0
  213. package/dist/subsystems/cognition-steward/consolidate/handler.js +93 -0
  214. package/dist/subsystems/wisdom-steward/evaluate/config.yaml +50 -0
  215. package/dist/subsystems/wisdom-steward/evaluate/handler.d.ts +18 -0
  216. package/dist/subsystems/wisdom-steward/evaluate/handler.js +91 -0
  217. package/dist/subsystems/wisdom-steward/formation/SUBSYSTEM.md +42 -0
  218. package/dist/subsystems/wisdom-steward/formation/config.yaml +54 -0
  219. package/dist/subsystems/wisdom-steward/formation/handler.d.ts +18 -0
  220. package/dist/subsystems/wisdom-steward/formation/handler.js +212 -0
  221. package/dist/subsystems/wisdom-steward/govern/config.yaml +55 -0
  222. package/dist/subsystems/wisdom-steward/govern/handler.d.ts +17 -0
  223. package/dist/subsystems/wisdom-steward/govern/handler.js +68 -0
  224. package/dist/worker-path.d.ts +8 -0
  225. package/dist/worker-path.js +15 -0
  226. package/package.json +2 -2
@@ -1,6 +1,7 @@
1
1
  /**
2
2
  * KnowledgeRetriever — hybrid 检索 + auto-ground 降级(P3)
3
3
  */
4
+ import { hasEnoughQuerySignal, tokenizeKeywordDetail, } from '../memory/sqlite/keyword-search.js';
4
5
  /**
5
6
  * 检索门面:可见集 → keyword + vector → 融合
6
7
  */
@@ -20,8 +21,8 @@ export class KnowledgeRetriever {
20
21
  this.index = options.indexStore;
21
22
  this.embedding = options.embeddingProvider ?? null;
22
23
  this.keywordWeight = options.keywordWeight ?? 0.45;
23
- this.injectMinScore = options.injectMinScore ?? 0.78;
24
- this.hintMinScore = options.hintMinScore ?? 0.55;
24
+ this.injectMinScore = options.injectMinScore ?? 0.72;
25
+ this.hintMinScore = options.hintMinScore ?? 0.5;
25
26
  this.minCoverage = options.minCoverage ?? 0.5;
26
27
  this.maxChunks = options.maxChunks ?? 4;
27
28
  this.recall = options.recall ?? 'hybrid';
@@ -30,8 +31,11 @@ export class KnowledgeRetriever {
30
31
  /**
31
32
  * 可见 sourceId 列表
32
33
  */
33
- visibleSourceIds(agentId, sessionId) {
34
- return this.sources.listVisible(agentId, sessionId).map((s) => s.id);
34
+ visibleSourceIds(agentId, sessionId, identity) {
35
+ return this.sources
36
+ .listVisible(agentId, sessionId, identity)
37
+ .filter((s) => this.sources.isGatewayVisible(s, identity))
38
+ .map((s) => s.id);
35
39
  }
36
40
  /** 可见集 ∩ 请求过滤;无过滤时原样返回可见集 */
37
41
  filterSourceIds(visible, opts) {
@@ -68,27 +72,37 @@ export class KnowledgeRetriever {
68
72
  * hybrid 检索
69
73
  */
70
74
  async search(query, opts) {
71
- const visible = this.visibleSourceIds(opts.agentId, opts.sessionId);
75
+ const visible = this.visibleSourceIds(opts.agentId, opts.sessionId, {
76
+ tenantId: opts.tenantId,
77
+ gatewayId: opts.gatewayId,
78
+ });
72
79
  const sourceIds = this.filterSourceIds(visible, opts);
73
80
  const limit = opts.limit ?? 8;
74
- const keywordHits = this.hybridKeyword
75
- ? this.index.search(query, { sourceIds, limit: limit * 2 })
76
- : [];
81
+ const vectorBackend = this.index.vectorBackend();
82
+ // 百万级:无 KNN / 配置强制时走关键词腿(闸门看结果而非 flag)
83
+ const preferKeyword = this.hybridKeyword ||
84
+ vectorBackend === 'js-bucket' ||
85
+ vectorBackend === 'disabled';
77
86
  let vectorHits = [];
78
87
  let usedVector = false;
79
88
  if (this.embedding && !opts.keywordOnly) {
80
89
  try {
81
90
  const qEmb = await this.embedding.embed(query);
82
91
  vectorHits = this.index.vectorSearch(qEmb, { sourceIds, limit: limit * 2 });
83
- usedVector = true;
92
+ usedVector = vectorHits.length > 0;
84
93
  }
85
94
  catch {
86
95
  usedVector = false;
87
96
  vectorHits = [];
88
97
  }
89
98
  }
90
- const kwW = !this.hybridKeyword ? 0 : usedVector ? this.keywordWeight : 1;
91
- const vecW = usedVector ? (this.hybridKeyword ? 1 - this.keywordWeight : 1) : 0;
99
+ const forceKeyword = preferKeyword || !usedVector;
100
+ const keywordHits = forceKeyword
101
+ ? this.index.search(query, { sourceIds, limit: limit * 2 })
102
+ : [];
103
+ const useKw = forceKeyword || !usedVector;
104
+ const kwW = !useKw ? 0 : usedVector ? this.keywordWeight : 1;
105
+ const vecW = usedVector ? (useKw ? 1 - this.keywordWeight : 1) : 0;
92
106
  const fused = fuseHits(keywordHits, vectorHits, {
93
107
  keywordWeight: kwW,
94
108
  vectorWeight: vecW,
@@ -101,14 +115,16 @@ export class KnowledgeRetriever {
101
115
  vectorHits: vectorHits.length,
102
116
  coverage,
103
117
  sourceIds,
118
+ vectorBackend,
104
119
  };
105
120
  }
106
121
  /**
107
- * auto-ground 分档(高分注入 / 中分 hint / 低分忽略);coverage 低时抬地板
108
- * 受 recall 模式约束:off/hint 会压制 inject
122
+ * auto-ground 分档(高分注入 / 中分 hint / 低分忽略);indexCoverage 低时抬地板
123
+ * 受 recall 模式约束:off/hint 会压制 inject;闲聊短查询直接 none
109
124
  */
110
125
  async autoGround(query, opts) {
111
- if (this.recall === 'off') {
126
+ const recall = opts.recall ?? this.recall;
127
+ if (recall === 'off') {
112
128
  return {
113
129
  mode: 'none',
114
130
  hits: [],
@@ -117,17 +133,27 @@ export class KnowledgeRetriever {
117
133
  scoreFloor: 1,
118
134
  };
119
135
  }
136
+ if (!hasEnoughQuerySignal(tokenizeKeywordDetail(query))) {
137
+ return {
138
+ mode: 'none',
139
+ hits: [],
140
+ reason: 'query_signal_too_weak',
141
+ coverage: 1,
142
+ scoreFloor: this.injectMinScore,
143
+ };
144
+ }
120
145
  const result = await this.search(query, { ...opts, limit: Math.max(opts.limit ?? 8, this.maxChunks * 2) });
121
146
  const lowCoverage = result.coverage < this.minCoverage;
122
147
  // inject 模式更积极;hint/hybrid 维持默认地板
123
- const baseFloor = this.recall === 'inject'
148
+ const baseFloor = recall === 'inject'
124
149
  ? Math.max(0.15, this.injectMinScore - 0.15)
125
150
  : this.injectMinScore;
151
+ // indexCoverage 不足时更保守(索引完备度,非相关度)
126
152
  const scoreFloor = lowCoverage
127
- ? Math.min(0.95, baseFloor + 0.12)
153
+ ? Math.min(0.9, baseFloor + 0.1)
128
154
  : baseFloor;
129
155
  const injectable = result.hits.filter((h) => h.score >= scoreFloor).slice(0, this.maxChunks);
130
- if (injectable.length > 0 && this.recall !== 'hint') {
156
+ if (injectable.length > 0 && recall !== 'hint') {
131
157
  return {
132
158
  mode: 'inject',
133
159
  hits: injectable,
@@ -143,7 +169,7 @@ export class KnowledgeRetriever {
143
169
  mode: 'hint',
144
170
  hits: [],
145
171
  hint: `存在与本问相关的材料(${hintable.length} 处):${titles}`,
146
- reason: this.recall === 'hint' && injectable.length > 0
172
+ reason: recall === 'hint' && injectable.length > 0
147
173
  ? 'hint_suppress_inject'
148
174
  : lowCoverage
149
175
  ? 'hint_low_coverage'
@@ -181,36 +207,41 @@ export class KnowledgeRetriever {
181
207
  }
182
208
  function fuseHits(keyword, vector, weights) {
183
209
  const byId = new Map();
184
- const kMax = Math.max(...keyword.map((h) => h.score), 1);
185
210
  const useKw = weights.keywordWeight > 0;
186
211
  const useVec = weights.vectorWeight > 0;
187
212
  for (const h of keyword) {
188
213
  if (!useKw)
189
214
  continue;
190
- // 关键词:列表内相对分(0–1)
191
- const score = h.score / kMax;
192
- const prev = byId.get(h.chunkId);
193
- byId.set(h.chunkId, { ...h, score: (prev?.score ?? 0) + score });
215
+ // 关键词已是绝对覆盖分(0–1),勿再 min-max
216
+ const score = Math.max(0, Math.min(1, h.score));
217
+ const prev = byId.get(h.chunkId) ?? { hit: h, kw: null, vec: null };
218
+ byId.set(h.chunkId, { ...prev, hit: { ...prev.hit, ...h }, kw: score });
194
219
  }
195
220
  for (const h of vector) {
196
221
  if (!useVec)
197
222
  continue;
198
- // 向量:用原始 cosine(勿 min-max,否则单条命中恒为 1.0)
199
223
  const score = Math.max(0, Math.min(1, h.score));
200
- const prev = byId.get(h.chunkId);
201
- if (prev) {
202
- byId.set(h.chunkId, { ...prev, score: prev.score + score });
224
+ const prev = byId.get(h.chunkId) ?? { hit: h, kw: null, vec: null };
225
+ byId.set(h.chunkId, { ...prev, hit: { ...prev.hit, ...h }, vec: score });
226
+ }
227
+ const kwW = weights.keywordWeight;
228
+ const vecW = weights.vectorWeight;
229
+ return [...byId.values()]
230
+ .map(({ hit, kw, vec }) => {
231
+ let score;
232
+ if (kw != null && vec != null) {
233
+ // 加权平均,保持 0–1 语义;双高分一致性命中微升
234
+ score = kwW * kw + vecW * vec;
235
+ if (kw >= 0.5 && vec >= 0.5)
236
+ score = Math.min(1, score + 0.05);
237
+ }
238
+ else if (kw != null) {
239
+ score = kw;
203
240
  }
204
241
  else {
205
- byId.set(h.chunkId, { ...h, score });
242
+ score = vec ?? 0;
206
243
  }
207
- }
208
- const kwIds = new Set(keyword.map((k) => k.chunkId));
209
- const vecIds = new Set(vector.map((v) => v.chunkId));
210
- return [...byId.values()]
211
- .map((h) => {
212
- const dual = kwIds.has(h.chunkId) && vecIds.has(h.chunkId);
213
- return { ...h, score: Math.min(1, dual ? h.score * 0.85 + 0.15 : h.score) };
244
+ return { ...hit, score };
214
245
  })
215
246
  .sort((a, b) => b.score - a.score);
216
247
  }
@@ -1,7 +1,7 @@
1
1
  /**
2
- * 密钥/敏感形态扫描 — auto-describe 抽样外发前的门
2
+ * 密钥/敏感形态扫描 — 外发前的门(auto-describe / embedding)
3
3
  *
4
- * 只做形态匹配,不做语义理解。命中则拒绝外发该样本。
4
+ * 只做形态匹配,不做语义理解。
5
5
  */
6
6
  /**
7
7
  * 扫描文本中的密钥形态
@@ -9,3 +9,14 @@
9
9
  * @returns 命中的规则名列表(空 = 未发现)
10
10
  */
11
11
  export declare function scanSecretShapes(text: string): string[];
12
+ export interface RedactResult {
13
+ text: string;
14
+ hits: string[];
15
+ }
16
+ /**
17
+ * 脱敏:命中形态替换为 `[REDACTED:rule]`,供 embedding 外发前使用。
18
+ *
19
+ * @param text - 原文
20
+ * @returns 脱敏后文本与命中规则名
21
+ */
22
+ export declare function redactSecretShapes(text: string): RedactResult;
@@ -1,7 +1,7 @@
1
1
  /**
2
- * 密钥/敏感形态扫描 — auto-describe 抽样外发前的门
2
+ * 密钥/敏感形态扫描 — 外发前的门(auto-describe / embedding)
3
3
  *
4
- * 只做形态匹配,不做语义理解。命中则拒绝外发该样本。
4
+ * 只做形态匹配,不做语义理解。
5
5
  */
6
6
  const SECRET_PATTERNS = [
7
7
  { name: 'aws_access_key', re: /\bAKIA[0-9A-Z]{16}\b/ },
@@ -15,6 +15,9 @@ const SECRET_PATTERNS = [
15
15
  re: /\b(?:password|passwd|pwd|api[_-]?key|secret[_-]?key)\s*[=:]\s*(?:['"][^'"]{6,}['"]|\S{6,})/i,
16
16
  },
17
17
  ];
18
+ function withGlobal(re) {
19
+ return re.flags.includes('g') ? re : new RegExp(re.source, `${re.flags}g`);
20
+ }
18
21
  /**
19
22
  * 扫描文本中的密钥形态
20
23
  *
@@ -30,3 +33,23 @@ export function scanSecretShapes(text) {
30
33
  }
31
34
  return hits;
32
35
  }
36
+ /**
37
+ * 脱敏:命中形态替换为 `[REDACTED:rule]`,供 embedding 外发前使用。
38
+ *
39
+ * @param text - 原文
40
+ * @returns 脱敏后文本与命中规则名
41
+ */
42
+ export function redactSecretShapes(text) {
43
+ if (!text)
44
+ return { text: '', hits: [] };
45
+ let out = text;
46
+ const hits = [];
47
+ for (const { name, re } of SECRET_PATTERNS) {
48
+ const g = withGlobal(re);
49
+ if (g.test(out)) {
50
+ hits.push(name);
51
+ out = out.replace(g, `[REDACTED:${name}]`);
52
+ }
53
+ }
54
+ return { text: out, hits };
55
+ }
@@ -0,0 +1,48 @@
1
+ /**
2
+ * serve 入口线程模型(v0.64+)
3
+ *
4
+ * - 主线程:listen · /health · token 鉴权 · **纯读直达 Meta/Search Worker** · SSE 泵出
5
+ * - Engine/API Worker:写路由编排 + SSE;**不得**打开可写 knowledge.db
6
+ * - Writer Worker:knowledge.db 唯一写者 + ingest(允许阻塞)
7
+ * - Meta Worker:只读列表/控制面(projects / sources / jobs / ready)
8
+ * - Search Worker:只读检索(search / catalog / files / chunks)
9
+ * - 嵌套 Parse Worker:抽取 / 切块 / FTS token
10
+ *
11
+ * 纯读 **不得** postMessage 进 Engine;变更 **不得** 在 API 线程直接写库。
12
+ */
13
+ import { type Server } from 'node:http';
14
+ import type { KnowledgeServiceToken } from './http-app.js';
15
+ export interface KnowledgeServeOptions {
16
+ dbPath: string;
17
+ host?: string;
18
+ port?: number;
19
+ tokens: KnowledgeServiceToken[];
20
+ autoRegisterPrincipals?: boolean;
21
+ documentConfig?: import('../capabilities/document/factory.js').DocumentCapabilityConfig | null;
22
+ /** 测试 stub(闭包无法进 Worker);生产用 embeddingModels */
23
+ testEmbeddingStub?: boolean;
24
+ embed?: {
25
+ enabled?: boolean;
26
+ embedBatch?: number;
27
+ embedMinIntervalMs?: number;
28
+ embedConcurrency?: number;
29
+ embedSecretPolicy?: 'allow' | 'redact' | 'skip';
30
+ } | null;
31
+ sqliteVecExtensionPath?: string;
32
+ embeddingModels?: {
33
+ providers?: Record<string, unknown>;
34
+ embedding?: unknown;
35
+ } | null;
36
+ }
37
+ export interface KnowledgeServeHandle {
38
+ server: Server;
39
+ port: number;
40
+ close(): Promise<void>;
41
+ }
42
+ /**
43
+ * 启动 Knowledge HTTP 服务(唯一写者在 Writer Worker;纯读直达 Meta/Search)。
44
+ *
45
+ * @param opts - db 路径 / 监听 / token 表 / embedding
46
+ * @returns 可关闭的 listen handle
47
+ */
48
+ export declare function startKnowledgeService(opts: KnowledgeServeOptions): Promise<KnowledgeServeHandle>;
@@ -0,0 +1,336 @@
1
+ /**
2
+ * serve 入口线程模型(v0.64+)
3
+ *
4
+ * - 主线程:listen · /health · token 鉴权 · **纯读直达 Meta/Search Worker** · SSE 泵出
5
+ * - Engine/API Worker:写路由编排 + SSE;**不得**打开可写 knowledge.db
6
+ * - Writer Worker:knowledge.db 唯一写者 + ingest(允许阻塞)
7
+ * - Meta Worker:只读列表/控制面(projects / sources / jobs / ready)
8
+ * - Search Worker:只读检索(search / catalog / files / chunks)
9
+ * - 嵌套 Parse Worker:抽取 / 切块 / FTS token
10
+ *
11
+ * 纯读 **不得** postMessage 进 Engine;变更 **不得** 在 API 线程直接写库。
12
+ */
13
+ import { createServer } from 'node:http';
14
+ import { Worker } from 'node:worker_threads';
15
+ import { resolveWorkerUrl } from '../../worker-path.js';
16
+ import { matchKnowledgeToken, } from './http-bridge.js';
17
+ import { authenticateRead, handleKnowledgeReadHttp, isPureReadRoute, } from './read-http.js';
18
+ async function readBody(req) {
19
+ if (req.method === 'GET' || req.method === 'HEAD')
20
+ return undefined;
21
+ const chunks = [];
22
+ for await (const c of req)
23
+ chunks.push(c);
24
+ if (chunks.length === 0)
25
+ return undefined;
26
+ return Buffer.concat(chunks).toString('utf8');
27
+ }
28
+ function toBridgeHeaders(headers) {
29
+ return { ...headers };
30
+ }
31
+ async function startQueryWorker(opts) {
32
+ const { createKnowledgeQueryService } = await import('./query-service.js');
33
+ return createKnowledgeQueryService({
34
+ dbPath: opts.dbPath,
35
+ mode: 'worker',
36
+ role: opts.role,
37
+ embeddingModels: opts.embeddingModels ?? null,
38
+ ...(opts.sqliteVecExtensionPath
39
+ ? { sqliteVecExtensionPath: opts.sqliteVecExtensionPath }
40
+ : {}),
41
+ });
42
+ }
43
+ /**
44
+ * 启动 Knowledge HTTP 服务(唯一写者在 Writer Worker;纯读直达 Meta/Search)。
45
+ *
46
+ * @param opts - db 路径 / 监听 / token 表 / embedding
47
+ * @returns 可关闭的 listen handle
48
+ */
49
+ export async function startKnowledgeService(opts) {
50
+ const useSplitQuery = opts.dbPath !== ':memory:';
51
+ // Engine 先起:建库/建表后 Query Worker 才能只读打开(缺文件时 readOnly 会炸)
52
+ const entry = fileURLToWorker('./engine-thread.js');
53
+ const worker = new Worker(entry, {
54
+ workerData: {
55
+ dbPath: opts.dbPath,
56
+ tokens: opts.tokens,
57
+ autoRegisterPrincipals: opts.autoRegisterPrincipals ?? true,
58
+ documentConfig: opts.documentConfig ?? null,
59
+ sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
60
+ embeddingModels: opts.embeddingModels ?? null,
61
+ embed: opts.embed ?? null,
62
+ testEmbeddingStub: opts.testEmbeddingStub ?? false,
63
+ // 文件库:纯读在主线程 Meta/Search;Engine 只做写编排 + 残留短读
64
+ skipQueryWorker: useSplitQuery,
65
+ },
66
+ // 嵌套解析 worker 需要相对 dist 加载
67
+ env: process.env,
68
+ });
69
+ await new Promise((resolve, reject) => {
70
+ const timer = setTimeout(() => reject(new Error('knowledge_engine_thread_boot_timeout')), 60_000);
71
+ const onMsg = (msg) => {
72
+ if (msg?.type === 'ready') {
73
+ clearTimeout(timer);
74
+ worker.off('message', onMsg);
75
+ resolve();
76
+ }
77
+ else if (msg?.type === 'error') {
78
+ clearTimeout(timer);
79
+ worker.off('message', onMsg);
80
+ reject(new Error(msg.message ?? 'knowledge_engine_thread_failed'));
81
+ }
82
+ };
83
+ worker.on('message', onMsg);
84
+ worker.once('error', (err) => {
85
+ clearTimeout(timer);
86
+ reject(err);
87
+ });
88
+ worker.once('exit', (code) => {
89
+ clearTimeout(timer);
90
+ reject(new Error(`knowledge_engine_thread_exit_${code}`));
91
+ });
92
+ });
93
+ let metaQuery = null;
94
+ let searchQuery = null;
95
+ if (useSplitQuery) {
96
+ try {
97
+ [metaQuery, searchQuery] = await Promise.all([
98
+ startQueryWorker({
99
+ dbPath: opts.dbPath,
100
+ role: 'meta',
101
+ sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
102
+ embeddingModels: opts.embeddingModels,
103
+ }),
104
+ startQueryWorker({
105
+ dbPath: opts.dbPath,
106
+ role: 'search',
107
+ sqliteVecExtensionPath: opts.sqliteVecExtensionPath,
108
+ embeddingModels: opts.embeddingModels,
109
+ }),
110
+ ]);
111
+ }
112
+ catch (e) {
113
+ await Promise.all([
114
+ metaQuery?.dispose() ?? Promise.resolve(),
115
+ searchQuery?.dispose() ?? Promise.resolve(),
116
+ ]);
117
+ await worker.terminate().catch(() => undefined);
118
+ throw e;
119
+ }
120
+ }
121
+ let nextId = 1;
122
+ /** id → 非 SSE 响应等待 / SSE res */
123
+ const waiters = new Map();
124
+ const onBridge = (msg) => {
125
+ if (msg.id == null)
126
+ return;
127
+ const w = waiters.get(msg.id);
128
+ if (!w)
129
+ return;
130
+ if (msg.type === 'end') {
131
+ waiters.delete(msg.id);
132
+ w.resolve({ status: msg.status, headers: msg.headers, body: msg.body });
133
+ return;
134
+ }
135
+ if (msg.type === 'sse-start') {
136
+ if (w.res && !w.res.headersSent) {
137
+ w.res.writeHead(msg.status, msg.headers);
138
+ }
139
+ return;
140
+ }
141
+ if (msg.type === 'sse-write') {
142
+ w.res?.write(msg.chunk);
143
+ return;
144
+ }
145
+ if (msg.type === 'sse-end') {
146
+ waiters.delete(msg.id);
147
+ w.res?.end();
148
+ }
149
+ };
150
+ worker.on('message', onBridge);
151
+ // Engine 崩溃:立即冲刷在途请求,禁止静默挂死到客户端超时
152
+ const failAllWaiters = (reason) => {
153
+ for (const [, w] of waiters) {
154
+ if (w.sse && w.res && !w.res.headersSent) {
155
+ try {
156
+ w.res.writeHead(503, { 'content-type': 'application/json; charset=utf-8' });
157
+ w.res.end(JSON.stringify({ ok: false, error: reason }));
158
+ }
159
+ catch {
160
+ /* ignore */
161
+ }
162
+ }
163
+ else if (w.sse) {
164
+ try {
165
+ w.res?.end();
166
+ }
167
+ catch {
168
+ /* ignore */
169
+ }
170
+ }
171
+ else {
172
+ w.reject(new Error(reason));
173
+ }
174
+ }
175
+ waiters.clear();
176
+ };
177
+ worker.on('exit', (code, signal) => {
178
+ failAllWaiters(`knowledge_engine_thread_exit_${signal ?? code}`);
179
+ });
180
+ worker.on('error', (err) => {
181
+ failAllWaiters(`knowledge_engine_thread_error:${err.message}`);
182
+ });
183
+ const tokens = opts.tokens;
184
+ const server = createServer((req, res) => {
185
+ void (async () => {
186
+ const id = nextId++;
187
+ try {
188
+ const method = (req.method ?? 'GET').toUpperCase();
189
+ const urlPath = (req.url ?? '/').split('?')[0]?.replace(/\/+$/, '') || '/';
190
+ // 存活探测:Engine 再忙也必须立刻应答(进程级健康,不是业务健康)
191
+ if (method === 'GET' && urlPath === '/health') {
192
+ res.writeHead(200, {
193
+ 'content-type': 'application/json; charset=utf-8',
194
+ });
195
+ res.end(JSON.stringify({ ok: true, service: 'knowledge', version: '0.60.0', engine: 'thread' }));
196
+ return;
197
+ }
198
+ if (method === 'OPTIONS') {
199
+ res.writeHead(204);
200
+ res.end();
201
+ return;
202
+ }
203
+ // Token 鉴权:内存表查找,与 Engine 共用 matchKnowledgeToken
204
+ if (urlPath !== '/health') {
205
+ const hit = matchKnowledgeToken(tokens, req.headers.authorization);
206
+ if (!hit) {
207
+ res.writeHead(401, { 'content-type': 'application/json; charset=utf-8' });
208
+ res.end(JSON.stringify({ ok: false, error: { code: 'unauthorized', message: 'missing or invalid token' } }));
209
+ return;
210
+ }
211
+ }
212
+ // 纯读:主线程 → Meta/Search Worker(不进 Engine 事件循环)
213
+ if (useSplitQuery && metaQuery && searchQuery && isPureReadRoute(method, urlPath)) {
214
+ const auth = authenticateRead(tokens, req);
215
+ if (!auth) {
216
+ res.writeHead(401, { 'content-type': 'application/json; charset=utf-8' });
217
+ res.end(JSON.stringify({ ok: false, error: { code: 'unauthorized', message: 'missing or invalid token' } }));
218
+ return;
219
+ }
220
+ const handled = await handleKnowledgeReadHttp({
221
+ meta: metaQuery,
222
+ search: searchQuery,
223
+ tokens,
224
+ autoRegisterPrincipals: opts.autoRegisterPrincipals ?? true,
225
+ }, req, res, auth);
226
+ if (handled)
227
+ return;
228
+ }
229
+ const body = await readBody(req);
230
+ const bridgeReq = {
231
+ method,
232
+ url: req.url ?? '/',
233
+ headers: toBridgeHeaders(req.headers),
234
+ body,
235
+ };
236
+ const sse = urlPath === '/v1/events';
237
+ const result = await new Promise((resolve, reject) => {
238
+ waiters.set(id, {
239
+ resolve,
240
+ reject,
241
+ res: sse ? res : undefined,
242
+ sse,
243
+ });
244
+ const payload = { id, type: 'http', req: bridgeReq };
245
+ worker.postMessage(payload);
246
+ req.on('close', () => {
247
+ if (sse) {
248
+ worker.postMessage({ type: 'client-close', id });
249
+ waiters.delete(id);
250
+ try {
251
+ res.end();
252
+ }
253
+ catch {
254
+ /* already ended */
255
+ }
256
+ }
257
+ });
258
+ });
259
+ if (!res.headersSent) {
260
+ res.writeHead(result.status, result.headers);
261
+ res.end(result.body);
262
+ }
263
+ }
264
+ catch (e) {
265
+ if (!res.headersSent) {
266
+ res.writeHead(500, { 'content-type': 'application/json; charset=utf-8' });
267
+ res.end(JSON.stringify({ ok: false, error: e instanceof Error ? e.message : String(e) }));
268
+ }
269
+ }
270
+ })();
271
+ });
272
+ const host = opts.host ?? '127.0.0.1';
273
+ const port = opts.port ?? 18280;
274
+ await new Promise((resolve, reject) => {
275
+ server.once('error', reject);
276
+ server.listen(port, host, () => {
277
+ server.off('error', reject);
278
+ resolve();
279
+ });
280
+ });
281
+ const addr = server.address();
282
+ const bound = typeof addr === 'object' && addr ? addr.port : port;
283
+ return {
284
+ server,
285
+ port: bound,
286
+ close: async () => {
287
+ for (const [, w] of waiters) {
288
+ w.reject(new Error('knowledge_service_closing'));
289
+ }
290
+ waiters.clear();
291
+ try {
292
+ server.closeAllConnections?.();
293
+ }
294
+ catch {
295
+ /* old Node */
296
+ }
297
+ await new Promise((resolve) => {
298
+ const timer = setTimeout(resolve, 2_000);
299
+ timer.unref?.();
300
+ server.close(() => {
301
+ clearTimeout(timer);
302
+ resolve();
303
+ });
304
+ });
305
+ await Promise.all([
306
+ metaQuery?.dispose() ?? Promise.resolve(),
307
+ searchQuery?.dispose() ?? Promise.resolve(),
308
+ ]);
309
+ await new Promise((resolve) => {
310
+ const timer = setTimeout(() => {
311
+ void worker.terminate().finally(() => resolve());
312
+ }, 3_000);
313
+ timer.unref?.();
314
+ const onAck = (msg) => {
315
+ if (msg?.type === 'shutdown-ack') {
316
+ clearTimeout(timer);
317
+ worker.off('message', onAck);
318
+ resolve();
319
+ }
320
+ };
321
+ worker.on('message', onAck);
322
+ try {
323
+ worker.postMessage({ type: 'shutdown' });
324
+ }
325
+ catch {
326
+ clearTimeout(timer);
327
+ resolve();
328
+ }
329
+ });
330
+ await worker.terminate().catch(() => undefined);
331
+ },
332
+ };
333
+ }
334
+ function fileURLToWorker(moduleFile) {
335
+ return resolveWorkerUrl(moduleFile, import.meta.url);
336
+ }