dsh-data-domain-knowledge 0.1.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +360 -0
  3. package/cordis.patch.yml +410 -0
  4. package/eval/README.md +76 -0
  5. package/eval/evaluate-retrieval.mjs +121 -0
  6. package/eval/queries.example.jsonl +2 -0
  7. package/lib/client.js +111415 -0
  8. package/lib/types/config-schema.d.ts +35 -0
  9. package/lib/types/config-schema.d.ts.map +1 -0
  10. package/lib/types/config-schema.js +34 -0
  11. package/lib/types/config-schema.js.map +1 -0
  12. package/lib/types/convert/core-loader.d.ts +32 -0
  13. package/lib/types/convert/core-loader.d.ts.map +1 -0
  14. package/lib/types/convert/core-loader.js +60 -0
  15. package/lib/types/convert/core-loader.js.map +1 -0
  16. package/lib/types/convert/host.d.ts +88 -0
  17. package/lib/types/convert/host.d.ts.map +1 -0
  18. package/lib/types/convert/host.js +479 -0
  19. package/lib/types/convert/host.js.map +1 -0
  20. package/lib/types/convert/ingest.d.ts +125 -0
  21. package/lib/types/convert/ingest.d.ts.map +1 -0
  22. package/lib/types/convert/ingest.js +228 -0
  23. package/lib/types/convert/ingest.js.map +1 -0
  24. package/lib/types/convert/settings-schema.d.ts +18 -0
  25. package/lib/types/convert/settings-schema.d.ts.map +1 -0
  26. package/lib/types/convert/settings-schema.js +26 -0
  27. package/lib/types/convert/settings-schema.js.map +1 -0
  28. package/lib/types/convert/types.d.ts +46 -0
  29. package/lib/types/convert/types.d.ts.map +1 -0
  30. package/lib/types/convert/types.js +41 -0
  31. package/lib/types/convert/types.js.map +1 -0
  32. package/lib/types/diagnosis/cards.d.ts +71 -0
  33. package/lib/types/diagnosis/cards.d.ts.map +1 -0
  34. package/lib/types/diagnosis/cards.js +482 -0
  35. package/lib/types/diagnosis/cards.js.map +1 -0
  36. package/lib/types/diagnosis/problem-classes.d.ts +51 -0
  37. package/lib/types/diagnosis/problem-classes.d.ts.map +1 -0
  38. package/lib/types/diagnosis/problem-classes.js +66 -0
  39. package/lib/types/diagnosis/problem-classes.js.map +1 -0
  40. package/lib/types/diagnosis/schema.d.ts +65 -0
  41. package/lib/types/diagnosis/schema.d.ts.map +1 -0
  42. package/lib/types/diagnosis/schema.js +208 -0
  43. package/lib/types/diagnosis/schema.js.map +1 -0
  44. package/lib/types/diagnosis/seed-source.d.ts +20 -0
  45. package/lib/types/diagnosis/seed-source.d.ts.map +1 -0
  46. package/lib/types/diagnosis/seed-source.js +96 -0
  47. package/lib/types/diagnosis/seed-source.js.map +1 -0
  48. package/lib/types/diagnosis/seed.d.ts +50 -0
  49. package/lib/types/diagnosis/seed.d.ts.map +1 -0
  50. package/lib/types/diagnosis/seed.js +242 -0
  51. package/lib/types/diagnosis/seed.js.map +1 -0
  52. package/lib/types/diagnosis/store.d.ts +81 -0
  53. package/lib/types/diagnosis/store.d.ts.map +1 -0
  54. package/lib/types/diagnosis/store.js +409 -0
  55. package/lib/types/diagnosis/store.js.map +1 -0
  56. package/lib/types/diagnosis/types.d.ts +124 -0
  57. package/lib/types/diagnosis/types.d.ts.map +1 -0
  58. package/lib/types/diagnosis/types.js +13 -0
  59. package/lib/types/diagnosis/types.js.map +1 -0
  60. package/lib/types/directory-picker.d.ts +94 -0
  61. package/lib/types/directory-picker.d.ts.map +1 -0
  62. package/lib/types/directory-picker.js +110 -0
  63. package/lib/types/directory-picker.js.map +1 -0
  64. package/lib/types/extract-model.d.ts +76 -0
  65. package/lib/types/extract-model.d.ts.map +1 -0
  66. package/lib/types/extract-model.js +193 -0
  67. package/lib/types/extract-model.js.map +1 -0
  68. package/lib/types/extract.d.ts +121 -0
  69. package/lib/types/extract.d.ts.map +1 -0
  70. package/lib/types/extract.js +549 -0
  71. package/lib/types/extract.js.map +1 -0
  72. package/lib/types/fts.d.ts +96 -0
  73. package/lib/types/fts.d.ts.map +1 -0
  74. package/lib/types/fts.js +184 -0
  75. package/lib/types/fts.js.map +1 -0
  76. package/lib/types/index.d.ts +39 -0
  77. package/lib/types/index.d.ts.map +1 -0
  78. package/lib/types/index.js +1112 -0
  79. package/lib/types/index.js.map +1 -0
  80. package/lib/types/intent/engine.d.ts +108 -0
  81. package/lib/types/intent/engine.d.ts.map +1 -0
  82. package/lib/types/intent/engine.js +501 -0
  83. package/lib/types/intent/engine.js.map +1 -0
  84. package/lib/types/intent/host.d.ts +88 -0
  85. package/lib/types/intent/host.d.ts.map +1 -0
  86. package/lib/types/intent/host.js +389 -0
  87. package/lib/types/intent/host.js.map +1 -0
  88. package/lib/types/intent/preset-tools.d.ts +37 -0
  89. package/lib/types/intent/preset-tools.d.ts.map +1 -0
  90. package/lib/types/intent/preset-tools.js +45 -0
  91. package/lib/types/intent/preset-tools.js.map +1 -0
  92. package/lib/types/intent/settings-schema.d.ts +59 -0
  93. package/lib/types/intent/settings-schema.d.ts.map +1 -0
  94. package/lib/types/intent/settings-schema.js +57 -0
  95. package/lib/types/intent/settings-schema.js.map +1 -0
  96. package/lib/types/intent/store.d.ts +75 -0
  97. package/lib/types/intent/store.d.ts.map +1 -0
  98. package/lib/types/intent/store.js +139 -0
  99. package/lib/types/intent/store.js.map +1 -0
  100. package/lib/types/intent/tools.d.ts +63 -0
  101. package/lib/types/intent/tools.d.ts.map +1 -0
  102. package/lib/types/intent/tools.js +108 -0
  103. package/lib/types/intent/tools.js.map +1 -0
  104. package/lib/types/intent/types.d.ts +189 -0
  105. package/lib/types/intent/types.d.ts.map +1 -0
  106. package/lib/types/intent/types.js +62 -0
  107. package/lib/types/intent/types.js.map +1 -0
  108. package/lib/types/knowledge-bases.d.ts +266 -0
  109. package/lib/types/knowledge-bases.d.ts.map +1 -0
  110. package/lib/types/knowledge-bases.js +1107 -0
  111. package/lib/types/knowledge-bases.js.map +1 -0
  112. package/lib/types/layout.d.ts +262 -0
  113. package/lib/types/layout.d.ts.map +1 -0
  114. package/lib/types/layout.js +312 -0
  115. package/lib/types/layout.js.map +1 -0
  116. package/lib/types/migrate.d.ts +110 -0
  117. package/lib/types/migrate.d.ts.map +1 -0
  118. package/lib/types/migrate.js +280 -0
  119. package/lib/types/migrate.js.map +1 -0
  120. package/lib/types/recall-rank.d.ts +158 -0
  121. package/lib/types/recall-rank.d.ts.map +1 -0
  122. package/lib/types/recall-rank.js +261 -0
  123. package/lib/types/recall-rank.js.map +1 -0
  124. package/lib/types/runtime.d.ts +568 -0
  125. package/lib/types/runtime.d.ts.map +1 -0
  126. package/lib/types/runtime.js +3045 -0
  127. package/lib/types/runtime.js.map +1 -0
  128. package/lib/types/settings-schema.d.ts +117 -0
  129. package/lib/types/settings-schema.d.ts.map +1 -0
  130. package/lib/types/settings-schema.js +92 -0
  131. package/lib/types/settings-schema.js.map +1 -0
  132. package/lib/types/settings-volatile.d.ts +102 -0
  133. package/lib/types/settings-volatile.d.ts.map +1 -0
  134. package/lib/types/settings-volatile.js +114 -0
  135. package/lib/types/settings-volatile.js.map +1 -0
  136. package/lib/types/types.d.ts +799 -0
  137. package/lib/types/types.d.ts.map +1 -0
  138. package/lib/types/types.js +387 -0
  139. package/lib/types/types.js.map +1 -0
  140. package/lib/types/wiki-types.d.ts +68 -0
  141. package/lib/types/wiki-types.d.ts.map +1 -0
  142. package/lib/types/wiki-types.js +322 -0
  143. package/lib/types/wiki-types.js.map +1 -0
  144. package/package.json +101 -0
  145. package/preset/tita-data-agent/agent.cordis.yml +376 -0
  146. package/preset/tita-data-agent/preset.yml +3 -0
  147. package/python/requirements.txt +8 -0
  148. package/scripts/doctor.mjs +70 -0
  149. package/scripts/e2e-pipeline.mjs +355 -0
  150. package/scripts/install-agent-preset.mjs +92 -0
  151. package/scripts/setup-embedding.sh +34 -0
  152. package/scripts/sync-agent-preset.mjs +188 -0
  153. package/src/client/AgentPresetCard.tsx +247 -0
  154. package/src/client/DirectoryPicker.tsx +357 -0
  155. package/src/client/InfoHint.tsx +65 -0
  156. package/src/client/IntentLogicTab.tsx +185 -0
  157. package/src/client/IntentTab.tsx +715 -0
  158. package/src/client/KnowledgeBaseTab.tsx +1454 -0
  159. package/src/client/KnowledgeGraphView.tsx +306 -0
  160. package/src/client/KnowledgePanel.tsx +381 -0
  161. package/src/client/KnowledgeStepRail.tsx +60 -0
  162. package/src/client/NodeSettings.tsx +392 -0
  163. package/src/client/PreviewSections.tsx +116 -0
  164. package/src/client/RecallPanel.tsx +399 -0
  165. package/src/client/TypeDefinitionDrawer.tsx +215 -0
  166. package/src/client/WikiFlowCard.tsx +226 -0
  167. package/src/client/WikiTypeEditor.tsx +332 -0
  168. package/src/client/agent-preset.ts +121 -0
  169. package/src/client/arch/AgentArchitectureGraph.tsx +219 -0
  170. package/src/client/arch/NodeLogicPanel.tsx +133 -0
  171. package/src/client/arch/ProblemDetailPanel.tsx +88 -0
  172. package/src/client/arch/model.ts +691 -0
  173. package/src/client/architecture.css +191 -0
  174. package/src/client/architecture.tsx +530 -0
  175. package/src/client/attribution/AttributionGraph.tsx +104 -0
  176. package/src/client/attribution/AttributionTab.tsx +332 -0
  177. package/src/client/attribution/CaseView.tsx +102 -0
  178. package/src/client/attribution/GraphStage.tsx +87 -0
  179. package/src/client/attribution/RunView.tsx +184 -0
  180. package/src/client/attribution/StageChrome.tsx +403 -0
  181. package/src/client/attribution/ViewSwitch.tsx +107 -0
  182. package/src/client/attribution/graph-options.ts +381 -0
  183. package/src/client/attribution/model.ts +462 -0
  184. package/src/client/config-scope.ts +119 -0
  185. package/src/client/convert/ConvertPanel.tsx +594 -0
  186. package/src/client/convert/styles.css +138 -0
  187. package/src/client/convert/styles.ts +49 -0
  188. package/src/client/format.ts +39 -0
  189. package/src/client/graph/layout.ts +181 -0
  190. package/src/client/graph.css +113 -0
  191. package/src/client/index.ts +98 -0
  192. package/src/client/problem-domains.ts +114 -0
  193. package/src/client/styles.css +940 -0
  194. package/src/client/styles.ts +11 -0
  195. package/src/client/useFitScale.ts +68 -0
  196. package/src/config-schema.ts +45 -0
  197. package/src/convert/core-loader.ts +68 -0
  198. package/src/convert/host.ts +579 -0
  199. package/src/convert/ingest.ts +367 -0
  200. package/src/convert/settings-schema.ts +27 -0
  201. package/src/convert/types.ts +65 -0
  202. package/src/diagnosis/cards.ts +498 -0
  203. package/src/diagnosis/problem-classes.ts +117 -0
  204. package/src/diagnosis/schema.ts +260 -0
  205. package/src/diagnosis/seed-source.ts +107 -0
  206. package/src/diagnosis/seed.ts +291 -0
  207. package/src/diagnosis/store.ts +432 -0
  208. package/src/diagnosis/types.ts +152 -0
  209. package/src/directory-picker.ts +164 -0
  210. package/src/extract-model.ts +227 -0
  211. package/src/extract.ts +632 -0
  212. package/src/fts.ts +203 -0
  213. package/src/index.ts +1091 -0
  214. package/src/intent/README.md +63 -0
  215. package/src/intent/engine.ts +527 -0
  216. package/src/intent/host.ts +462 -0
  217. package/src/intent/preset-tools.ts +48 -0
  218. package/src/intent/settings-schema.ts +63 -0
  219. package/src/intent/store.ts +143 -0
  220. package/src/intent/tools.ts +139 -0
  221. package/src/intent/types.ts +245 -0
  222. package/src/knowledge-bases.ts +1160 -0
  223. package/src/layout.ts +350 -0
  224. package/src/migrate.ts +262 -0
  225. package/src/python/embedding_worker.py +471 -0
  226. package/src/recall-rank.ts +299 -0
  227. package/src/runtime.ts +3069 -0
  228. package/src/settings-schema.ts +94 -0
  229. package/src/settings-volatile.ts +122 -0
  230. package/src/types.ts +1140 -0
  231. package/src/wiki-types.ts +357 -0
@@ -0,0 +1,355 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * SourceRaw → 编译 → 构建 → 召回 → 重排 的端到端实跑(0029)。
4
+ *
5
+ * ## 为什么要一个可重跑的脚本
6
+ *
7
+ * 这条链路跨五个阶段、三个存储层和两个进程外模型(WeMM 嵌入、BERT 重排),而单元
8
+ * 测试每一段都用替身把邻居挡掉了 —— 于是「每一段都对、连起来不对」这类问题只会在
9
+ * 真跑一次时才露出来。这个脚本就是把那条链路**按真实顺序跑一遍**,并把它每一步的
10
+ * 真实数字打出来,好让「哪里不对」是一个可读的结论而不是一句「好像有问题」。
11
+ *
12
+ * ## 两条纪律
13
+ *
14
+ * 1. **只跑副本。** 默认把 `--from` 指向的语料**复制**进一个临时 agent root 再跑,
15
+ * 所以它不会碰任何真实知识树(版本号、编译产物、记忆快照都不会被动)。
16
+ * 2. **失败不中断。** 每个阶段单独 try/catch,失败记下来继续跑后面的 —— 链路排查
17
+ * 最怕的是第一步就抛,于是看不到后面还有没有别的问题。
18
+ *
19
+ * 用法:
20
+ * node scripts/e2e-pipeline.mjs # 用内置夹具
21
+ * node scripts/e2e-pipeline.mjs --from ~/domain_knowledge # 用真实语料的副本
22
+ * node scripts/e2e-pipeline.mjs --root /tmp/x --keep # 指定 root 且跑完不删
23
+ * node scripts/e2e-pipeline.mjs --no-models # 跳过嵌入/重排(快,只看文本链路)
24
+ */
25
+ import { cp, mkdir, mkdtemp, readdir, rm, stat } from 'node:fs/promises'
26
+ import { existsSync } from 'node:fs'
27
+ import { tmpdir } from 'node:os'
28
+ import { join, resolve } from 'node:path'
29
+ import { fileURLToPath } from 'node:url'
30
+ import { KnowledgeRuntime } from '../lib/types/runtime.js'
31
+
32
+ const args = process.argv.slice(2)
33
+ const flag = name => args.includes(name)
34
+ const value = (name, fallback) => {
35
+ const index = args.indexOf(name)
36
+ return index >= 0 && args[index + 1] ? args[index + 1] : fallback
37
+ }
38
+
39
+ const from = value('--from', null)
40
+ const keep = flag('--keep')
41
+ const noModels = flag('--no-models')
42
+ const explicitRoot = value('--root', null)
43
+
44
+ const pluginRoot = resolve(fileURLToPath(new URL('.', import.meta.url)), '..')
45
+
46
+ /** 内置夹具:三种来源形态,覆盖「直写」「转换而来」「同一文档的第二个版本」。 */
47
+ const FILLER = '补充说明:这一节的结论来自当期对账口径复核,与财务确认后生效,口径变更需要同时通知数据与财务两侧,并在下一次月结前完成回归验证。'
48
+ const FIXTURES = [
49
+ {
50
+ // 经转换内核进来的形态:`source_path` 让扩展名推导拿到真实扩展名,
51
+ // `parser` 是内核写下的真值。
52
+ sourceType: 'document',
53
+ title: 'GMV 口径说明',
54
+ externalId: 'doc:gmv-metric',
55
+ content: `# GMV 口径说明\n\nGMV 指成交总额,含未支付订单。它与营收的差别在于退款与取消是否从总额里扣减,对账时必须先说清用的是哪一个。\n\n${FILLER}\n\n## 计算方式\n\nGMV 等于订单金额之和,按**下单时间**归集,不按支付时间。跨月订单算在下单月,这是它与营收最容易对不上的地方。\n\n${FILLER}\n\n## 与营收的区别\n\n营收扣减退款与取消,GMV 不扣。两者对账时必然不等,差额就是当期退款总额,这也就是退款表的核对入口。\n\n${FILLER}\n`,
56
+ metadata: { source_path: '/incoming/GMV口径说明.pdf', parser: 'anydoc@1/pdf' },
57
+ },
58
+ {
59
+ // 直写进来的形态:没有 parser,扩展名靠 source_format 推。
60
+ sourceType: 'document',
61
+ title: '退款处理流程',
62
+ externalId: 'doc:refund-flow',
63
+ content: `# 退款处理流程\n\n退款分三步:申请、审核、打款。审核超过 48 小时未处理会自动升级到主管,避免卡在单个人身上。\n\n${FILLER}\n\n## 打款失败怎么办\n\n打款失败会退回待处理状态,并在 [[GMV 口径说明]] 里那条差额上体现出来,所以对账前要先看退款表。\n\n${FILLER}\n`,
64
+ metadata: { source_format: 'markdown' },
65
+ },
66
+ {
67
+ // 什么都没有的形态:扩展名回退成 .md。
68
+ sourceType: 'document',
69
+ title: '数据表清单',
70
+ externalId: 'doc:tables',
71
+ content: `# 数据表清单\n\ndwd_order 是订单明细表,一行一个订单。dwd_refund 是退款表,一行一次退款,两表用 order_id 关联。\n\n${FILLER}\n\n## 关联关系\n\ndwd_refund.order_id 关联 dwd_order.order_id。GMV 与退款额的差额在 [[GMV 口径说明]] 里有定义,不要各算各的。\n\n${FILLER}\n`,
72
+ metadata: {},
73
+ },
74
+ ]
75
+
76
+ function hr(title) {
77
+ console.log(`\n${'─'.repeat(4)} ${title} ${'─'.repeat(Math.max(0, 62 - title.length))}`)
78
+ }
79
+
80
+ function show(label, value) {
81
+ console.log(` ${label.padEnd(28)} ${value}`)
82
+ }
83
+
84
+ /** 把一个可能很长的错误压成一行,免得把报告淹掉。 */
85
+ function short(error) {
86
+ const text = error instanceof Error ? `${error.name}: ${error.message}` : String(error)
87
+ return text.length > 300 ? `${text.slice(0, 300)}…` : text
88
+ }
89
+
90
+ async function main() {
91
+ const root = explicitRoot ? resolve(explicitRoot) : await mkdtemp(join(tmpdir(), 'tita-e2e-'))
92
+ await mkdir(root, { recursive: true })
93
+ const knowledgeRoot = join(root, 'domain_knowledge', 'default')
94
+ console.log(`agent root ${root}`)
95
+ console.log(`知识库根 ${knowledgeRoot}`)
96
+
97
+ const problems = []
98
+ const note = (stage, message) => { problems.push({ stage, message }); console.log(` ⚠ ${stage}: ${message}`) }
99
+
100
+ // ---------------------------------------------------------------- 0. 语料
101
+ hr('0. 语料')
102
+ let corpusSource = '内置夹具'
103
+ if (from) {
104
+ const source = resolve(from.replace(/^~/, process.env.HOME ?? '~'))
105
+ if (!existsSync(source)) throw new Error(`--from 指向的语料不存在:${source}`)
106
+ /*
107
+ * `--from` 可以给**知识库目录**(里面有 01_source_truth 等段),也可以给它自然的
108
+ * 父目录(`domain_knowledge/`)。父目录是最顺手的写法,但直接 `cp` 会把它当成目标
109
+ * 本身,于是真正的 `default/` 被多套一层 —— 运行时读到 0 份文档,然后后面每个阶段
110
+ * 都报一串**看起来很确定**的假问题(第一次跑就是这么得到 17 条)。
111
+ *
112
+ * 诊断工具说谎比没有诊断更糟,所以这里认出来并自己降级;认不出就明确报错,而不是
113
+ * 让下游去猜。
114
+ */
115
+ const segmentPattern = /^\d{2}_[a-z_]+$/
116
+ const looksLikeKb = async dir => (await readdir(dir).catch(() => [])).some(entry => segmentPattern.test(entry))
117
+ let corpus = source
118
+ if (!(await looksLikeKb(source))) {
119
+ const children = (await readdir(source, { withFileTypes: true }).catch(() => []))
120
+ .filter(entry => entry.isDirectory())
121
+ .map(entry => join(source, entry.name))
122
+ const candidates = []
123
+ for (const child of children) if (await looksLikeKb(child)) candidates.push(child)
124
+ if (candidates.length === 1) {
125
+ corpus = candidates[0]
126
+ show('语料修正', `--from 给的是父目录,实际使用 ${corpus}`)
127
+ } else if (candidates.length === 0) {
128
+ throw new Error(`--from 指向的目录里找不到知识库段(01_source_truth 等):${source}`)
129
+ } else {
130
+ throw new Error(`--from 指向的目录下有多个知识库,请直接指定其中一个:\n ${candidates.join('\n ')}`)
131
+ }
132
+ }
133
+ await cp(corpus, knowledgeRoot, { recursive: true })
134
+ corpusSource = `${corpus}(副本)`
135
+ const entries = await readdir(knowledgeRoot).catch(() => [])
136
+ show('来源', corpusSource)
137
+ show('搬进来的段', entries.filter(entry => entry !== '.DS_Store').join(' '))
138
+ } else {
139
+ show('来源', corpusSource)
140
+ }
141
+
142
+ /*
143
+ * 家当(`resource/`:Python 环境 + 本地模型)**不在这次副本里** —— 只复制了知识树。
144
+ * 于是模型腿会以「No module named 'torch'」这类形式报不可用,而那是**环境缺失**,
145
+ * 不是链路坏了(真实 agent root 里模型是好的)。这两件事在报告里必须分得开,否则
146
+ * 每次跑都会得到一串听起来很严重的假警报。
147
+ */
148
+ const envHint = from
149
+ ? `副本只含知识树,不含家当(resource/)。模型腿若报 No module named …,那是这次副本没有模型环境,不代表真实库不可用;真实库请用 /knowledge/stats 核对。`
150
+ : null
151
+
152
+ const config = {
153
+ // 0029 的默认值:抽取开着(它就是「SourceRaw → Wiki」唯一不依赖模型的一步)。
154
+ extractCandidates: true,
155
+ // `--no-models` 时把两条模型腿关掉:文本链路照样能跑,且报告会如实说它们没跑。
156
+ embeddingProvider: noModels ? 'none' : 'wemm-python',
157
+ rerankProvider: noModels ? 'none' : 'bert-python',
158
+ recallTopK: 10,
159
+ recallMode: 'hybrid',
160
+ contextTopN: 5,
161
+ }
162
+ const runtime = new KnowledgeRuntime(knowledgeRoot, config)
163
+ const t0 = Date.now()
164
+
165
+ try {
166
+ // ------------------------------------------------------------ 1. SourceRaw
167
+ hr('1. SourceRaw 收录')
168
+ if (!from) {
169
+ for (const fixture of FIXTURES) {
170
+ const result = await runtime.addSource(fixture)
171
+ const status = result.ok ? 'ok' : `失败 ${result.error?.code}`
172
+ show(fixture.externalId, status)
173
+ if (!result.ok) note('SourceRaw', `${fixture.externalId} 写入失败:${result.error?.code} ${result.error?.message}`)
174
+ }
175
+ // 幂等:同一个 externalId 再写一次应该是「更新」,不是新增。
176
+ const again = await runtime.addSource({ ...FIXTURES[0], content: `${FIXTURES[0].content}\n\n## 补充\n\n口径每年复核一次。\n` })
177
+ show('同 externalId 重写', again.ok ? `ok(版本 +1)` : `失败 ${again.error?.code}`)
178
+ if (!again.ok) note('SourceRaw', '同 externalId 重写失败:幂等契约不成立')
179
+ }
180
+
181
+ let documents = await runtime.listDocuments()
182
+ show('文档数', documents.length)
183
+ const withExt = documents.filter(document => document.ext && document.ext !== '.md').length
184
+ show('非 .md 扩展名', `${withExt} / ${documents.length}(靠 metadata.source_path 推导)`)
185
+ const parsers = [...new Set(documents.map(document => document.parser))]
186
+ show('parser 取值', parsers.join(' ') || '(无)')
187
+ show('扩展名分布', Object.entries(documents.reduce((acc, d) => ({ ...acc, [d.ext]: (acc[d.ext] ?? 0) + 1 }), {})).map(([k, v]) => `${k}:${v}`).join(' '))
188
+ const versions = documents.reduce((sum, document) => sum + (document.versionCount ?? 0), 0)
189
+ show('版本快照总数', versions)
190
+ if (documents.length === 0) note('SourceRaw', '一份文档都没有 —— 后面的阶段都会是空的')
191
+
192
+ // ------------------------------------------------------------ 2. 编译(Pass 0 + 落卡)
193
+ hr('2. 编译:Pass 0 抽取 + 归一 + 落卡')
194
+ const wiki = await runtime.rebuild('wiki')
195
+ if (!wiki.ok) {
196
+ note('编译', `rebuild('wiki') 失败:${wiki.error?.code} ${wiki.error?.message}`)
197
+ } else {
198
+ const report = wiki.data
199
+ show('卡片', `${report.cards}(新增 ${report.cardsAdded ?? 0} / 复用 ${report.cardsReused ?? 0} / 重编 ${report.cardsRecompiled ?? 0})`)
200
+ show('wikiNotes', report.notes)
201
+ show('版本', `v${report.version}`)
202
+ // 八个 Pass 0 计数**直接在 report 上**(`lastBuild` 是 build-manifest.json 的
203
+ // 形状,不是 RebuildReport 的字段 —— 这里第一次就写错了,读出来全是 0)。
204
+ show('抽取候选', `${report.candidatesExtracted ?? 0} 抽出 / ${report.candidatesRejected ?? 0} 拒绝`)
205
+ show('归一结果', `${report.cardsMerged ?? 0} 合并进已有卡片 / ${report.cardsCreated ?? 0} 新建`)
206
+ show('抽出的关系', `${report.relationsExtracted ?? 0} 条,丢掉 ${report.relationsDropped ?? 0} 条`)
207
+ show('来源层', `${report.sourcesExtracted ?? 0} 篇参与 / ${report.sourcesSkipped ?? 0} 篇跳过`)
208
+ if ((report.candidatesExtracted ?? 0) === 0 && (report.sourcesExtracted ?? 0) > 0) {
209
+ const shortest = Math.min(...documents.map(document => document.content?.length ?? 0))
210
+ note('编译', `Pass 0 参与了 ${report.sourcesExtracted} 篇来源却一个候选都没抽出来。报告只说「0 抽出 / 0 拒绝」,不区分「小节都短于阈值」与「抽取坏了」—— 这是一处报告缺口(本批最短文档 ${shortest} 字)`)
211
+ }
212
+ if ((report.relationsDropped ?? 0) > 0) note('编译', `有 ${report.relationsDropped} 条关系被丢掉:两端没同时存在([[链接]] 指向了还没落卡的实体)`)
213
+ if ((report.cardsCreated ?? 0) > 0 && report.cardsAdded !== report.cardsCreated) {
214
+ show('计数自洽性', `cardsAdded=${report.cardsAdded} 与 cardsCreated=${report.cardsCreated} 不等(前者是 stage 2 的「新指纹」,后者是 Pass 0 的「新建卡片」)`)
215
+ }
216
+ }
217
+
218
+ const cards = await runtime.loadCards()
219
+ show('卡片类型分布', Object.entries(cards.reduce((acc, card) => ({ ...acc, [card.entityType]: (acc[card.entityType] ?? 0) + 1 }), {})).map(([k, v]) => `${k}:${v}`).join(' ') || '(无)')
220
+ const noRefs = cards.filter(card => !card.sourceRefs || card.sourceRefs.length === 0)
221
+ if (noRefs.length > 0) note('编译', `${noRefs.length} 张卡片没有 source_refs —— 它们无法追溯回原文`)
222
+ // 引用必须真的指向存在的文档。
223
+ const knownIds = new Set(documents.map(document => document.docId))
224
+ for (const card of cards) {
225
+ const dangling = (card.sourceRefs ?? []).filter(ref => !knownIds.has(ref))
226
+ if (dangling.length > 0) note('编译', `卡片 ${card.entityId} 的 source_refs 指向不存在的文档:${dangling.join(' ')}`)
227
+ }
228
+
229
+ // 增量:语料没变时重跑应该是空操作。
230
+ const wikiAgain = await runtime.rebuild('wiki')
231
+ if (wikiAgain.ok) {
232
+ const report = wikiAgain.data
233
+ show('原地重跑', `${report.skipped ? 'skipped' : '没有跳过'} · 复用 ${report.cardsReused ?? 0} / 重编 ${report.cardsRecompiled ?? 0} / notes 复用 ${report.notesReused ?? 0}`)
234
+ if (!report.skipped && (report.cardsReused ?? 0) === 0 && report.cards > 0) {
235
+ note('编译', `语料没变,${report.cards} 张卡片却一张都没复用(reused=0、recompiled=${report.cardsRecompiled ?? 0})—— 增量指纹没生效`)
236
+ }
237
+ } else {
238
+ note('编译', `第二次 rebuild('wiki') 失败:${wikiAgain.error?.code}`)
239
+ }
240
+
241
+ // ------------------------------------------------------------ 3. 构建(向量 + 图)
242
+ hr('3. 构建:wikiNotes + 向量索引 + 知识图谱')
243
+ const build = await runtime.rebuild('all')
244
+ if (!build.ok) {
245
+ note('构建', `rebuild('all') 失败:${build.error?.code} ${build.error?.message}`)
246
+ } else {
247
+ const report = build.data
248
+ show('卡片 / notes', `${report.cards} / ${report.notes}`)
249
+ show('图节点 / 边', `${report.graphNodes} / ${report.graphEdges}`)
250
+ show('嵌入', `${report.embedding}${report.embeddingItems === undefined ? '' : ` · ${report.embeddingItems} 条`}${report.embeddingReused === undefined ? '' : ` · 复用 ${report.embeddingReused}`}${report.embeddingDimension ? ` · ${report.embeddingDimension} 维` : ''}`)
251
+ if (report.embeddingError) note('构建', `嵌入不可用:${report.embeddingError}`)
252
+ if (report.embedding === 'unavailable') note('构建', '向量腿没起来 —— 召回会退化成关键词 + 图')
253
+ if (report.graphEdges === 0) note('构建', '图一条边都没有:溯源边 supported_by 应该至少等于「卡片引用了几篇文档」')
254
+ }
255
+
256
+ const graph = await runtime.graphQuery('types')
257
+ if (graph.ok) show('图上的关系类型', (graph.data.relationTypes ?? []).join(' ') || '(无)')
258
+ const listed = await runtime.graphQuery('list', undefined, undefined, 1, { limit: 50 })
259
+ if (listed.ok) {
260
+ const nodes = listed.data.nodes ?? []
261
+ const edges = listed.data.edges ?? []
262
+ const byType = nodes.reduce((acc, node) => ({ ...acc, [node.type ?? '?']: (acc[node.type ?? '?'] ?? 0) + 1 }), {})
263
+ show('图节点类型分布', Object.entries(byType).map(([k, v]) => `${k}:${v}`).join(' ') || '(无)')
264
+ // 图节点的键是 `${type}:${id}`,没有现成的 degree —— 自己按边算。
265
+ const touched = new Set()
266
+ for (const edge of edges) { touched.add(String(edge.from)); touched.add(String(edge.to)) }
267
+ const orphan = nodes.filter(node => !touched.has(`${node.type}:${node.id}`))
268
+ show('被边连上的节点', `${nodes.length - orphan.length} / ${nodes.length}`)
269
+ if (orphan.length > 0) show('孤立节点', orphan.map(node => `${node.type}:${node.id}`).slice(0, 6).join(' ') + (orphan.length > 6 ? ` …共 ${orphan.length} 个` : ''))
270
+ const byEdge = edges.reduce((acc, edge) => ({ ...acc, [edge.type ?? '?']: (acc[edge.type ?? '?'] ?? 0) + 1 }), {})
271
+ show('边类型分布', Object.entries(byEdge).map(([k, v]) => `${k}:${v}`).join(' ') || '(无)')
272
+ } else {
273
+ note('构建', `graphQuery('list') 失败:${listed.error?.code}`)
274
+ }
275
+
276
+ // ------------------------------------------------------------ 4. 召回
277
+ hr('4. 召回:三路 + 融合')
278
+ const health = await runtime.embeddingHealth()
279
+ if (health.ok) {
280
+ const index = health.data.vectorIndex ?? {}
281
+ show('嵌入后端', `${health.data.provider} · 索引 ${index.ready ? 'ready' : 'not-ready'} · ${index.itemCount ?? 0} 条 · ${health.data.dimension ?? '?'} 维`)
282
+ if (health.data.lastError) show('嵌入上次错误', String(health.data.lastError).slice(0, 160))
283
+ if (!index.ready) note('召回', `向量索引没就绪(${index.itemCount ?? 0} 条)—— vector 路会退化成关键词路`)
284
+ } else {
285
+ note('召回', `embeddingHealth 失败:${health.error?.code}`)
286
+ }
287
+
288
+ const queries = ['GMV 怎么算', '退款审核超时怎么办', 'dwd_refund 关联哪张表', '知识库里有没有讲过营收']
289
+ for (const mode of ['fts', 'vector', 'graph', 'hybrid']) {
290
+ for (const query of queries.slice(0, 2)) {
291
+ const result = await runtime.recall(query, 10, [], false, mode)
292
+ if (!result.ok) { note('召回', `${mode} / 「${query}」失败:${result.error?.code} ${result.error?.message}`); continue }
293
+ const data = result.data
294
+ const routes = (data.routes ?? []).join('+') || '(无)'
295
+ show(`${mode} · ${query}`, `${(data.hits ?? []).length} 命中 · top=${data.hits?.[0]?.score?.toFixed?.(3) ?? '—'} · 实际走 ${routes} · backend ${data.backend}`)
296
+ if ((data.hits ?? []).length === 0) note('召回', `${mode} 对「${query}」零命中`)
297
+ // 请求的路由与实际走的路由不一致时必须说清 —— 这是「我说了算」与「它实际怎么跑」的差别。
298
+ if (mode === 'vector' && !(data.routes ?? []).includes('vector')) note('召回', `请求 mode=vector,实际只走了 ${routes}:向量腿不可用时会静默退化成关键词路`)
299
+ if (mode === 'graph' && !(data.routes ?? []).includes('graph')) note('召回', `请求 mode=graph,实际只走了 ${routes}`)
300
+ }
301
+ }
302
+
303
+ // ------------------------------------------------------------ 5. 重排
304
+ hr('5. 重排:BERT cross-encoder → 上下文包')
305
+ const reranked = await runtime.recall('GMV 口径与营收的差别', 10, [], false, 'hybrid', undefined, { contextTopN: 5, rerank: true })
306
+ if (!reranked.ok) {
307
+ note('重排', `带 rerank 的召回失败:${reranked.error?.code} ${reranked.error?.message}`)
308
+ } else {
309
+ const data = reranked.data
310
+ show('重排', `${data.rerank?.status} · ${data.rerank?.provider}${data.rerank?.model ? ` · ${data.rerank.model}` : ''}${data.rerank?.scored === undefined ? '' : ` · 打分了 ${data.rerank.scored} 条`}`)
311
+ show('进入上下文的条数', (data.context ?? []).length)
312
+ show('命中数', (data.hits ?? []).length)
313
+ if (data.rerank?.status === 'unavailable') note('重排', `精排不可用:${data.rerank.error ?? '未说明原因'} —— 顺序按融合分数,不是 cross-encoder 的分数`)
314
+ if (data.rerank?.status === 'skipped') show('重排跳过原因', (data.hits ?? []).length < 2 ? '命中不足 2 条,没有可排的东西' : 'provider 配成了 none')
315
+ if ((data.context ?? []).length === 0) note('重排', '上下文包是空的:模型这一轮拿不到任何知识')
316
+ const first = data.context?.[0]
317
+ if (first) {
318
+ const text = first.note?.content ?? ''
319
+ show('上下文首条', `${first.note?.entityType ?? '?'}:${first.note?.entityId ?? '?'}「${first.note?.title ?? '?'}」(正文 ${text.length} 字)`)
320
+ if (text.length === 0) note('重排', '上下文首条的正文是空的 —— 模型拿到的是一条没有内容的「知识」')
321
+ if (!first.wiki) note('重排', `上下文首条没有解出 wiki 卡片(note 指向 ${first.note?.entityId})—— 卡片可能在编译后被删了`)
322
+ }
323
+ }
324
+
325
+ // ------------------------------------------------------------ 6. 统计
326
+ hr('6. 落盘事实(/knowledge/stats 的来源)')
327
+ // `stats()` 直接返回统计对象(不是 Envelope)—— 别按 ok 判。
328
+ const data = await runtime.stats()
329
+ show('来源 / 卡片 / notes', `${data.sourceCount} / ${data.cards ?? '?'} / ${data.notes ?? '?'}`)
330
+ show('版本', `v${data.buildVersion}`)
331
+ show('各段文件数', Object.entries(data.layout ?? {}).map(([k, v]) => `${k}:${v}`).join(' '))
332
+ show('卡片版本文件数', data.cardVersions ?? '?')
333
+ if (data.notes === 0) note('统计', 'notes 为 0:召回没有可检索的单元')
334
+
335
+ hr('结论')
336
+ console.log(` 语料来源 ${corpusSource}`)
337
+ console.log(` 耗时 ${((Date.now() - t0) / 1000).toFixed(1)}s`)
338
+ if (problems.length === 0) console.log(' 没有发现问题。')
339
+ else {
340
+ console.log(` 发现 ${problems.length} 处问题:`)
341
+ for (const problem of problems) console.log(` · [${problem.stage}] ${problem.message}`)
342
+ }
343
+ if (envHint) console.log(`\n 读数前提 ${envHint}`)
344
+ return problems
345
+ } finally {
346
+ await runtime.close?.()
347
+ if (!keep && !explicitRoot) await rm(root, { recursive: true, force: true })
348
+ else console.log(`\n(保留在 ${root})`)
349
+ }
350
+ }
351
+
352
+ main().then(problems => { process.exitCode = problems.length > 0 ? 2 : 0 }).catch(error => {
353
+ console.error(`\n实跑中断:${short(error)}`)
354
+ process.exitCode = 1
355
+ })
@@ -0,0 +1,92 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Tita Data Agent preset 的**迁移与自检**(0.1.7 改写)。
4
+ *
5
+ * ## 这个脚本以前做什么,为什么不能那么做了
6
+ *
7
+ * 它以前把 `preset/tita-data-agent/` 拷进 `$DSH_HOME/.agent-presets/<id>/`,
8
+ * 因为 0.1.5 的用户 preset 就是那个目录里的 `preset.yml` + `agent.cordis.yml`。
9
+ *
10
+ * **0.1.7 不再读那个目录。** registry 只接受 `agentPresets.register(config)`,
11
+ * 没有任何文件系统扫描;preset 的唯一形态是 profile 组合里的一条
12
+ * `@deepseek-ai/dsh-agent-preset` 声明行。所以「安装」这一步没有了 —— preset 现在
13
+ * 随插件自己的 bundle patch 分发(`cordis.patch.yml` 的生成标记区),装插件就是装
14
+ * preset。
15
+ *
16
+ * ## 它现在做什么
17
+ *
18
+ * 1. **校验同步**:`cordis.patch.yml` 的声明必须与 `preset/` 真源一致。真源改了
19
+ * 没跑 `pnpm sync:preset` 的话,DSH 读到的还是旧组合 —— 这正是「我改了提示词
20
+ * 但没生效」最可能的原因。
21
+ * 2. **报告遗留目录**:`$DSH_HOME/.agent-presets/` 里的旧 preset 已经没有任何
22
+ * 消费者。留着它只会让人继续以为 preset 装在那里,所以这里如实报出来,并给
23
+ * `--remove-legacy` 让人显式清掉。
24
+ * 3. **打印设默认的两条路径** —— 这个脚本不替用户决定默认 preset。
25
+ *
26
+ * 用法:
27
+ * node scripts/install-agent-preset.mjs
28
+ * node scripts/install-agent-preset.mjs --remove-legacy # 顺手删掉旧目录
29
+ */
30
+ import { rm, stat } from 'node:fs/promises'
31
+ import { homedir } from 'node:os'
32
+ import { dirname, join, resolve } from 'node:path'
33
+ import { fileURLToPath } from 'node:url'
34
+ import { spawnSync } from 'node:child_process'
35
+
36
+ /** 与 `src/client/agent-preset.ts` 的 `TITA_PRESET_ID` 一致。 */
37
+ const PRESET_ID = 'tita-data-agent'
38
+
39
+ const pluginRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
40
+ const dshHome = resolve(process.env.DSH_HOME || join(homedir(), '.dsh'))
41
+ const legacyDir = join(dshHome, '.agent-presets', PRESET_ID)
42
+
43
+ const removeLegacy = process.argv.includes('--remove-legacy')
44
+
45
+ async function exists(path) {
46
+ try { await stat(path); return true } catch { return false }
47
+ }
48
+
49
+ async function main() {
50
+ let problems = 0
51
+
52
+ // ── 1. 声明与真源是否同步 ────────────────────────────────────────────────
53
+ console.log('检查 cordis.patch.yml 里的 preset 声明 …')
54
+ const sync = spawnSync(process.execPath, [join(pluginRoot, 'scripts', 'sync-agent-preset.mjs'), '--check'], {
55
+ stdio: 'inherit',
56
+ cwd: pluginRoot,
57
+ })
58
+ if (sync.status !== 0) {
59
+ problems += 1
60
+ console.log(' → 不同步。跑 `pnpm sync:preset` 把 preset/ 真源编译进 patch。')
61
+ }
62
+
63
+ // ── 2. 遗留的旧目录 ──────────────────────────────────────────────────────
64
+ console.log(`\n检查旧目录 ${legacyDir} …`)
65
+ if (await exists(legacyDir)) {
66
+ console.log(' 存在,但**现在没有任何东西读它**(0.1.7 起 DSH 不扫描 .agent-presets/)。')
67
+ if (removeLegacy) {
68
+ await rm(legacyDir, { recursive: true, force: true })
69
+ console.log(' 已按 --remove-legacy 删除。')
70
+ } else {
71
+ console.log(' 它不会生效,也不会造成冲突;要清掉加 --remove-legacy。')
72
+ problems += 1
73
+ }
74
+ } else {
75
+ console.log(' 不存在,无需迁移。')
76
+ }
77
+
78
+ // ── 3. 怎么让它成为默认 ─────────────────────────────────────────────────
79
+ console.log('\n让它成为默认(脚本不替你做这个决定):')
80
+ // 只有一个入口:DSH 设置里那一项。TitaAgent 面板里曾经有同一个下拉,已经撤了 ——
81
+ // 它写的是同一个字段,摆两处只会让用户猜哪处算数。
82
+ console.log(` · DSH 设置 → Agent 预设 →「${PRESET_ID}」→ 设为默认`)
83
+ console.log(' · TitaAgent 面板只**告诉你它还不是默认**,并给出提示词的编辑入口(打开 preset 目录)')
84
+ console.log('\n改动生效需要**重启 DSH**;preset 在会话创建时固定,正在跑的会话不受影响。')
85
+
86
+ if (problems > 0) process.exitCode = 1
87
+ }
88
+
89
+ main().catch(error => {
90
+ console.error(`检查失败:${error instanceof Error ? error.message : String(error)}`)
91
+ process.exitCode = 1
92
+ })
@@ -0,0 +1,34 @@
1
+ #!/usr/bin/env bash
2
+ # Install the optional local WeMM runtime and download only the current model
3
+ # files. The model is intentionally outside the repository and is not bundled
4
+ # into the DSH plugin package.
5
+ set -euo pipefail
6
+
7
+ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
8
+ TITA_HOME="${TITA_DATA_AGENT_HOME:-$HOME/.tita-data-agent}"
9
+ VENV="${DSH_EMBEDDING_VENV:-$TITA_HOME/embedding-venv}"
10
+ MODEL_DIR="${DSH_EMBEDDING_MODEL_PATH:-$TITA_HOME/models/WeMM-Embedding-2B-Quantized}"
11
+ PYTHON_BIN="$VENV/bin/python"
12
+
13
+ mkdir -p "$TITA_HOME" "$MODEL_DIR"
14
+ if [ ! -x "$PYTHON_BIN" ]; then
15
+ uv venv --python 3.11 "$VENV"
16
+ fi
17
+ uv pip install --python "$PYTHON_BIN" -r "$ROOT/python/requirements.txt"
18
+
19
+ if [ ! -f "$MODEL_DIR/config.json" ] || [ ! -f "$MODEL_DIR/model.safetensors" ]; then
20
+ # `hf download` accepts either exact positional filenames or one --include
21
+ # glob. Exact filenames avoid pulling repository history and work across the
22
+ # current and older huggingface_hub CLIs.
23
+ "$VENV/bin/hf" download ewin-reg/WeMM-Embedding-2B-Quantized \
24
+ model.safetensors tokenizer.json tokenizer_config.json config.json \
25
+ config_sentence_transformers.json sentence_bert_config.json modules.json \
26
+ processor_config.json modeling_st_wemm.py modeling_wemm_embedding.py \
27
+ chat_template.jinja \
28
+ --local-dir "$MODEL_DIR"
29
+ fi
30
+
31
+ echo "Embedding runtime ready. Configure Tita Data Agent with:"
32
+ echo " provider = wemm-python"
33
+ echo " python = $PYTHON_BIN"
34
+ echo " model = $MODEL_DIR"