dsh-data-domain-knowledge 0.1.0-alpha.3 → 0.1.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -22
- package/cordis.patch.yml +25 -16
- package/lib/client.js +2144 -1571
- package/lib/doc-page.js +40 -19
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +27 -2
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +7 -4
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +52 -14
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/convert/types.d.ts +2 -2
- package/lib/types/convert/types.d.ts.map +1 -1
- package/lib/types/convert/types.js +2 -2
- package/lib/types/convert/types.js.map +1 -1
- package/lib/types/diagnosis/cards.js +2 -2
- package/lib/types/diagnosis/run.d.ts +7 -1
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +8 -2
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/seed-source.js +1 -1
- package/lib/types/diagnosis/seed.d.ts +1 -1
- package/lib/types/diagnosis/seed.js +1 -1
- package/lib/types/diagnosis/store.d.ts +13 -31
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +33 -71
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +1 -1
- package/lib/types/diagnosis/types.js +1 -1
- package/lib/types/extract-model.d.ts +67 -27
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js +225 -97
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract.d.ts +16 -0
- package/lib/types/extract.d.ts.map +1 -1
- package/lib/types/extract.js +19 -2
- package/lib/types/extract.js.map +1 -1
- package/lib/types/fs-atomic.d.ts +3 -4
- package/lib/types/fs-atomic.d.ts.map +1 -1
- package/lib/types/fs-atomic.js +3 -4
- package/lib/types/fs-atomic.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +302 -89
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/types.js +1 -1
- package/lib/types/intent/types.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts +13 -10
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +18 -27
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/layout.d.ts +3 -3
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +4 -4
- package/lib/types/layout.js.map +1 -1
- package/lib/types/migrate.d.ts +1 -1
- package/lib/types/migrate.js +2 -2
- package/lib/types/migrate.js.map +1 -1
- package/lib/types/model-text.d.ts +99 -0
- package/lib/types/model-text.d.ts.map +1 -0
- package/lib/types/model-text.js +103 -0
- package/lib/types/model-text.js.map +1 -0
- package/lib/types/present.d.ts +239 -0
- package/lib/types/present.d.ts.map +1 -0
- package/lib/types/present.js +255 -0
- package/lib/types/present.js.map +1 -0
- package/lib/types/recall-rank.d.ts +37 -0
- package/lib/types/recall-rank.d.ts.map +1 -1
- package/lib/types/recall-rank.js +41 -1
- package/lib/types/recall-rank.js.map +1 -1
- package/lib/types/runtime.d.ts +419 -78
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +2104 -334
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/settings-schema.d.ts +2 -0
- package/lib/types/settings-schema.d.ts.map +1 -1
- package/lib/types/settings-schema.js +2 -0
- package/lib/types/settings-schema.js.map +1 -1
- package/lib/types/source-images.d.ts +118 -0
- package/lib/types/source-images.d.ts.map +1 -0
- package/lib/types/source-images.js +288 -0
- package/lib/types/source-images.js.map +1 -0
- package/lib/types/types.d.ts +316 -39
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +57 -11
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-finalize.d.ts +140 -0
- package/lib/types/wiki-finalize.d.ts.map +1 -0
- package/lib/types/wiki-finalize.js +269 -0
- package/lib/types/wiki-finalize.js.map +1 -0
- package/lib/types/wiki-types.d.ts +2 -1
- package/lib/types/wiki-types.d.ts.map +1 -1
- package/lib/types/wiki-types.js +13 -7
- package/lib/types/wiki-types.js.map +1 -1
- package/package.json +3 -3
- package/preset/tita-data-agent/agent.cordis.yml +25 -16
- package/src/client/DrawerSection.tsx +18 -0
- package/src/client/KnowledgeBaseTab.tsx +294 -104
- package/src/client/KnowledgePanel.tsx +4 -3
- package/src/client/KnowledgeToolRow.tsx +331 -0
- package/src/client/PreviewSections.tsx +115 -34
- package/src/client/RecallPanel.tsx +418 -127
- package/src/client/TypeDefinitionDrawer.tsx +82 -16
- package/src/client/WikiCardReview.tsx +266 -0
- package/src/client/WikiFlowCard.tsx +57 -95
- package/src/client/WikiTypeEditor.tsx +6 -20
- package/src/client/arch/AgentArchitectureGraph.tsx +22 -135
- package/src/client/architecture.css +22 -27
- package/src/client/architecture.tsx +12 -12
- package/src/client/asset-images.ts +125 -0
- package/src/client/attribution/AttributionTab.tsx +4 -10
- package/src/client/attribution/StageChrome.tsx +8 -24
- package/src/client/convert/ConvertPanel.tsx +30 -3
- package/src/client/doc-page.tsx +6 -27
- package/src/client/index.ts +18 -0
- package/src/client/problem-domains.ts +1 -1
- package/src/client/styles.css +80 -11
- package/src/convert/host.ts +27 -2
- package/src/convert/ingest.ts +61 -18
- package/src/convert/types.ts +2 -2
- package/src/diagnosis/cards.ts +2 -2
- package/src/diagnosis/run.ts +8 -2
- package/src/diagnosis/seed-source.ts +1 -1
- package/src/diagnosis/seed.ts +1 -1
- package/src/diagnosis/store.ts +32 -79
- package/src/diagnosis/types.ts +2 -2
- package/src/extract-model.ts +254 -99
- package/src/extract.ts +35 -3
- package/src/fs-atomic.ts +3 -4
- package/src/index.ts +338 -88
- package/src/intent/types.ts +1 -1
- package/src/knowledge-bases.ts +31 -32
- package/src/layout.ts +4 -4
- package/src/migrate.ts +2 -2
- package/src/model-text.ts +126 -0
- package/src/present.ts +367 -0
- package/src/recall-rank.ts +57 -2
- package/src/runtime.ts +2282 -374
- package/src/settings-schema.ts +2 -0
- package/src/source-images.ts +326 -0
- package/src/types.ts +354 -47
- package/src/wiki-finalize.ts +310 -0
- package/src/wiki-types.ts +13 -7
- package/lib/types/proposals.d.ts +0 -166
- package/lib/types/proposals.d.ts.map +0 -1
- package/lib/types/proposals.js +0 -238
- package/lib/types/proposals.js.map +0 -1
- package/src/client/ProposalReview.tsx +0 -313
- package/src/proposals.ts +0 -337
package/src/extract-model.ts
CHANGED
|
@@ -1,47 +1,63 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* 模型驱动的**按 schema 实体抽取与归并**(#3b / 0044)。
|
|
3
3
|
*
|
|
4
|
-
* ##
|
|
4
|
+
* ## 它解决的问题
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
* 于是同一件东西只要换个标题写法,就是两个实体。这不是理论问题,是实测出来的:
|
|
6
|
+
* 规则路径(`extract.ts`)拿 `##` 小节标题当实体名。实测的后果有两个,都不是理论问题:
|
|
8
7
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
8
|
+
* 1. **抽出来的是目录,不是知识。** 年报里的小节叫「毛利率变动及其驱动因素」,
|
|
9
|
+
* 口径手册里的小节叫「毛利率(口径补充)」,于是同一件东西落成两张互不相干的卡片
|
|
10
|
+
* (`新建 2 / 并入 0`),而「实体已存在就追加正文」那条分支**永远不会触发**。
|
|
11
|
+
* 想去掉括号后缀、去掉「及其驱动因素」,是语义判断,阈值化的字符串规则在第二步
|
|
12
|
+
* 必然出错(`应收账款周转天数` 与 `库存周转天数` 字面都带「周转天数」)。
|
|
13
|
+
* 2. **卡片正文是切下来的原文。** 一张 `metric` 卡片应该有「口径定义 / 计算逻辑」两个
|
|
14
|
+
* 必填小节,而按标题切出来的正文与类型定义毫无关系 —— 它长得像一篇文章,
|
|
15
|
+
* 不像一张卡片。0044 之前用户在界面上看到的「编译只是按目录截取」说的就是这件事。
|
|
13
16
|
*
|
|
14
|
-
*
|
|
15
|
-
* `毛利率变动及其驱动因素` 是同一个东西,而 `应收账款周转天数` 与 `库存周转天数`
|
|
16
|
-
* 不是 —— 两者字面都带「周转天数」。阈值化的字符串规则在第二步上必然出错,
|
|
17
|
-
* 所以这件事必须交给模型。
|
|
17
|
+
* 所以这一层让模型回答三件它擅长、而规则做不好的事:
|
|
18
18
|
*
|
|
19
|
-
*
|
|
19
|
+
* 1. 「这份来源里有哪些**实体**」;
|
|
20
|
+
* 2. 「每个实体按它所属类型的 schema **该写成什么样**」—— 必填小节当骨架、必填字段当值;
|
|
21
|
+
* 3. 「这个实体是不是已经在下表里了」(给出已有卡片的 `existing_id`)。
|
|
20
22
|
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
+
* 第 3 问的**判断权在模型,决定权在这里**:模型说「是同一个」,我们就走追加;
|
|
24
|
+
* 说「不是」,就新建。模型给出的 `existing_id` 必须能在已有卡片里找到,
|
|
25
|
+
* 找不到就退回新建 —— 不能让模型凭一个不存在的 id 把内容写丢。
|
|
23
26
|
*
|
|
24
|
-
*
|
|
25
|
-
* 2. 「这个实体是不是已经在下表里了」(给出已有卡片的 `existing_id`)。
|
|
27
|
+
* ## 缺信息也抽(0044)
|
|
26
28
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
29
|
+
* 0040 那一版要求每条候选「必须带正文」,「必须来自已登记类型」,否则整条丢掉。
|
|
30
|
+
* 结果是**知识进不来**:一份只写了半句口径的文档,抽出来的东西全被闸门拒了,
|
|
31
|
+
* 而「半句口径」本身正是最该记下来的东西 —— 后面再来一份文档就能补上。
|
|
32
|
+
*
|
|
33
|
+
* 现在只做两件事,不做拒绝:
|
|
34
|
+
* - 类型不在注册表里 → 退回一个已启用的兜底类型 + 一条 warning(**不丢**);
|
|
35
|
+
* - 正文为空 → 用 `summary` 撑起一个带必填小节骨架的最小正文(**不丢**)。
|
|
36
|
+
* 缺什么,由 `schema_issues` 如实标出来,人在 review 时补。
|
|
30
37
|
*
|
|
31
38
|
* ## 为什么要注入 `call`
|
|
32
39
|
*
|
|
33
40
|
* 模型调用是这一层唯一的非纯部分。把它作为参数注入,这个模块就能在没有模型的环境里
|
|
34
|
-
* 被完整测试(见 `tests/extract-model.test.mjs
|
|
35
|
-
*
|
|
41
|
+
* 被完整测试(见 `tests/extract-model.test.mjs`)——否则「schema 是否真的传给了模型」
|
|
42
|
+
* 这件事只能靠真跑一次模型来验,而那是不可重复的。
|
|
36
43
|
*
|
|
37
44
|
* @module
|
|
38
45
|
*/
|
|
39
46
|
|
|
40
47
|
import { candidateEntityId, type ExtractionCandidate } from './extract.js'
|
|
48
|
+
import { templateCardFor } from './wiki-types.js'
|
|
41
49
|
import type { KnowledgeCard, SchemaType, SourceRaw, WikiTypeDef } from './types.js'
|
|
42
50
|
|
|
43
|
-
/**
|
|
44
|
-
|
|
51
|
+
/**
|
|
52
|
+
* 一次纯文本模型调用:给出 system 与 prompt,拿回模型的原始文本。
|
|
53
|
+
*
|
|
54
|
+
* `sessionId` 是**路由与缓存亲和**的身份,不是对话(0044 实机验证第五批)。
|
|
55
|
+
* OpenCode 系的路由要求请求带 `x-opencode-session`,而 `dsh-opencode-session` 插件
|
|
56
|
+
* 只在 `options.sessionId` 存在时才打那个头 —— 缺了它,400 `MissingSessionID`
|
|
57
|
+
* 会一路伪装成「模型抽取不可用」。这里把它做成可选字段:假模型不必关心,
|
|
58
|
+
* 真接缝按知识库给一个稳定值。
|
|
59
|
+
*/
|
|
60
|
+
export type ModelCall = (request: { system: string; prompt: string; sessionId?: string }) => Promise<string>
|
|
45
61
|
|
|
46
62
|
/** 归并条目:与规则抽取的 `NormalizedCandidates['merges']` 保持同一形状。 */
|
|
47
63
|
export interface ModelMerge {
|
|
@@ -50,6 +66,8 @@ export interface ModelMerge {
|
|
|
50
66
|
entityType: SchemaType
|
|
51
67
|
sourceId: string
|
|
52
68
|
body: string
|
|
69
|
+
summary?: string
|
|
70
|
+
fields?: Record<string, string>
|
|
53
71
|
}
|
|
54
72
|
|
|
55
73
|
export interface ModelExtractionResult {
|
|
@@ -62,15 +80,28 @@ export interface ModelExtractionResult {
|
|
|
62
80
|
|
|
63
81
|
export interface ModelExtractionInput {
|
|
64
82
|
source: SourceRaw
|
|
65
|
-
/** 已启用的 wiki 类型:模型只能从这些 id
|
|
83
|
+
/** 已启用的 wiki 类型:模型只能从这些 id 里选,而且它们的 schema 就是卡片骨架。 */
|
|
66
84
|
wikiTypes: WikiTypeDef[]
|
|
67
85
|
/** 当前库里的卡片:既是「已存在」的候选集,也是给模型的上下文。 */
|
|
68
86
|
existing: KnowledgeCard[]
|
|
69
87
|
call: ModelCall
|
|
88
|
+
/**
|
|
89
|
+
* 路由与缓存亲和的身份,按知识库稳定(`knowledge:<库 id>`)。
|
|
90
|
+
*
|
|
91
|
+
* 由运行时给:**只有它知道自己在写哪个库**。接缝层拿到什么就传什么,
|
|
92
|
+
* 没有时退回 `knowledge:default`(假模型、独立 runtime 的测试都属于这一档)。
|
|
93
|
+
*/
|
|
94
|
+
sessionId?: string
|
|
70
95
|
}
|
|
71
96
|
|
|
72
|
-
/**
|
|
73
|
-
const BODY_LIMIT =
|
|
97
|
+
/** 单个实体给模型的正文长度上限:够判断语义,又不至于把上下文撑爆。 */
|
|
98
|
+
const BODY_LIMIT = 2000
|
|
99
|
+
|
|
100
|
+
/** 文档给模型的长度上限。 */
|
|
101
|
+
const DOCUMENT_LIMIT = BODY_LIMIT * 8
|
|
102
|
+
|
|
103
|
+
/** 已有卡片目录给模型多少行。 */
|
|
104
|
+
const CATALOG_LIMIT = 200
|
|
74
105
|
|
|
75
106
|
/**
|
|
76
107
|
* 系统提示。**刻意写死在这里**而不是做成设置项:它不是用户的偏好,是这个模块与模型
|
|
@@ -78,16 +109,21 @@ const BODY_LIMIT = 1200
|
|
|
78
109
|
* 解析器改坏。
|
|
79
110
|
*/
|
|
80
111
|
const SYSTEM = [
|
|
81
|
-
'You
|
|
112
|
+
'You turn one business document into wiki cards for a knowledge base.',
|
|
82
113
|
'Answer with a single JSON object and nothing else. No prose, no code fence.',
|
|
83
|
-
'Shape: {"entities":[{"name":string,"type":string,"summary":string,"
|
|
114
|
+
'Shape: {"entities":[{"name":string,"type":string,"summary":string,"content":string,"fields":{string:string},"existing_id":string|null}]}',
|
|
84
115
|
'Rules:',
|
|
85
|
-
'- "name" is the entity name as a business person would say it (e.g. 毛利率).
|
|
116
|
+
'- "name" is the entity name as a business person would say it (e.g. 毛利率). Never a section heading.',
|
|
86
117
|
'- "type" MUST be one of the allowed type ids given in the prompt.',
|
|
87
|
-
'- "
|
|
118
|
+
'- "content" MUST be markdown for that type: use its required section labels as "## <label>" headings, in order.',
|
|
119
|
+
' Fill each section with facts stated in the document. Copy wording, do not paraphrase loosely.',
|
|
120
|
+
' If the document does not say enough for a section, still write the heading and put only what IS known.',
|
|
121
|
+
' NEVER invent numbers, owners, formulas or dates. Missing information is acceptable; made-up information is not.',
|
|
122
|
+
'- "fields" holds the type\'s required fields whose values the document states. Omit a field the document does not answer.',
|
|
123
|
+
'- "summary" is one sentence saying what this entity is.',
|
|
88
124
|
'- "existing_id" is the entityId of the existing card this entity is the SAME thing as, or null.',
|
|
89
125
|
' Same thing means the same business concept; a narrower or different concept is NOT the same.',
|
|
90
|
-
'- Prefer few, real entities over many
|
|
126
|
+
'- Prefer few, real entities over many sections. Extract an entity even when the document only covers part of it.',
|
|
91
127
|
].join('\n')
|
|
92
128
|
|
|
93
129
|
/** 从一串可能带 ``` 围栏的文本里取出 JSON 对象并解析。 */
|
|
@@ -110,100 +146,227 @@ function parseJsonObject(raw: string): unknown {
|
|
|
110
146
|
}
|
|
111
147
|
}
|
|
112
148
|
|
|
113
|
-
/** 把模型的回答收敛成「合法实体数组」;任何不合法的一条都丢掉并记一条 warning。 */
|
|
114
|
-
function readEntities(raw: unknown, allowedTypes: Set<string>, warnings: string[]): Array<Record<string, unknown>> {
|
|
115
|
-
if (raw === null || typeof raw !== 'object') {
|
|
116
|
-
warnings.push('模型输出不是 JSON 对象,本次抽取按空处理')
|
|
117
|
-
return []
|
|
118
|
-
}
|
|
119
|
-
const list = (raw as { entities?: unknown }).entities
|
|
120
|
-
if (!Array.isArray(list)) {
|
|
121
|
-
warnings.push('模型输出里没有 entities 数组,本次抽取按空处理')
|
|
122
|
-
return []
|
|
123
|
-
}
|
|
124
|
-
const kept: Array<Record<string, unknown>> = []
|
|
125
|
-
for (const item of list) {
|
|
126
|
-
if (item === null || typeof item !== 'object') continue
|
|
127
|
-
const record = item as Record<string, unknown>
|
|
128
|
-
const name = typeof record.name === 'string' ? record.name.trim() : ''
|
|
129
|
-
const type = typeof record.type === 'string' ? record.type.trim() : ''
|
|
130
|
-
const body = typeof record.body === 'string' ? record.body.trim() : ''
|
|
131
|
-
if (!name) continue
|
|
132
|
-
if (!allowedTypes.has(type)) {
|
|
133
|
-
warnings.push(`模型给出的类型「${type}」不在已启用类型里,候选「${name}」被丢掉`)
|
|
134
|
-
continue
|
|
135
|
-
}
|
|
136
|
-
if (!body) {
|
|
137
|
-
warnings.push(`候选「${name}」没有正文,被丢掉(卡片不能没有正文)`)
|
|
138
|
-
continue
|
|
139
|
-
}
|
|
140
|
-
kept.push({ ...record, name, type, body })
|
|
141
|
-
}
|
|
142
|
-
return kept
|
|
143
|
-
}
|
|
144
|
-
|
|
145
149
|
/**
|
|
146
|
-
*
|
|
150
|
+
* 一个类型的 schema 怎么讲给模型听。
|
|
147
151
|
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
152
|
+
* 这是 0044 的关键一步:**必填小节与必填字段必须出现在提示词里**,否则模型写出来的
|
|
153
|
+
* 「正文」只能是一段自由文本 —— 而卡片是按小节校验的。导出它,是为了让
|
|
154
|
+
* 「schema 到底有没有下发」这件事可以在测试里被断言,而不是只能靠读代码相信。
|
|
150
155
|
*/
|
|
151
|
-
export
|
|
152
|
-
const {
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
+
export function renderTypeSpec(def: WikiTypeDef): string {
|
|
157
|
+
const sections = def.sections.map(section => `${section.label}${section.required ? '(必填)' : '(可选)'}`).join(' / ')
|
|
158
|
+
const fields = def.fields.filter(field => field.required).map(field => `${field.name}(${field.note})`).join(';')
|
|
159
|
+
return [
|
|
160
|
+
`- ${def.id} :: ${def.label}${def.description ? ` — ${def.description}` : ''}`,
|
|
161
|
+
sections ? ` 小节:${sections}` : '',
|
|
162
|
+
fields ? ` 必填字段:${fields}` : '',
|
|
163
|
+
def.examples.length > 0 ? ` 例如:${def.examples.join(';')}` : '',
|
|
164
|
+
` 模板:\n${indent(templateCardFor(def.sections, def.fields))}`,
|
|
165
|
+
].filter(Boolean).join('\n')
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function indent(value: string): string {
|
|
169
|
+
return value.split('\n').map(line => ` ${line}`).join('\n')
|
|
170
|
+
}
|
|
156
171
|
|
|
172
|
+
/** 抽取用的完整提示词。导出以便测试断言「schema 真的在里面」。 */
|
|
173
|
+
export function buildExtractionPrompt(input: Pick<ModelExtractionInput, 'source' | 'wikiTypes' | 'existing'>): string {
|
|
174
|
+
const { source, wikiTypes, existing } = input
|
|
157
175
|
const catalog = existing
|
|
158
|
-
.slice(0,
|
|
176
|
+
.slice(0, CATALOG_LIMIT)
|
|
159
177
|
.map(card => `- ${card.entityId} :: ${card.title}(${card.entityType})`)
|
|
160
178
|
.join('\n')
|
|
161
|
-
|
|
162
|
-
.map(def => `- ${def.id} :: ${def.label}${def.description ? ` — ${def.description}` : ''}`)
|
|
163
|
-
.join('\n')
|
|
164
|
-
|
|
165
|
-
const prompt = [
|
|
179
|
+
return [
|
|
166
180
|
'# Allowed wiki type ids',
|
|
167
|
-
|
|
181
|
+
wikiTypes.map(renderTypeSpec).join('\n') || '(没有任何已启用类型)',
|
|
168
182
|
'',
|
|
169
|
-
'# Existing wiki cards (entityId :: title)',
|
|
183
|
+
'# Existing wiki cards (entityId :: title(type))',
|
|
170
184
|
catalog || '(这个库还是空的)',
|
|
171
185
|
'',
|
|
172
186
|
'# Document',
|
|
173
187
|
`title: ${source.title}`,
|
|
174
188
|
`source_id: ${source.sourceId}`,
|
|
175
189
|
'',
|
|
176
|
-
source.content.slice(0,
|
|
190
|
+
source.content.slice(0, DOCUMENT_LIMIT),
|
|
177
191
|
].join('\n')
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** 把模型的 `fields` 收敛成一个字符串到字符串的字典;非字符串值丢掉。 */
|
|
195
|
+
function readFields(raw: unknown): Record<string, string> {
|
|
196
|
+
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) return {}
|
|
197
|
+
const result: Record<string, string> = {}
|
|
198
|
+
for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
|
|
199
|
+
const name = key.trim()
|
|
200
|
+
if (!name) continue
|
|
201
|
+
if (typeof value === 'string' && value.trim()) result[name] = value.trim()
|
|
202
|
+
else if (typeof value === 'number' && Number.isFinite(value)) result[name] = String(value)
|
|
203
|
+
}
|
|
204
|
+
return result
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* 一个已启用的兜底类型。
|
|
209
|
+
*
|
|
210
|
+
* 「类型不认识」不该丢实体 —— 0040 那一版就是在这一步丢的。退回 `concept`,
|
|
211
|
+
* 它不在注册表里就退第一个启用的类型;一个都没有(注册表被清空)时返回 undefined,
|
|
212
|
+
* 那时调用方也没有别的选择。
|
|
213
|
+
*/
|
|
214
|
+
function fallbackType(enabled: WikiTypeDef[]): WikiTypeDef | undefined {
|
|
215
|
+
return enabled.find(def => def.id === 'concept') ?? enabled[0]
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
interface ReadEntityOptions {
|
|
219
|
+
enabled: WikiTypeDef[]
|
|
220
|
+
warnings: string[]
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* 把模型的一条输出收敛成候选;不合法的一条记 warning 而不是静默丢弃。
|
|
225
|
+
*
|
|
226
|
+
* 「缺信息也抽」在这里落成代码:**名字是唯一的硬要求**。类型不认识就退回兜底、
|
|
227
|
+
* 正文为空就用 summary 撑一个带必填小节骨架的最小正文。
|
|
228
|
+
*/
|
|
229
|
+
function readEntity(item: unknown, options: ReadEntityOptions): { candidate: ExtractionCandidate; existingId?: string } | undefined {
|
|
230
|
+
const { enabled, warnings } = options
|
|
231
|
+
if (item === null || typeof item !== 'object') return undefined
|
|
232
|
+
const record = item as Record<string, unknown>
|
|
233
|
+
const name = typeof record.name === 'string' ? record.name.trim() : ''
|
|
234
|
+
if (!name) return undefined
|
|
235
|
+
|
|
236
|
+
const wanted = typeof record.type === 'string' ? record.type.trim() : ''
|
|
237
|
+
const matched = enabled.find(def => def.id === wanted)
|
|
238
|
+
let type = matched?.id as SchemaType | undefined
|
|
239
|
+
if (!type) {
|
|
240
|
+
const fallback = fallbackType(enabled)
|
|
241
|
+
if (!fallback) {
|
|
242
|
+
warnings.push(`候选「${name}」没有任何已启用类型可用,被丢掉`)
|
|
243
|
+
return undefined
|
|
244
|
+
}
|
|
245
|
+
warnings.push(`候选「${name}」的类型「${wanted || '(未给)'}」不在已启用类型里,已按「${fallback.id}」入库`)
|
|
246
|
+
type = fallback.id as SchemaType
|
|
247
|
+
}
|
|
248
|
+
const def = enabled.find(candidate => candidate.id === type)!
|
|
249
|
+
|
|
250
|
+
// 正文:优先用模型写的 `content`,兼容旧字段名 `body`。
|
|
251
|
+
const rawBody = typeof record.content === 'string' ? record.content.trim() : typeof record.body === 'string' ? record.body.trim() : ''
|
|
252
|
+
const summary = typeof record.summary === 'string' ? record.summary.trim() : ''
|
|
253
|
+
const content = rawBody ? rawBody.slice(0, BODY_LIMIT) : skeletonBody(def, summary)
|
|
254
|
+
|
|
255
|
+
const existingId = typeof record.existing_id === 'string' ? record.existing_id.trim() : ''
|
|
256
|
+
const fields = readFields(record.fields)
|
|
257
|
+
return {
|
|
258
|
+
existingId: existingId || undefined,
|
|
259
|
+
candidate: {
|
|
260
|
+
kind: type === 'concept' ? 'concept' : 'entity',
|
|
261
|
+
name,
|
|
262
|
+
type,
|
|
263
|
+
body: content,
|
|
264
|
+
...(summary ? { summary } : {}),
|
|
265
|
+
...(Object.keys(fields).length > 0 ? { fields } : {}),
|
|
266
|
+
aliases: [],
|
|
267
|
+
evidence: { sourceId: '' },
|
|
268
|
+
confidence: existingId ? 0.75 : 0.7,
|
|
269
|
+
},
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* 正文为空时的最小骨架。
|
|
275
|
+
*
|
|
276
|
+
* 不是「占位符」:占位符(`(必填:写什么)`)会被编译进召回单元、塞进模型上下文,
|
|
277
|
+
* 等于管线自己造噪声。这里只放**真实存在的信息** —— 模型给的 summary。
|
|
278
|
+
*
|
|
279
|
+
* 连 summary 都没有时返回空串。**这一层不替编译侧做丢弃决定**:它照样把候选交出去,
|
|
280
|
+
* 由 `runtime.compileSource` 按「正文是不是真的空」统一裁决(那是唯一保留的拒绝)。
|
|
281
|
+
* 在这里悄悄丢掉一条候选,会让「为什么这份文档一张卡都没抽出来」变成一个查不到答案的问题。
|
|
282
|
+
*/
|
|
283
|
+
export function skeletonBody(def: WikiTypeDef, summary: string): string {
|
|
284
|
+
const first = def.sections[0]?.label ?? '定义'
|
|
285
|
+
return summary ? `## ${first}\n\n${summary}` : ''
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* 用模型抽取实体,并把它们判归到「新建」或「追加到已有卡片」两条路上。
|
|
290
|
+
*
|
|
291
|
+
* 失败一律降级而不抛出:模型不可用时返回空结果 + 一条 warning,让上层继续跑规则抽取。
|
|
292
|
+
* 「因为模型挂了所以整次编译失败」是不能接受的 —— 那条链路本来就不依赖模型。
|
|
293
|
+
*/
|
|
294
|
+
/**
|
|
295
|
+
* 模型实际回了什么 —— 只留开头一小段,把 warning 变成可排查的东西。
|
|
296
|
+
*
|
|
297
|
+
* 「模型输出不是 JSON 对象」这句话本身没有下一步:是模型在讲道理、是被截断,
|
|
298
|
+
* 还是接缝根本没取到文字?三种情况的处理完全不同,而在实机上它们长得一模一样。
|
|
299
|
+
* 贴上开头 160 字符,第一次就能分辨。
|
|
300
|
+
*/
|
|
301
|
+
function outputPreview(raw: string): string {
|
|
302
|
+
const flat = raw.replace(/\s+/g, ' ').trim()
|
|
303
|
+
if (flat.length === 0) return '(模型回了空字符串)'
|
|
304
|
+
return flat.length > 160 ? `${flat.slice(0, 160)}…` : flat
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
export async function extractWithModel(input: ModelExtractionInput): Promise<ModelExtractionResult> {
|
|
308
|
+
const { source, wikiTypes, existing, call, sessionId } = input
|
|
309
|
+
const warnings: string[] = []
|
|
310
|
+
const enabled = wikiTypes.filter(def => def.enabled)
|
|
311
|
+
if (enabled.length === 0) {
|
|
312
|
+
return { fresh: [], merges: [], warnings: ['没有任何已启用的 wiki 类型,模型抽取已跳过(先在设置里登记一个类型)'] }
|
|
313
|
+
}
|
|
178
314
|
|
|
179
315
|
let raw: string
|
|
180
316
|
try {
|
|
181
|
-
raw = await call({
|
|
317
|
+
raw = await call({
|
|
318
|
+
system: SYSTEM,
|
|
319
|
+
prompt: buildExtractionPrompt({ source, wikiTypes: enabled, existing }),
|
|
320
|
+
...(sessionId ? { sessionId } : {}),
|
|
321
|
+
})
|
|
182
322
|
} catch (cause) {
|
|
183
323
|
warnings.push(`模型抽取不可用:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
184
324
|
return { fresh: [], merges: [], warnings }
|
|
185
325
|
}
|
|
186
326
|
|
|
327
|
+
const parsed = parseJsonObject(raw)
|
|
328
|
+
if (parsed === null || typeof parsed !== 'object') {
|
|
329
|
+
warnings.push(`模型输出不是 JSON 对象,本次抽取按空处理。模型实际回了:${outputPreview(raw)}`)
|
|
330
|
+
return { fresh: [], merges: [], warnings }
|
|
331
|
+
}
|
|
332
|
+
const list = (parsed as { entities?: unknown }).entities
|
|
333
|
+
if (!Array.isArray(list)) {
|
|
334
|
+
warnings.push('模型输出里没有 entities 数组,本次抽取按空处理')
|
|
335
|
+
return { fresh: [], merges: [], warnings }
|
|
336
|
+
}
|
|
337
|
+
|
|
187
338
|
const byId = new Map(existing.map(card => [`${card.entityType}:${card.entityId}`, card]))
|
|
188
339
|
const byEntityId = new Map(existing.map(card => [card.entityId, card]))
|
|
189
340
|
const fresh: ExtractionCandidate[] = []
|
|
190
341
|
const merges: ModelMerge[] = []
|
|
191
342
|
const claimed = new Set<string>()
|
|
192
343
|
|
|
193
|
-
for (const entity of
|
|
194
|
-
const
|
|
195
|
-
|
|
196
|
-
const
|
|
197
|
-
|
|
344
|
+
for (const entity of list) {
|
|
345
|
+
const read = readEntity(entity, { enabled, warnings })
|
|
346
|
+
if (!read) continue
|
|
347
|
+
const { candidate, existingId } = read
|
|
348
|
+
candidate.evidence = { sourceId: source.sourceId }
|
|
349
|
+
const name = candidate.name
|
|
350
|
+
const type = candidate.type
|
|
351
|
+
const body = candidate.body ?? ''
|
|
198
352
|
// 模型给的 id 必须真的存在:它指着不存在的卡片时退回新建,绝不能让正文丢在空中。
|
|
199
|
-
const target = (
|
|
353
|
+
const target = (existingId ? byEntityId.get(existingId) : undefined) ?? byId.get(`${type}:${name}`)
|
|
200
354
|
|
|
201
355
|
if (target) {
|
|
202
356
|
const key = `${target.entityType}:${target.entityId}`
|
|
203
357
|
// 同一张卡在一次抽取里被多个候选指向时只并一次,正文在这里先合起来。
|
|
204
358
|
const already = merges.find(entry => entry.key === key)
|
|
205
|
-
if (already)
|
|
206
|
-
|
|
359
|
+
if (already) {
|
|
360
|
+
already.body = `${already.body}\n\n${body}`
|
|
361
|
+
if (!already.summary && candidate.summary) already.summary = candidate.summary
|
|
362
|
+
already.fields = { ...(already.fields ?? {}), ...(candidate.fields ?? {}) }
|
|
363
|
+
} else {
|
|
364
|
+
merges.push({
|
|
365
|
+
key, entityId: target.entityId, entityType: target.entityType, sourceId: source.sourceId, body,
|
|
366
|
+
...(candidate.summary ? { summary: candidate.summary } : {}),
|
|
367
|
+
...(candidate.fields && Object.keys(candidate.fields).length > 0 ? { fields: candidate.fields } : {}),
|
|
368
|
+
})
|
|
369
|
+
}
|
|
207
370
|
claimed.add(key)
|
|
208
371
|
continue
|
|
209
372
|
}
|
|
@@ -212,15 +375,7 @@ export async function extractWithModel(input: ModelExtractionInput): Promise<Mod
|
|
|
212
375
|
const key = `${type}:${entityId}`
|
|
213
376
|
if (claimed.has(key)) continue
|
|
214
377
|
claimed.add(key)
|
|
215
|
-
fresh.push(
|
|
216
|
-
kind: 'entity',
|
|
217
|
-
name,
|
|
218
|
-
type,
|
|
219
|
-
body,
|
|
220
|
-
aliases: [],
|
|
221
|
-
evidence: { sourceId: source.sourceId },
|
|
222
|
-
confidence: 0.7,
|
|
223
|
-
})
|
|
378
|
+
fresh.push(candidate)
|
|
224
379
|
}
|
|
225
380
|
|
|
226
381
|
return { fresh, merges, warnings }
|
package/src/extract.ts
CHANGED
|
@@ -34,6 +34,20 @@ export interface ExtractionCandidate {
|
|
|
34
34
|
aliases?: string[]
|
|
35
35
|
/** Markdown body that backs this candidate (used as the card body). */
|
|
36
36
|
body?: string
|
|
37
|
+
/**
|
|
38
|
+
* 一句话摘要(0044)。模型按 schema 抽取时会给出;规则路径不产它。
|
|
39
|
+
*
|
|
40
|
+
* 它有独立价值:正文还没写全时,摘要是这张卡片**唯一能说的那句话** —— 缺信息也入库
|
|
41
|
+
* 时,卡片至少还有它。
|
|
42
|
+
*/
|
|
43
|
+
summary?: string
|
|
44
|
+
/**
|
|
45
|
+
* 类型定义里的字段值(0044),例如 `metric` 的 `formula` / `owner`。
|
|
46
|
+
*
|
|
47
|
+
* 落在卡片 front matter 的 `metadata` 里,不占正文。模型按 schema 抽取时填;
|
|
48
|
+
* 规则路径不产它,所以缺字段的卡片会在 `schema_issues` 里如实标出来等人补。
|
|
49
|
+
*/
|
|
50
|
+
fields?: Record<string, string>
|
|
37
51
|
/** For kind === 'relation': the other end. */
|
|
38
52
|
target?: { name: string; type?: SchemaType }
|
|
39
53
|
/** For kind === 'relation': the relation id, e.g. 'relates_to'. */
|
|
@@ -320,6 +334,18 @@ function entryBody(entry: unknown): string | undefined {
|
|
|
320
334
|
return typeof value === 'string' && value.trim() ? value.trim() : undefined
|
|
321
335
|
}
|
|
322
336
|
|
|
337
|
+
/**
|
|
338
|
+
* 声明的摘要(0044)。
|
|
339
|
+
*
|
|
340
|
+
* 有了它,一条「只知道叫什么、还写不全」的声明式候选也能落成卡片:正文由摘要撑成
|
|
341
|
+
* 一个带必填小节骨架的最小正文(`runtime.compileSource` 的 `skeletonBody`)。
|
|
342
|
+
* 没有它,声明式候选只能靠正文自重 —— 而那正是「缺信息就不入库」的老毛病。
|
|
343
|
+
*/
|
|
344
|
+
function entrySummary(entry: unknown): string | undefined {
|
|
345
|
+
const value = plainRecord(entry).summary
|
|
346
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined
|
|
347
|
+
}
|
|
348
|
+
|
|
323
349
|
function entryHeading(entry: unknown): string | undefined {
|
|
324
350
|
const value = plainRecord(entry).heading
|
|
325
351
|
return typeof value === 'string' && value.trim() ? value.trim() : undefined
|
|
@@ -372,11 +398,13 @@ export function extractCandidates(source: SourceRaw, wikiTypes: WikiTypeDef[]):
|
|
|
372
398
|
const aliases = stringList(plainRecord(entry).aliases)
|
|
373
399
|
const heading = entryHeading(entry) ?? sectionForName(sections, name)?.heading
|
|
374
400
|
const body = entryBody(entry) ?? sectionForName(sections, name)?.markdown
|
|
401
|
+
const summary = entrySummary(entry)
|
|
375
402
|
declaredEntities.push({
|
|
376
403
|
kind: kind === 'entity' ? 'entity' : kindFor(type), name, type,
|
|
377
404
|
evidence: evidence(heading, entryQuote(entry)),
|
|
378
405
|
...(aliases.length > 0 ? { aliases } : {}),
|
|
379
406
|
...(body ? { body } : {}),
|
|
407
|
+
...(summary ? { summary } : {}),
|
|
380
408
|
confidence: entryConfidence(entry, 0.95),
|
|
381
409
|
})
|
|
382
410
|
}
|
|
@@ -526,7 +554,7 @@ export interface NormalizedCandidates {
|
|
|
526
554
|
* 除了「记一笔来源」什么也做不了 —— 新写进来源里的内容会**静默消失**,卡片永远
|
|
527
555
|
* 停在第一次抽取时的样子(落点是 `runtime.ts` 的 `appendExtracted`)。
|
|
528
556
|
*/
|
|
529
|
-
merges: Array<{ key: string; entityId: string; entityType: SchemaType; sourceId: string; body: string }>
|
|
557
|
+
merges: Array<{ key: string; entityId: string; entityType: SchemaType; sourceId: string; body: string; summary?: string; fields?: Record<string, string> }>
|
|
530
558
|
/**
|
|
531
559
|
* Relation candidates are never merged — they become graph edges.
|
|
532
560
|
*
|
|
@@ -606,7 +634,11 @@ export function normalizeCandidates(candidates: ExtractionCandidate[], index: Ma
|
|
|
606
634
|
if (freshKeys.has(key)) continue
|
|
607
635
|
const existing = resolved.get(key)
|
|
608
636
|
if (existing) {
|
|
609
|
-
merges.push({
|
|
637
|
+
merges.push({
|
|
638
|
+
key, entityId: existing.entityId, entityType: existing.entityType, sourceId: candidate.evidence.sourceId, body: candidate.body ?? '',
|
|
639
|
+
...(candidate.summary ? { summary: candidate.summary } : {}),
|
|
640
|
+
...(candidate.fields && Object.keys(candidate.fields).length > 0 ? { fields: candidate.fields } : {}),
|
|
641
|
+
})
|
|
610
642
|
continue
|
|
611
643
|
}
|
|
612
644
|
fresh.push(candidate)
|
|
@@ -617,7 +649,7 @@ export function normalizeCandidates(candidates: ExtractionCandidate[], index: Ma
|
|
|
617
649
|
status: 'unreviewed',
|
|
618
650
|
content: candidate.body ?? '',
|
|
619
651
|
sourceRefs: [candidate.evidence.sourceId],
|
|
620
|
-
metadata: {},
|
|
652
|
+
metadata: { ...(candidate.fields ?? {}) },
|
|
621
653
|
updatedAt: '',
|
|
622
654
|
}
|
|
623
655
|
freshKeys.add(key)
|
package/src/fs-atomic.ts
CHANGED
|
@@ -4,10 +4,9 @@ import { dirname } from 'node:path'
|
|
|
4
4
|
/**
|
|
5
5
|
* 原子写:先写同目录的临时文件,再 `rename` 覆盖目标。
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* 都是被 runtime 消费的领域模块)。
|
|
7
|
+
* 抽成独立模块的理由是**宿主运行时与领域模块都要用它**:SourceRaw / 卡片 / 编译产物
|
|
8
|
+
* 由 runtime 写,一次排查的样本与迁移由 `diagnosis/` 写。把它放在 runtime 里会让
|
|
9
|
+
* `diagnosis/` 反过来依赖 runtime 的实现细节 —— 分层就倒过来了。
|
|
11
10
|
*/
|
|
12
11
|
|
|
13
12
|
export async function writeTextAtomic(path: string, content: string): Promise<void> {
|