dsh-data-domain-knowledge 0.1.0-alpha.3 → 0.1.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -22
- package/cordis.patch.yml +25 -16
- package/lib/client.js +2144 -1571
- package/lib/doc-page.js +40 -19
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +27 -2
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +7 -4
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +52 -14
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/convert/types.d.ts +2 -2
- package/lib/types/convert/types.d.ts.map +1 -1
- package/lib/types/convert/types.js +2 -2
- package/lib/types/convert/types.js.map +1 -1
- package/lib/types/diagnosis/cards.js +2 -2
- package/lib/types/diagnosis/run.d.ts +7 -1
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +8 -2
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/seed-source.js +1 -1
- package/lib/types/diagnosis/seed.d.ts +1 -1
- package/lib/types/diagnosis/seed.js +1 -1
- package/lib/types/diagnosis/store.d.ts +13 -31
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +33 -71
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +1 -1
- package/lib/types/diagnosis/types.js +1 -1
- package/lib/types/extract-model.d.ts +67 -27
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js +225 -97
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract.d.ts +16 -0
- package/lib/types/extract.d.ts.map +1 -1
- package/lib/types/extract.js +19 -2
- package/lib/types/extract.js.map +1 -1
- package/lib/types/fs-atomic.d.ts +3 -4
- package/lib/types/fs-atomic.d.ts.map +1 -1
- package/lib/types/fs-atomic.js +3 -4
- package/lib/types/fs-atomic.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +302 -89
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/types.js +1 -1
- package/lib/types/intent/types.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts +13 -10
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +18 -27
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/layout.d.ts +3 -3
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +4 -4
- package/lib/types/layout.js.map +1 -1
- package/lib/types/migrate.d.ts +1 -1
- package/lib/types/migrate.js +2 -2
- package/lib/types/migrate.js.map +1 -1
- package/lib/types/model-text.d.ts +99 -0
- package/lib/types/model-text.d.ts.map +1 -0
- package/lib/types/model-text.js +103 -0
- package/lib/types/model-text.js.map +1 -0
- package/lib/types/present.d.ts +239 -0
- package/lib/types/present.d.ts.map +1 -0
- package/lib/types/present.js +255 -0
- package/lib/types/present.js.map +1 -0
- package/lib/types/recall-rank.d.ts +37 -0
- package/lib/types/recall-rank.d.ts.map +1 -1
- package/lib/types/recall-rank.js +41 -1
- package/lib/types/recall-rank.js.map +1 -1
- package/lib/types/runtime.d.ts +419 -78
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +2104 -334
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/settings-schema.d.ts +2 -0
- package/lib/types/settings-schema.d.ts.map +1 -1
- package/lib/types/settings-schema.js +2 -0
- package/lib/types/settings-schema.js.map +1 -1
- package/lib/types/source-images.d.ts +118 -0
- package/lib/types/source-images.d.ts.map +1 -0
- package/lib/types/source-images.js +288 -0
- package/lib/types/source-images.js.map +1 -0
- package/lib/types/types.d.ts +316 -39
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +57 -11
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-finalize.d.ts +140 -0
- package/lib/types/wiki-finalize.d.ts.map +1 -0
- package/lib/types/wiki-finalize.js +269 -0
- package/lib/types/wiki-finalize.js.map +1 -0
- package/lib/types/wiki-types.d.ts +2 -1
- package/lib/types/wiki-types.d.ts.map +1 -1
- package/lib/types/wiki-types.js +13 -7
- package/lib/types/wiki-types.js.map +1 -1
- package/package.json +3 -3
- package/preset/tita-data-agent/agent.cordis.yml +25 -16
- package/src/client/DrawerSection.tsx +18 -0
- package/src/client/KnowledgeBaseTab.tsx +294 -104
- package/src/client/KnowledgePanel.tsx +4 -3
- package/src/client/KnowledgeToolRow.tsx +331 -0
- package/src/client/PreviewSections.tsx +115 -34
- package/src/client/RecallPanel.tsx +418 -127
- package/src/client/TypeDefinitionDrawer.tsx +82 -16
- package/src/client/WikiCardReview.tsx +266 -0
- package/src/client/WikiFlowCard.tsx +57 -95
- package/src/client/WikiTypeEditor.tsx +6 -20
- package/src/client/arch/AgentArchitectureGraph.tsx +22 -135
- package/src/client/architecture.css +22 -27
- package/src/client/architecture.tsx +12 -12
- package/src/client/asset-images.ts +125 -0
- package/src/client/attribution/AttributionTab.tsx +4 -10
- package/src/client/attribution/StageChrome.tsx +8 -24
- package/src/client/convert/ConvertPanel.tsx +30 -3
- package/src/client/doc-page.tsx +6 -27
- package/src/client/index.ts +18 -0
- package/src/client/problem-domains.ts +1 -1
- package/src/client/styles.css +80 -11
- package/src/convert/host.ts +27 -2
- package/src/convert/ingest.ts +61 -18
- package/src/convert/types.ts +2 -2
- package/src/diagnosis/cards.ts +2 -2
- package/src/diagnosis/run.ts +8 -2
- package/src/diagnosis/seed-source.ts +1 -1
- package/src/diagnosis/seed.ts +1 -1
- package/src/diagnosis/store.ts +32 -79
- package/src/diagnosis/types.ts +2 -2
- package/src/extract-model.ts +254 -99
- package/src/extract.ts +35 -3
- package/src/fs-atomic.ts +3 -4
- package/src/index.ts +338 -88
- package/src/intent/types.ts +1 -1
- package/src/knowledge-bases.ts +31 -32
- package/src/layout.ts +4 -4
- package/src/migrate.ts +2 -2
- package/src/model-text.ts +126 -0
- package/src/present.ts +367 -0
- package/src/recall-rank.ts +57 -2
- package/src/runtime.ts +2282 -374
- package/src/settings-schema.ts +2 -0
- package/src/source-images.ts +326 -0
- package/src/types.ts +354 -47
- package/src/wiki-finalize.ts +310 -0
- package/src/wiki-types.ts +13 -7
- package/lib/types/proposals.d.ts +0 -166
- package/lib/types/proposals.d.ts.map +0 -1
- package/lib/types/proposals.js +0 -238
- package/lib/types/proposals.js.map +0 -1
- package/src/client/ProposalReview.tsx +0 -313
- package/src/proposals.ts +0 -337
package/src/runtime.ts
CHANGED
|
@@ -3,33 +3,39 @@ import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
|
|
3
3
|
import { existsSync } from 'node:fs'
|
|
4
4
|
import { appendFile, mkdir, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
5
5
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
|
6
|
+
import { tmpdir } from 'node:os'
|
|
6
7
|
import { fileURLToPath } from 'node:url'
|
|
7
8
|
import {
|
|
8
9
|
buildSourceTypeCatalog, defaultSourceTypes, DEFAULT_KNOWLEDGE_CONFIG, deriveSourceExt,
|
|
9
10
|
FALLBACK_EMBEDDING_MODEL, FALLBACK_RERANK_MODEL, legacyResourceHome, SOURCE_TYPE_ID_PATTERN,
|
|
10
11
|
LOCAL_EMBEDDING_MODEL_DIR, LOCAL_RERANK_MODEL_DIR,
|
|
11
|
-
|
|
12
|
+
ANOMALY_SCORE, CARD_REVIEW_BASE_SCORE, REVIEWED_SCORE_FLOOR,
|
|
13
|
+
type CardAnnotation, type CardReview, type CardSchemaIssue, type ContextGraph, type ContextItem, type DocumentVersion, type Envelope,
|
|
12
14
|
type KnowledgeAssetRef, type KnowledgeCard, type KnowledgeConfig, type KnowledgeDocument,
|
|
13
15
|
type KnowledgeDocumentDetail, type KnowledgeError, type KnowledgeStatus, type KnowledgeUnit,
|
|
14
|
-
type RecallDiagnostics, type RecallEvidence, type RecallHit, type RecallMode, type RecallResult, type RecallRoute, type RecallRouteDiagnostic, type RebuildReport,
|
|
15
|
-
type RerankResult, type RuntimeKnowledgeConfig, type SchemaType, type SearchHit, type SourceRaw, type SourceRawRead, type SourceType,
|
|
16
|
+
type ContextBudgetReport, type RecallDiagnostics, type RecallEvidence, type RecallExpansion, type RecallHit, type RecallLedger, type RecallLedgerRow, type RecallMode, type RecallResult, type RecallTimings, type RecallRoute, type RecallRouteDiagnostic, type RebuildReport,
|
|
17
|
+
type RerankResult, type RuntimeKnowledgeConfig, type SchemaType, type SearchHit, type SourceMetadataField, type SourceOriginRef, type SourceProvenance, type SourceRaw, type SourceRawRead, type SourceType,
|
|
16
18
|
type SourceTypeCatalog, type SourceTypeDef, type WikiCardVersion, type WikiNote,
|
|
17
|
-
type
|
|
19
|
+
type WikiTypeCatalog, type WikiTypeDef,
|
|
18
20
|
} from './types.js'
|
|
19
21
|
import { defaultWikiTypes, validateCardAgainstType, WIKI_TYPE_RULES } from './wiki-types.js'
|
|
20
22
|
import { LEGACY_ENTITY_TYPES } from './diagnosis/schema.js'
|
|
21
|
-
import { CardProposalStore } from './proposals.js'
|
|
22
23
|
import { DiagnosisStore } from './diagnosis/store.js'
|
|
23
24
|
import { buildAliasIndex, candidateEntityId, extractCandidates, normalizeCandidates, RELATES_TO, type NormalizedCandidates } from './extract.js'
|
|
24
|
-
import { extractWithModel, type ModelCall } from './extract-model.js'
|
|
25
|
+
import { extractWithModel, skeletonBody, type ModelCall } from './extract-model.js'
|
|
25
26
|
import { FTS_FILE, FtsIndex, loadSqlite, segment } from './fts.js'
|
|
26
|
-
import {
|
|
27
|
+
import { insertImage, planSourceImages, type SourceImage } from './source-images.js'
|
|
28
|
+
import {
|
|
29
|
+
applyRelatedSection, buildWikiIndex, cardKey, deadLinkIssue, deadLinksOf, planRelated, stripRelatedSection,
|
|
30
|
+
type DeadLink, type RelatedEntry,
|
|
31
|
+
} from './wiki-finalize.js'
|
|
32
|
+
import { applyRerank, applyThreshold, evidenceExcerpts, graphBoost, matchOf, noteIndexText, reviewWeight, rrfFuse, tokenSet, type RouteRanking } from './recall-rank.js'
|
|
27
33
|
import { legacyKnowledgeRoot } from './layout.js'
|
|
28
34
|
import { unlinkIfExists, writeBufferAtomic, writeTextAtomic } from './fs-atomic.js'
|
|
29
35
|
import { envelopeFailure, envelopeSuccess } from 'dsh-data-contracts/envelope'
|
|
30
36
|
import type { LineageContext, LineageRef } from 'dsh-data-contracts/refs'
|
|
31
37
|
|
|
32
|
-
// **知识树只有三段**(v1.1 起):`
|
|
38
|
+
// **知识树只有三段**(v1.1 起):`01_source_raw` / `02_wiki` / `03_compiled`。
|
|
33
39
|
//
|
|
34
40
|
// 撤掉的两段,理由都写在这里,避免下一个人重新加回来:
|
|
35
41
|
//
|
|
@@ -39,9 +45,9 @@ import type { LineageContext, LineageRef } from 'dsh-data-contracts/refs'
|
|
|
39
45
|
// 本身是一种承诺,承诺了却空着就是误导。**一句话:schema 在设置里编辑,不在磁盘上。**
|
|
40
46
|
// - `05_diagnosis`(v1.1 撤):它同时装着两类东西 —— 「一次排查的产物」(源头)与
|
|
41
47
|
// 「五类模板卡」(配置)—— 一个编号段混装两种性质,`rebuild --only` 的语义立刻含糊。
|
|
42
|
-
// 现在:一次排查的产物是一条 `agent_diagnosis_case` 的 SourceRaw(落 `
|
|
48
|
+
// 现在:一次排查的产物是一条 `agent_diagnosis_case` 的 SourceRaw(落 `01_source_raw/`),
|
|
43
49
|
// 模板卡由类型定义派生(不落盘)。旧目录里的 `*.jsonl` 降为**只读的迁移输入与历史快照**。
|
|
44
|
-
const LAYOUT = ['
|
|
50
|
+
const LAYOUT = ['01_source_raw', '02_wiki', '03_compiled'] as const
|
|
45
51
|
/**
|
|
46
52
|
* 卡片正文里那个机器可读块的围栏标记。
|
|
47
53
|
*
|
|
@@ -60,10 +66,27 @@ const DIAGNOSIS_BLOCK_FENCE = 'json diagnosis'
|
|
|
60
66
|
* 读路径不依赖这个层级:`sourceIndex()` 递归扫全段建 slug → path 索引,所以旧
|
|
61
67
|
* 平铺布局与迁移中间态都照常可读(见 `migrateSourceTruthLayout`)。
|
|
62
68
|
*/
|
|
63
|
-
const SOURCE_TRUTH = '
|
|
69
|
+
const SOURCE_TRUTH = '01_source_raw'
|
|
70
|
+
/**
|
|
71
|
+
* 0051 之前的段名。**只留这一处**,两个用途:
|
|
72
|
+
* 1. **兼容解析**:存量正文里的图片引用写的是 `../../01_source_truth/_assets/…`
|
|
73
|
+
* (0047 的三种历史写法之一),改段名不改文件内容,所以解析器要同时认两个名字;
|
|
74
|
+
* 2. **迁移**:把整棵树搬过来(`migrateSourceTree`)。
|
|
75
|
+
*
|
|
76
|
+
* 卡片正文里的老引用**不改写**:改正文会造内容版本、让卡片重编重编码,而代价只是为了
|
|
77
|
+
* 让一个字符串好看一点。解析器认两个名字,读得到就行。
|
|
78
|
+
*/
|
|
79
|
+
const LEGACY_SOURCE_TRUTH = '01_source_truth'
|
|
64
80
|
/** Wiki 段;段内按 `entity_type` 分目录(与 SourceRaw 的分类分层同一个思路)。 */
|
|
65
81
|
const WIKI = '02_wiki'
|
|
66
82
|
const STATUS: KnowledgeStatus[] = ['unreviewed', 'draft', 'reviewed', 'active', 'deprecated', 'problem']
|
|
83
|
+
/**
|
|
84
|
+
* 一次 `import_sources` 能带多少份来源(0045)。
|
|
85
|
+
*
|
|
86
|
+
* 这不是产品配额,是**一次调用能被按住的时长**:编译逐份来源串行抽实体,50 份已经要跑
|
|
87
|
+
* 很久。超了就分批,每批各自编译 —— 而不是让一次调用变成没人知道什么时候结束的东西。
|
|
88
|
+
*/
|
|
89
|
+
export const MAX_IMPORT_SOURCES = 50
|
|
67
90
|
const SAFE_ID = /^[a-zA-Z0-9._:-]{1,160}$/
|
|
68
91
|
const VECTOR_MANIFEST = 'vector-manifest.json'
|
|
69
92
|
const VECTOR_BINARY = 'vectors.binary'
|
|
@@ -129,6 +152,14 @@ const CARD_VERSIONS = 'card-versions'
|
|
|
129
152
|
* 只会让下一次重建把关系重新抽一遍。
|
|
130
153
|
*/
|
|
131
154
|
const EXTRACTED_RELATIONS = 'extracted_relations.json'
|
|
155
|
+
/**
|
|
156
|
+
* 索引页(0049):**不打开界面也能知道这个库里有什么**。
|
|
157
|
+
*
|
|
158
|
+
* 落在 `03_compiled/` 而不是 `02_wiki/`:后者的每一份 `.md` 都会被 `loadCards`
|
|
159
|
+
* 当成一张卡片读进卡片清单,索引页放进去就会变成一张标题叫「知识库索引」的卡片,
|
|
160
|
+
* 而且每次编译都变一次。它是派生产物,所以归派生目录。
|
|
161
|
+
*/
|
|
162
|
+
const WIKI_INDEX = 'wiki-index.md'
|
|
132
163
|
/** Sections shorter than this stay in a single note; longer cards split by heading. */
|
|
133
164
|
const NOTE_SPLIT_MIN_CHARS = 800
|
|
134
165
|
/**
|
|
@@ -187,8 +218,21 @@ const ENCODER_SELF_SIMILARITY_MAX = 0.99
|
|
|
187
218
|
const MIN_QUERY_COVERAGE = 0.25
|
|
188
219
|
/** 每条命中最多带几段证据原文。 */
|
|
189
220
|
const EVIDENCE_PER_HIT = 3
|
|
190
|
-
|
|
191
|
-
|
|
221
|
+
/*
|
|
222
|
+
* 0050 删掉了 `CONTEXT_PER_WIKI_CAP`(同一张 wiki 最多几条进上下文)。
|
|
223
|
+
* 它是「按 note 组装上下文」时代的补丁:那时同一张卡的多个小节能占满名额。
|
|
224
|
+
* 现在上下文**按卡片去重**,一张卡只有一个条目 —— 上限这个概念不再需要,
|
|
225
|
+
* 取而代之的是卡片级的 1 跳扩展上限(`MAX_EXPANSION_CARDS`)。
|
|
226
|
+
*/
|
|
227
|
+
/**
|
|
228
|
+
* 1 跳扩展最多拉几张卡片(0050)。
|
|
229
|
+
*
|
|
230
|
+
* 上限是**有意的**:一张卡片在图上的邻居可以很多(一份来源编出的卡片彼此都是邻居),
|
|
231
|
+
* 不设限时「1 跳」会把整个连通分量拖进上下文。宁可少拉,也不要让扩展压过命中。
|
|
232
|
+
*/
|
|
233
|
+
const MAX_EXPANSION_CARDS = 20
|
|
234
|
+
/** 溢出文件的保留时长:超过它的旧文件在下次召回时清掉(0050)。 */
|
|
235
|
+
const RECALL_OVERFLOW_TTL_MS = 24 * 60 * 60 * 1000
|
|
192
236
|
const BIT_COUNTS = Array.from({ length: 256 }, (_, value) => {
|
|
193
237
|
let count = 0
|
|
194
238
|
for (let bit = value; bit !== 0; bit &= bit - 1) count += 1
|
|
@@ -249,15 +293,15 @@ interface ExtractionOutcome {
|
|
|
249
293
|
candidatesExtracted: number
|
|
250
294
|
candidatesRejected: number
|
|
251
295
|
/**
|
|
252
|
-
*
|
|
296
|
+
* 这一轮**新建**的 wiki 卡片数。
|
|
253
297
|
*
|
|
254
|
-
* 名字从 `
|
|
255
|
-
*
|
|
256
|
-
*
|
|
298
|
+
* 名字从 `proposalsCreated` 改回来(0044):Pass 0 重新直接落卡。0040 v1.5 那道
|
|
299
|
+
* 「人确认之后才入库」的门撤掉了 —— 入库在前、review 在后,所以计数必须如实说
|
|
300
|
+
* 「新建了 N 张卡片」,而磁盘上真的有 N 张。
|
|
257
301
|
*/
|
|
258
|
-
|
|
259
|
-
/**
|
|
260
|
-
|
|
302
|
+
cardsCreated: number
|
|
303
|
+
/** 这一轮**更新**(把新抽到的正文追加进去、或补上来源)的已有卡片数。 */
|
|
304
|
+
cardsUpdated: number
|
|
261
305
|
sourcesExtracted: number
|
|
262
306
|
sourcesSkipped: number
|
|
263
307
|
relations: Array<{ fromKey: string; toKey: string; relation: string; sourceId: string }>
|
|
@@ -269,6 +313,29 @@ interface ExtractionOutcome {
|
|
|
269
313
|
warnings: string[]
|
|
270
314
|
}
|
|
271
315
|
|
|
316
|
+
/**
|
|
317
|
+
* Finalize(收尾段)一次运行的结果(0049)。
|
|
318
|
+
*
|
|
319
|
+
* `related` 与 `deadLinks` 要**继续往下传**:正文层算出来的计划,产物层(索引页)
|
|
320
|
+
* 与接口层(卡片抽屉的相关分块)读的是同一份 —— 三处各算一遍就会漂移。
|
|
321
|
+
*/
|
|
322
|
+
interface FinalizeOutcome {
|
|
323
|
+
related: Map<string, RelatedEntry[]>
|
|
324
|
+
/** 相关链接总条数(含双向),报告里显示的就是它。 */
|
|
325
|
+
relatedLinks: number
|
|
326
|
+
/** 有几张卡片至少有了一条相关。 */
|
|
327
|
+
relatedCards: number
|
|
328
|
+
deadLinks: DeadLink[]
|
|
329
|
+
/** 这一轮因为相关块或断链问题而重写的卡片数。 */
|
|
330
|
+
rewritten: number
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** 卡片上的待补项(`metadata.schema_issues`),形状不对时当空。 */
|
|
334
|
+
function cardIssuesOf(card: KnowledgeCard): CardSchemaIssue[] {
|
|
335
|
+
const raw = card.metadata.schema_issues
|
|
336
|
+
return Array.isArray(raw) ? raw as CardSchemaIssue[] : []
|
|
337
|
+
}
|
|
338
|
+
|
|
272
339
|
/**
|
|
273
340
|
* Both helpers delegate to the shared envelope constructors: the five-key shape
|
|
274
341
|
* and the "warnings ⇒ partial" rule live in `dsh-data-contracts`,
|
|
@@ -301,23 +368,90 @@ function contentHashOf(content: string): string { return hash(content) }
|
|
|
301
368
|
* 已经在正文里出现过的那一段不再追加:同一份来源重建、或同一段被两个候选命中时,
|
|
302
369
|
* 卡片不该越编译越长。判重按整段文本,所以它是幂等的。
|
|
303
370
|
*/
|
|
371
|
+
/** 批量导入里成功写入的一条来源(0045)。`index` 回指输入数组,调用方能对上号。 */
|
|
372
|
+
export interface ImportedSourceReport {
|
|
373
|
+
index: number
|
|
374
|
+
sourceId: string
|
|
375
|
+
title?: string
|
|
376
|
+
path: string
|
|
377
|
+
bytes: number
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** 批量导入里失败的一条。**带着原因回来**,因为局部失败要求调用方能只重试那几条。 */
|
|
381
|
+
export interface ImportFailure {
|
|
382
|
+
index: number
|
|
383
|
+
title?: string
|
|
384
|
+
code: string
|
|
385
|
+
message: string
|
|
386
|
+
}
|
|
387
|
+
|
|
304
388
|
/**
|
|
305
|
-
*
|
|
389
|
+
* 批量导入顺带那次编译的结果。
|
|
306
390
|
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
*
|
|
391
|
+
* 只带**能被人读懂**的几个数:新建 / 并入 / 复用 / 重编 / 重抽 / 跳过 / 移除、索引版本、
|
|
392
|
+
* 向量那一段的状态。整份 `RebuildReport` 有三十多个字段(指纹、计数、诊断),
|
|
393
|
+
* 而这里要回答的是「刚才那一下让库里多了什么」。
|
|
310
394
|
*/
|
|
311
|
-
interface
|
|
395
|
+
export interface ImportCompileReport {
|
|
396
|
+
cardsCreated: number
|
|
397
|
+
cardsUpdated: number
|
|
398
|
+
cardsReused?: number
|
|
399
|
+
notesReused?: number
|
|
400
|
+
notesRecompiled?: number
|
|
401
|
+
sourcesExtracted?: number
|
|
402
|
+
sourcesSkipped?: number
|
|
403
|
+
cardsRemoved?: number
|
|
404
|
+
embedding?: string
|
|
405
|
+
version?: number
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/** 一次 `import_sources` 的账本:写了哪些、哪些没写成、顺带编译出了什么。 */
|
|
409
|
+
export interface ImportSourcesResult {
|
|
410
|
+
written: ImportedSourceReport[]
|
|
411
|
+
failed: ImportFailure[]
|
|
412
|
+
/** 输入里一共有几份 —— 成功数 + 失败数 = 它,除非整批在写入前就被拒了。 */
|
|
413
|
+
attempted: number
|
|
414
|
+
/** `compile: false` 时是 `null`:明确表示「这次没编译」,而不是「编译了但没数字」。 */
|
|
415
|
+
compile: ImportCompileReport | null
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Pass 0 收集到的一条**待落卡片**。
|
|
420
|
+
*
|
|
421
|
+
* 它存在的原因是**整轮结束才落盘**:一份来源里同一个实体可能被抽到两次,两份来源也
|
|
422
|
+
* 可能抽到同一个实体 —— 逐次写会得到 N 次覆盖,而增量指纹要的是「这一轮这张卡片
|
|
423
|
+
* 最终长什么样」。
|
|
424
|
+
*
|
|
425
|
+
* `card` 是**已有卡片**(更新时),带着它的状态、备注、评分与 metadata —— 更新一张
|
|
426
|
+
* 卡片不该把人写过的备注、review 结论、以及别处来源的字段一起抹掉。
|
|
427
|
+
*/
|
|
428
|
+
interface PendingCard {
|
|
312
429
|
action: 'create' | 'update'
|
|
313
430
|
entityType: SchemaType
|
|
314
431
|
entityId: string
|
|
315
432
|
title: string
|
|
316
433
|
content: string
|
|
317
434
|
sourceRefs: string[]
|
|
318
|
-
metadata
|
|
319
|
-
|
|
320
|
-
|
|
435
|
+
metadata: Record<string, unknown>
|
|
436
|
+
summary?: string
|
|
437
|
+
/**
|
|
438
|
+
* 更新的是哪张已有卡片。
|
|
439
|
+
*
|
|
440
|
+
* 带着它是为了**保住正文之外的一切**:状态、人工备注(`annotations`)、review 评分
|
|
441
|
+
* (`review`)、别处来源留下的字段。只凭 `type:id` 重新拼一张卡片,会把这些悄悄抹掉 ——
|
|
442
|
+
* 而「备注不会被编译覆盖」正是 0044 承诺的东西。
|
|
443
|
+
*/
|
|
444
|
+
existing?: KnowledgeCard
|
|
445
|
+
/** 这一轮判重命中过它的来源与候选键,写进 `metadata.extraction_merges`。 */
|
|
446
|
+
merges: Array<{ sourceId: string; keys: string[]; at: string }>
|
|
447
|
+
/**
|
|
448
|
+
* 这一轮有没有**改到卡片的字段值**(类型定义里的 `formula` / `owner` 之类,或摘要)。
|
|
449
|
+
*
|
|
450
|
+
* 单独记一个布尔而不是去比整个 `metadata`:`metadata` 里还有 `evidence`、`confidence`、
|
|
451
|
+
* `extraction_merges` 这些每次都不同的东西,整体比较只会永远为真,于是「没变的卡片不重写」
|
|
452
|
+
* 这条增量承诺就悄悄失效了。这里只回答那个真正要紧的问题 —— 字段值变了没有。
|
|
453
|
+
*/
|
|
454
|
+
fieldChanged?: boolean
|
|
321
455
|
}
|
|
322
456
|
|
|
323
457
|
/** 候选的别名集合:判重认的是名字,不是卡片 id(中文名会退化成哈希 id)。 */
|
|
@@ -325,29 +459,37 @@ function aliasesOf(candidate: { name: string; aliases?: string[] }): string[] {
|
|
|
325
459
|
return [...new Set([candidate.name, ...(candidate.aliases ?? [])].map(value => value.trim()).filter(Boolean))]
|
|
326
460
|
}
|
|
327
461
|
|
|
328
|
-
/**
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
* 已有卡片只读,所以累积的是**提议的正文**:`appendExtracted` 从卡片当时的正文出发,
|
|
332
|
-
* 把这一轮所有命中的正文依次追加进去。同一条提议被追加两次正文时不会重复
|
|
333
|
-
* (`appendExtracted` 自带 `includes` 判重)。
|
|
334
|
-
*/
|
|
335
|
-
function mergeProposal(
|
|
336
|
-
pending: Map<string, PendingProposal>,
|
|
462
|
+
/** 累积一次「并入已有卡片」。同一条 pending 被追加两次时 `appendExtracted` 自带判重。 */
|
|
463
|
+
function mergeIntoPending(
|
|
464
|
+
pending: Map<string, PendingCard>,
|
|
337
465
|
card: KnowledgeCard,
|
|
338
466
|
sourceId: string,
|
|
339
467
|
bodies: string[],
|
|
340
468
|
keys: string[],
|
|
469
|
+
at: string,
|
|
470
|
+
/**
|
|
471
|
+
* 这次新抽到的**字段值**(类型定义里的 `formula` / `owner` 等)与摘要。
|
|
472
|
+
*
|
|
473
|
+
* 它们必须跟着走完整条路径。0044 的实现里漏过一次:模型在第二份文档里给出了更完整的
|
|
474
|
+
* `formula`,卡片只追加了正文,字段值停在第一份文档的版本上 —— 而正文里写着新的公式。
|
|
475
|
+
* 这种「正文与字段互相矛盾」比缺字段更难发现。
|
|
476
|
+
*/
|
|
477
|
+
fields?: Record<string, string>,
|
|
478
|
+
summary?: string,
|
|
341
479
|
): void {
|
|
342
480
|
const key = `${card.entityType}:${card.entityId}`
|
|
343
481
|
const previous = pending.get(key)
|
|
344
|
-
|
|
345
|
-
//
|
|
346
|
-
const
|
|
482
|
+
// 字段值真的变了才算「改到」:同一段公式被两份文档重复写下是很常见的,
|
|
483
|
+
// 那种情况下这一轮没有任何新东西,不该把卡片重写一遍。
|
|
484
|
+
const changed = Object.entries(fields ?? {}).some(([name, value]) => String(card.metadata[name] ?? '') !== value)
|
|
485
|
+
|| Boolean(summary && summary !== (card.summary ?? ''))
|
|
347
486
|
if (previous) {
|
|
348
487
|
previous.content = appendExtracted(previous.content, bodies)
|
|
349
488
|
if (!previous.sourceRefs.includes(sourceId)) previous.sourceRefs.push(sourceId)
|
|
350
|
-
previous.metadata = { ...previous.metadata,
|
|
489
|
+
previous.metadata = { ...previous.metadata, ...(fields ?? {}) }
|
|
490
|
+
if (!previous.summary && summary) previous.summary = summary
|
|
491
|
+
if (changed) previous.fieldChanged = true
|
|
492
|
+
previous.merges.push({ sourceId, keys, at })
|
|
351
493
|
return
|
|
352
494
|
}
|
|
353
495
|
pending.set(key, {
|
|
@@ -358,22 +500,200 @@ function mergeProposal(
|
|
|
358
500
|
// **追加**而不是覆盖,也不是丢弃:实体已存在时,这次新抽到的正文并进原卡片,
|
|
359
501
|
// 于是知识是累积的,来源改写了也不会停在旧版本上。
|
|
360
502
|
content: appendExtracted(card.content, bodies),
|
|
361
|
-
sourceRefs: [...card.sourceRefs, sourceId],
|
|
362
|
-
metadata: { ...card.metadata,
|
|
363
|
-
|
|
503
|
+
sourceRefs: card.sourceRefs.includes(sourceId) ? [...card.sourceRefs] : [...card.sourceRefs, sourceId],
|
|
504
|
+
metadata: { ...card.metadata, ...(fields ?? {}) },
|
|
505
|
+
...(summary ?? card.summary ? { summary: summary ?? card.summary } : {}),
|
|
506
|
+
existing: card,
|
|
507
|
+
merges: [{ sourceId, keys, at }],
|
|
508
|
+
...(changed ? { fieldChanged: true } : {}),
|
|
364
509
|
})
|
|
365
510
|
}
|
|
366
511
|
|
|
512
|
+
/** 来源的判重留痕:只留最近 20 条,它是「这张卡被哪些来源命中过」的痕迹,不是审计日志。 */
|
|
513
|
+
function mergeHistoryOf(metadata: Record<string, unknown>, additions: PendingCard['merges']): Array<Record<string, unknown>> {
|
|
514
|
+
const history = Array.isArray(metadata.extraction_merges) ? metadata.extraction_merges : []
|
|
515
|
+
return [...history, ...additions].slice(-20) as Array<Record<string, unknown>>
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* 从 frontmatter 里读人工备注。
|
|
520
|
+
*
|
|
521
|
+
* 形状不对的一条**丢掉,而不是让整张卡片读不出来**:备注是附加信息,为了一条写坏的
|
|
522
|
+
* 备注把一张知识卡片变成「读不到」,代价方向错了。
|
|
523
|
+
*/
|
|
524
|
+
function parseAnnotations(value: unknown): CardAnnotation[] {
|
|
525
|
+
if (!Array.isArray(value)) return []
|
|
526
|
+
const result: CardAnnotation[] = []
|
|
527
|
+
for (const item of value) {
|
|
528
|
+
if (item === null || typeof item !== 'object') continue
|
|
529
|
+
const record = item as Record<string, unknown>
|
|
530
|
+
const note = typeof record.note === 'string' ? record.note.trim() : ''
|
|
531
|
+
if (!note) continue
|
|
532
|
+
result.push({
|
|
533
|
+
id: typeof record.id === 'string' && record.id.trim() ? record.id.trim() : `a-${hash(note).slice(0, 12)}`,
|
|
534
|
+
kind: record.kind === 'anomaly' ? 'anomaly' : 'note',
|
|
535
|
+
note,
|
|
536
|
+
...(typeof record.author === 'string' && record.author.trim() ? { author: record.author.trim() } : {}),
|
|
537
|
+
createdAt: typeof record.createdAt === 'string' && record.createdAt.trim() ? record.createdAt.trim() : '',
|
|
538
|
+
...(typeof record.sourceRef === 'string' && record.sourceRef.trim() ? { sourceRef: record.sourceRef.trim() } : {}),
|
|
539
|
+
})
|
|
540
|
+
}
|
|
541
|
+
return result
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** 从 frontmatter 里读 review 结论。缺省 = 还没人看过(调用方按 `CARD_REVIEW_BASE_SCORE` 处理)。 */
|
|
545
|
+
function parseReview(value: unknown): CardReview | undefined {
|
|
546
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) return undefined
|
|
547
|
+
const record = value as Record<string, unknown>
|
|
548
|
+
const score = Number(record.score)
|
|
549
|
+
const state = record.state === 'reviewed' ? 'reviewed' : 'unreviewed'
|
|
550
|
+
return {
|
|
551
|
+
state,
|
|
552
|
+
score: Number.isFinite(score) ? Math.min(1, Math.max(0, score)) : state === 'reviewed' ? REVIEWED_SCORE_FLOOR : CARD_REVIEW_BASE_SCORE,
|
|
553
|
+
...(typeof record.reviewedAt === 'string' && record.reviewedAt.trim() ? { reviewedAt: record.reviewedAt.trim() } : {}),
|
|
554
|
+
...(typeof record.reviewedBy === 'string' && record.reviewedBy.trim() ? { reviewedBy: record.reviewedBy.trim() } : {}),
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* 一张卡片的**有效评分**(0044):有异常点备注时封顶。
|
|
560
|
+
*
|
|
561
|
+
* 为什么不把「异常」直接写进 `review.score`:异常是**关于证据的判断**(这段内容和原始
|
|
562
|
+
* 文档对不上),review 是**关于覆盖度的判断**(这张卡该有的小节都有了)。两者会同时成立 ——
|
|
563
|
+
* 一张 review 通过的卡片里照样可能被标出一个数字对不上。分成两个字段之后,
|
|
564
|
+
* 撤掉异常备注,review 的结论原样回来;写成一个数就再也分不开了。
|
|
565
|
+
*/
|
|
566
|
+
function effectiveReviewScore(card: Pick<KnowledgeCard, 'review' | 'annotations'>): number {
|
|
567
|
+
const base = card.review?.score ?? CARD_REVIEW_BASE_SCORE
|
|
568
|
+
const hasAnomaly = (card.annotations ?? []).some(item => item.kind === 'anomaly')
|
|
569
|
+
return hasAnomaly ? Math.min(base, ANOMALY_SCORE) : base
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function annotationId(note: string, at: string): string {
|
|
573
|
+
return `a-${hash(`${at}|${note}`).slice(0, 16)}`
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* 一份正文的小节结构:`## 标签` + 它下面的内容。
|
|
578
|
+
*
|
|
579
|
+
* `heading` 保留原始那一行(`## 定义`),重建时逐字写回去,所以合并不会顺手把
|
|
580
|
+
* 「## 定义」改成别的写法。首个标题之前的内容收进 `label: ''` 的那一格 ——
|
|
581
|
+
* 模型偶尔会先写一句总述,丢掉它比多一格更糟。
|
|
582
|
+
*/
|
|
583
|
+
interface BodySection {
|
|
584
|
+
heading: string
|
|
585
|
+
label: string
|
|
586
|
+
body: string
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* 按 `## 小节` 切开正文。没有任何标题时返回 `null`,调用方据此退回「整段拼接」。
|
|
591
|
+
*/
|
|
592
|
+
function splitSections(markdown: string): BodySection[] | null {
|
|
593
|
+
const sections: Array<{ heading: string; label: string; lines: string[] }> = []
|
|
594
|
+
let current: { heading: string; label: string; lines: string[] } | undefined
|
|
595
|
+
let seenHeading = false
|
|
596
|
+
for (const line of markdown.split('\n')) {
|
|
597
|
+
const match = /^##\s+(.+?)\s*$/.exec(line)
|
|
598
|
+
if (match) {
|
|
599
|
+
seenHeading = true
|
|
600
|
+
current = { heading: line.trim(), label: match[1].trim(), lines: [] }
|
|
601
|
+
sections.push(current)
|
|
602
|
+
continue
|
|
603
|
+
}
|
|
604
|
+
if (!current) {
|
|
605
|
+
// 第一个标题出现之前的行(含前导空行)归入前言。
|
|
606
|
+
current = { heading: '', label: '', lines: [] }
|
|
607
|
+
sections.push(current)
|
|
608
|
+
}
|
|
609
|
+
current.lines.push(line)
|
|
610
|
+
}
|
|
611
|
+
if (!seenHeading) return null
|
|
612
|
+
return sections.map(section => ({ heading: section.heading, label: section.label, body: section.lines.join('\n').trim() }))
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
/**
|
|
616
|
+
* 一整节只是**一句括号里的占位**吗(`(文档未说明)` / `(本文档未说明适用场景。)`)?
|
|
617
|
+
*
|
|
618
|
+
* 占位是编译器的约定(`skeletonBody` 写「(文档未说明)」),模型会照着模仿。
|
|
619
|
+
* 判断成占位之后,后来的真实内容**替换**它而不是追加在它后面 —— 否则卡片会长成
|
|
620
|
+
* 「(文档未说明)」+ 具体内容,读起来像两句话,而第一句已经过期了。
|
|
621
|
+
*/
|
|
622
|
+
function isPlaceholderSection(text: string): boolean {
|
|
623
|
+
const trimmed = text.trim()
|
|
624
|
+
return /^[((][^(())]*[))]$/.test(trimmed)
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/** 把一节的新内容并进旧内容:占位替换、重复跳过、其余追加。 */
|
|
628
|
+
function mergeSectionText(current: string, addition: string): string {
|
|
629
|
+
const next = addition.trim()
|
|
630
|
+
if (!next) return current.trim()
|
|
631
|
+
const previous = current.trim()
|
|
632
|
+
if (!previous) return next
|
|
633
|
+
if (previous.includes(next)) return previous
|
|
634
|
+
if (isPlaceholderSection(previous)) return next
|
|
635
|
+
return `${previous}\n\n${next}`
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* 把一次新抽到的正文并进已有正文(0044)。
|
|
640
|
+
*
|
|
641
|
+
* ## 为什么不是简单的字符串拼接
|
|
642
|
+
*
|
|
643
|
+
* 实机验证抓到的形状:同一实体被第二份文档命中时,卡片变成
|
|
644
|
+
* `## 适用场景 / ## 步骤 / ## 验收与失败处理 / ## 适用场景 / ## 步骤 / …` ——
|
|
645
|
+
* 每一节都出现两次。正文是**按类型 schema 分节**的,重复标题意味着这张卡片不再
|
|
646
|
+
* 符合它的类型,读的人还得自己把两段同一节的内容在脑子里合起来。
|
|
647
|
+
*
|
|
648
|
+
* 所以按小节折进去:**每个标签只出现一次**,新内容并进同一节里;旧内容还是占位
|
|
649
|
+
* (「(文档未说明)」)时被新内容替换。两份正文里只要有一份没有 `##` 结构
|
|
650
|
+
* (规则路径切出来的原文就是这样),就退回整段拼接 —— 那种情况下没有可靠的
|
|
651
|
+
* 对齐依据,硬折会把内容塞错节。
|
|
652
|
+
*
|
|
653
|
+
* ## 为什么改在这里
|
|
654
|
+
*
|
|
655
|
+
* 所有「并入已有卡片」的路径都走这个函数(编译、`writeCards`、诊断卡片),
|
|
656
|
+
* 在上层各自折一次,必然有一处漏掉。
|
|
657
|
+
*/
|
|
367
658
|
function appendExtracted(existing: string, additions: string[]): string {
|
|
368
659
|
let content = existing
|
|
369
660
|
for (const addition of additions) {
|
|
370
661
|
const body = addition.trim()
|
|
371
662
|
if (!body || content.includes(body)) continue
|
|
372
|
-
content = content
|
|
663
|
+
content = foldBody(content.trim(), body)
|
|
373
664
|
}
|
|
374
665
|
return content
|
|
375
666
|
}
|
|
376
667
|
|
|
668
|
+
/** 把 `incoming` 按小节折进 `existing`;结构对不上时整段追加。 */
|
|
669
|
+
function foldBody(existing: string, incoming: string): string {
|
|
670
|
+
if (!existing) return incoming
|
|
671
|
+
const base = splitSections(existing)
|
|
672
|
+
const extra = splitSections(incoming)
|
|
673
|
+
if (!base || !extra) return `${existing}\n\n${incoming}`
|
|
674
|
+
|
|
675
|
+
const byLabel = new Map(base.map(section => [section.label, section]))
|
|
676
|
+
const appended: BodySection[] = []
|
|
677
|
+
for (const section of extra) {
|
|
678
|
+
const target = byLabel.get(section.label)
|
|
679
|
+
if (target) {
|
|
680
|
+
target.body = mergeSectionText(target.body, section.body)
|
|
681
|
+
continue
|
|
682
|
+
}
|
|
683
|
+
// 新标签:整节接在最后,标题逐字保留。
|
|
684
|
+
const fresh = { heading: section.heading, label: section.label, body: section.body }
|
|
685
|
+
byLabel.set(section.label, fresh)
|
|
686
|
+
appended.push(fresh)
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
const ordered = [...base, ...appended]
|
|
690
|
+
const rendered = ordered
|
|
691
|
+
.map(section => (section.heading ? `${section.heading}\n\n${section.body}`.trimEnd() : section.body))
|
|
692
|
+
.filter(part => part.trim().length > 0)
|
|
693
|
+
.join('\n\n')
|
|
694
|
+
return rendered
|
|
695
|
+
}
|
|
696
|
+
|
|
377
697
|
/**
|
|
378
698
|
* `2026-01-02T03:04:05.123Z` → `20260102T030405123Z`. Millisecond precision keeps
|
|
379
699
|
* two overwrites inside the same second from colliding on the same version id.
|
|
@@ -400,15 +720,31 @@ function plainRecord(value: unknown): Record<string, unknown> {
|
|
|
400
720
|
* `metadata.source_path`; otherwise the upstream `source_format` label
|
|
401
721
|
* (`markdown` / `office` / …); otherwise `markdown`, the store's default shape.
|
|
402
722
|
*/
|
|
723
|
+
/**
|
|
724
|
+
* 这份来源**是什么格式**(`web_page` / `office` / `pdf` / `markdown`…)。
|
|
725
|
+
*
|
|
726
|
+
* `source_format` 是转换内核报出来的**格式类**,有就以它为准 —— 按扩展名去猜会把
|
|
727
|
+
* 「网页」显示成「html」(0048 之前就是这样:`类型: .html · html`,而 manifest 里
|
|
728
|
+
* 明明写着 `web_page`)。没有 `source_format` 的手写来源才退回扩展名那条链。
|
|
729
|
+
*/
|
|
403
730
|
function sourceFormatOf(metadata: Record<string, unknown>): string {
|
|
404
|
-
const
|
|
405
|
-
if (
|
|
406
|
-
|
|
731
|
+
const format = typeof metadata.source_format === 'string' ? metadata.source_format.trim().toLowerCase() : ''
|
|
732
|
+
if (format) return format
|
|
733
|
+
const ext = sourceExtOf(metadata)
|
|
734
|
+
return ext ? ext.replace(/^\./, '') : 'markdown'
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
/** 底稿文件名 → 扩展名。`original.rel` 是 0048 起的持久位置,`source_path` 是旧形状。 */
|
|
738
|
+
function sourceExtOf(metadata: Record<string, unknown>): string {
|
|
739
|
+
const original = plainRecord(metadata.original)
|
|
740
|
+
const candidates = [original.rel, metadata.source_path]
|
|
741
|
+
for (const value of candidates) {
|
|
742
|
+
if (typeof value !== 'string' || !value.trim()) continue
|
|
743
|
+
const base = value.trim().split(/[\\/]/).pop() ?? ''
|
|
407
744
|
const dot = base.lastIndexOf('.')
|
|
408
|
-
if (dot > 0 && dot < base.length - 1) return base.slice(dot
|
|
745
|
+
if (dot > 0 && dot < base.length - 1) return base.slice(dot).toLowerCase()
|
|
409
746
|
}
|
|
410
|
-
|
|
411
|
-
return format || 'markdown'
|
|
747
|
+
return ''
|
|
412
748
|
}
|
|
413
749
|
|
|
414
750
|
/** Failure payload for a document lookup that could not produce a detail. */
|
|
@@ -432,6 +768,9 @@ function emptyDocumentDetail(docId: string): KnowledgeDocumentDetail {
|
|
|
432
768
|
*/
|
|
433
769
|
export const SOURCE_PREVIEW_CHARS = 4000
|
|
434
770
|
|
|
771
|
+
/** 耗时统一保留一位小数:界面上不需要微秒,而 0.0 与 0 的读法不一样。 */
|
|
772
|
+
function round1(value: number): number { return Math.round(value * 10) / 10 }
|
|
773
|
+
|
|
435
774
|
function generatedId(prefix: string): string { return `${prefix}_${hash(`${Date.now()}:${Math.random()}`)}` }
|
|
436
775
|
|
|
437
776
|
function parseLineage(value: unknown): LineageContext | undefined {
|
|
@@ -564,8 +903,198 @@ function markdownImageRefs(content: string): string[] {
|
|
|
564
903
|
return [...new Set(refs)]
|
|
565
904
|
}
|
|
566
905
|
|
|
567
|
-
|
|
568
|
-
|
|
906
|
+
/**
|
|
907
|
+
* 一份 SourceRaw 的 front matter(0048)。
|
|
908
|
+
*
|
|
909
|
+
* 键分成两类,**顺序与来源**都是刻意的:
|
|
910
|
+
*
|
|
911
|
+
* 1. **OKF v0.2 的顶层键**:`type` / `title` / `description` / `resource` /
|
|
912
|
+
* `tags` / `generated` / `sources`。它们是真 YAML 键,人能读、`grep` 得到、
|
|
913
|
+
* 外部工具能按公开规范读走。`generated.at` 就是**采集时间**(OKF 里没有「抓取
|
|
914
|
+
* 时间」这个概念,本文件何时被谁写出来就是它)。
|
|
915
|
+
* 2. **我们自己的运行键**:`source_id` / `external_id` / `created_at` / `updated_at` /
|
|
916
|
+
* `metadata` / `lineage`。OKF 明确允许自定义键,且要求消费者不得因为未知键拒收,
|
|
917
|
+
* 所以这两类可以同处一个 front matter。
|
|
918
|
+
*
|
|
919
|
+
* `metadata` 只剩**类型专属的扩展键**(转换任务 id、内容哈希、资产清单、格式…):
|
|
920
|
+
* 那些东西 OKF 没有对应概念,但也不该混进上面那组公开字段里。
|
|
921
|
+
*
|
|
922
|
+
* `type` 是 OKF 唯一必填键,值就是我们自己的来源分类(它本来就是「这个概念是什么」)。
|
|
923
|
+
* 旧文件里的 `source_type` 仍然读得到(见 `sourceTypeOf`),迁移只写新键。
|
|
924
|
+
*/
|
|
925
|
+
function sourceFrontMatter(record: SourceRaw, provenance?: SourceProvenance): Record<string, unknown> {
|
|
926
|
+
const frontMatter: Record<string, unknown> = {
|
|
927
|
+
type: record.sourceType,
|
|
928
|
+
title: record.title,
|
|
929
|
+
}
|
|
930
|
+
if (provenance?.description?.trim()) frontMatter.description = provenance.description.trim()
|
|
931
|
+
if (provenance?.resource?.trim()) frontMatter.resource = provenance.resource.trim()
|
|
932
|
+
if (provenance?.tags && provenance.tags.length > 0) frontMatter.tags = provenance.tags
|
|
933
|
+
const generated = {
|
|
934
|
+
...(provenance?.generatedBy?.trim() ? { by: provenance.generatedBy.trim() } : {}),
|
|
935
|
+
...(provenance?.generatedAt?.trim() ? { at: provenance.generatedAt.trim() } : {}),
|
|
936
|
+
}
|
|
937
|
+
if (Object.keys(generated).length > 0) frontMatter.generated = generated
|
|
938
|
+
const sources = (provenance?.sources ?? []).filter(origin => origin.resource.trim())
|
|
939
|
+
if (sources.length > 0) {
|
|
940
|
+
frontMatter.sources = sources.map((origin: SourceOriginRef) => ({
|
|
941
|
+
id: origin.id,
|
|
942
|
+
resource: origin.resource,
|
|
943
|
+
...(origin.title?.trim() ? { title: origin.title.trim() } : {}),
|
|
944
|
+
...(origin.author?.trim() ? { author: origin.author.trim() } : {}),
|
|
945
|
+
...(origin.lastModified?.trim() ? { last_modified: origin.lastModified.trim() } : {}),
|
|
946
|
+
}))
|
|
947
|
+
}
|
|
948
|
+
frontMatter.source_id = record.sourceId
|
|
949
|
+
frontMatter.external_id = record.externalId ?? ''
|
|
950
|
+
frontMatter.created_at = record.createdAt
|
|
951
|
+
frontMatter.updated_at = record.updatedAt
|
|
952
|
+
frontMatter.metadata = record.metadata
|
|
953
|
+
if (record.lineage) frontMatter.lineage = record.lineage
|
|
954
|
+
return frontMatter
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
/** 把 `sources[0].author` 拆成 `['sources', 0, 'author']`。 */
|
|
958
|
+
function metadataPath(key: string): Array<string | number> {
|
|
959
|
+
const tokens: Array<string | number> = []
|
|
960
|
+
for (const part of key.split('.')) {
|
|
961
|
+
const name = part.replace(/\[\d+\]/g, '').trim()
|
|
962
|
+
if (name) tokens.push(name)
|
|
963
|
+
for (const index of part.matchAll(/\[(\d+)\]/g)) tokens.push(Number(index[1]))
|
|
964
|
+
}
|
|
965
|
+
return tokens
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
/** 这份 front matter 里,分类声明的元数据字段缺了哪些(缺 = 没有值,不是「值为空串」)。 */
|
|
969
|
+
function missingMetadataFields(fields: SourceMetadataField[] | undefined, frontMatter: Record<string, unknown>): SourceMetadataField[] {
|
|
970
|
+
if (!fields || fields.length === 0) return []
|
|
971
|
+
const present = (key: string): boolean => {
|
|
972
|
+
let current: unknown = frontMatter
|
|
973
|
+
for (const token of metadataPath(key)) {
|
|
974
|
+
if (typeof token === 'number') {
|
|
975
|
+
if (!Array.isArray(current)) return false
|
|
976
|
+
current = current[token]
|
|
977
|
+
} else {
|
|
978
|
+
if (current === null || typeof current !== 'object') return false
|
|
979
|
+
current = (current as Record<string, unknown>)[token]
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
if (current === undefined || current === null) return false
|
|
983
|
+
if (typeof current === 'string') return current.trim().length > 0
|
|
984
|
+
if (Array.isArray(current)) return current.length > 0
|
|
985
|
+
return true
|
|
986
|
+
}
|
|
987
|
+
return fields.filter(field => field.required === true && !present(field.key))
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
/**
|
|
991
|
+
* 读一份 front matter 里的来源分类。
|
|
992
|
+
*
|
|
993
|
+
* 两个键都认:`type` 是 0048 起的规范键(OKF 必填),`source_type` 是旧文件里的键。
|
|
994
|
+
* 迁移会把存量文件改写成 `type`,但**读**这一侧永远两个都认 —— 别的库、别人手写的
|
|
995
|
+
* 文件、备份里翻出来的文件都可能还是旧形状,读不了就等于数据丢失。
|
|
996
|
+
*/
|
|
997
|
+
function sourceTypeOf(frontmatter: Record<string, unknown>): string {
|
|
998
|
+
const canonical = frontmatter.type
|
|
999
|
+
if (typeof canonical === 'string' && canonical.trim()) return canonical.trim()
|
|
1000
|
+
const legacy = frontmatter.source_type
|
|
1001
|
+
if (typeof legacy === 'string' && legacy.trim()) return legacy.trim()
|
|
1002
|
+
return ''
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
/**
|
|
1006
|
+
* 一个图片引用可能指向哪些磁盘路径,按优先级排。
|
|
1007
|
+
*
|
|
1008
|
+
* 知识库里同时存在三种写法,都是历史产物,谁也没错,所以解析要认全:
|
|
1009
|
+
*
|
|
1010
|
+
* | 写法 | 谁写的 | 从哪解析 |
|
|
1011
|
+
* | --- | --- | --- |
|
|
1012
|
+
* | `01_source_raw/_assets/<bucket>/<file>` | **规范形**:知识库根相对 | 根目录 |
|
|
1013
|
+
* | `_assets/<bucket>/<file>` | 转换内核早期写进 SourceRaw 的形式(相对 `01_source_raw/`) | 根目录 + `01_source_raw/` |
|
|
1014
|
+
* | `../../01_source_raw/_assets/…` | `cardHints` 给手工建卡用的形式 | 文件自己所在目录 |
|
|
1015
|
+
*
|
|
1016
|
+
* 顺序就是「先根目录、再 `01_source_raw/`、最后文件自己的目录」,而且只认**真实存在**
|
|
1017
|
+
* 的那个({@link markdownImagePaths} 用 `existsSync` 挑)。这条规则与 `assetsOf` 共用,
|
|
1018
|
+
* 因为「文档里有几张图」和「卡片正文里的图在哪」必须是同一个答案。
|
|
1019
|
+
*/
|
|
1020
|
+
function assetRefCandidates(ref: string, baseDir: string, rootDir: string): string[] {
|
|
1021
|
+
if (isAbsolute(ref)) return [resolve(ref)]
|
|
1022
|
+
return [
|
|
1023
|
+
resolve(rootDir, ref),
|
|
1024
|
+
resolve(rootDir, SOURCE_TRUTH, ref),
|
|
1025
|
+
// 老段名的引用(0051 之前写进正文的那些):换一个名字再试一遍,别让存量卡片丢图。
|
|
1026
|
+
...(ref.includes(LEGACY_SOURCE_TRUTH) ? [resolve(rootDir, ref.replace(LEGACY_SOURCE_TRUTH, SOURCE_TRUTH))] : []),
|
|
1027
|
+
...(ref.includes(SOURCE_TRUTH) ? [resolve(rootDir, ref.replace(SOURCE_TRUTH, LEGACY_SOURCE_TRUTH))] : []),
|
|
1028
|
+
resolve(baseDir, ref),
|
|
1029
|
+
]
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
/**
|
|
1033
|
+
* 把 `from` 里的每一项并进 `to`(0051 的迁移中间态用)。
|
|
1034
|
+
*
|
|
1035
|
+
* 目标里已有的**不覆盖**:迁移只负责「把还没过来的搬过来」,冲突留给下一次启动或
|
|
1036
|
+
* 人来处理。两边都是目录就递归 —— `_assets/<bucket>/` 是两层,一层不够。
|
|
1037
|
+
*/
|
|
1038
|
+
async function mergeDir(from: string, to: string): Promise<void> {
|
|
1039
|
+
await mkdir(to, { recursive: true })
|
|
1040
|
+
for (const entry of await readdir(from, { withFileTypes: true })) {
|
|
1041
|
+
const source = join(from, entry.name)
|
|
1042
|
+
const target = join(to, entry.name)
|
|
1043
|
+
if (existsSync(target)) {
|
|
1044
|
+
if (entry.isDirectory()) await mergeDir(source, target)
|
|
1045
|
+
continue
|
|
1046
|
+
}
|
|
1047
|
+
await rename(source, target)
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
/**
|
|
1052
|
+
* 文件名(不含扩展名):**用标题**(0051)。
|
|
1053
|
+
*
|
|
1054
|
+
* 三条规则,都是为了让它在任何文件系统上都落得下去:
|
|
1055
|
+
* 1. 路径不安全字符与不可见字符换成 `-`(`/` `\` `:` `*` `?` `"` `<` `>` `|` 与控制字符);
|
|
1056
|
+
* 2. 首尾的点与空格去掉(Windows 与 macOS 都不喜欢,且会让 `ls` 看起来少一个字符);
|
|
1057
|
+
* 3. 截断到 120 字符;标题为空(或全是非法字符)时**退回 `source_id` 的 slug** ——
|
|
1058
|
+
* 宁可文件名不好看,也不要没有文件名。
|
|
1059
|
+
*
|
|
1060
|
+
* 它不负责查重:目标路径上已经有人占了,由 `addSource` 按 `SOURCE_TITLE_DUPLICATED` 拒收。
|
|
1061
|
+
*/
|
|
1062
|
+
function sourceFileBase(title: string, sourceId: string): string {
|
|
1063
|
+
const cleaned = title
|
|
1064
|
+
.replace(/[/\\:*?"<>|\u0000-\u001f]/g, '-')
|
|
1065
|
+
.replace(/\s+/g, ' ')
|
|
1066
|
+
.replace(/^[.\s]+|[.\s]+$/g, '')
|
|
1067
|
+
.slice(0, 120)
|
|
1068
|
+
.trim()
|
|
1069
|
+
// 只剩替换字符(`///` → `---`)等于没有名字:一个叫 `---.md` 的文件比 id 更难认。
|
|
1070
|
+
return /^[-\s]*$/.test(cleaned) ? slug(sourceId) : cleaned
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
/**
|
|
1074
|
+
* 正文里的图片引用 → 绝对路径。
|
|
1075
|
+
*
|
|
1076
|
+
* 卡片这条路上原先少了这一步:`markdownImagePaths(content, dirname(path))` 只按文件
|
|
1077
|
+
* 自己的目录解析。于是卡片正文里那张从来源**原样搬过来**的图,被解析成
|
|
1078
|
+
* `02_wiki/<type>/_assets/…` —— 一个不存在的路径。0047 实机就是这么炸的:向量编码器
|
|
1079
|
+
* 报 `FileNotFoundError: …/02_wiki/concept/_assets/…/img-2d1e420e82a8.png`,
|
|
1080
|
+
* 整个 embedding 那一档直接降级成 unavailable。
|
|
1081
|
+
*/
|
|
1082
|
+
function markdownImagePaths(content: string, baseDir: string, rootDir?: string): string[] {
|
|
1083
|
+
return markdownImageRefs(content).map(value => {
|
|
1084
|
+
if (isAbsolute(value)) return resolve(value)
|
|
1085
|
+
if (rootDir) {
|
|
1086
|
+
const found = assetRefCandidates(value, baseDir, rootDir).find(candidate => existsSync(candidate))
|
|
1087
|
+
if (found) return found
|
|
1088
|
+
}
|
|
1089
|
+
return resolve(baseDir, value)
|
|
1090
|
+
})
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
/** 按来源数一数「有多少张图没落位」,用来生成一条人能照着查的 warning。 */
|
|
1094
|
+
function countBySource(images: SourceImage[]): Map<string, number> {
|
|
1095
|
+
const result = new Map<string, number>()
|
|
1096
|
+
for (const image of images) result.set(image.sourceId, (result.get(image.sourceId) ?? 0) + 1)
|
|
1097
|
+
return result
|
|
569
1098
|
}
|
|
570
1099
|
|
|
571
1100
|
async function imageFingerprints(paths: string[]): Promise<string[]> {
|
|
@@ -710,7 +1239,6 @@ export type SourceTypeProvider = () => SourceTypeDef[]
|
|
|
710
1239
|
/** Wiki 卡片类型注册表的读取入口(0024):类型定义 + schema 生效方式。 */
|
|
711
1240
|
export interface WikiSchemaSnapshot {
|
|
712
1241
|
types: WikiTypeDef[]
|
|
713
|
-
mode: WikiSchemaMode
|
|
714
1242
|
}
|
|
715
1243
|
export type WikiSchemaProvider = () => WikiSchemaSnapshot
|
|
716
1244
|
|
|
@@ -794,6 +1322,30 @@ export interface KnowledgeRuntimeOptions {
|
|
|
794
1322
|
* 不能让「没有模型就跑不动」成为新的前提。
|
|
795
1323
|
*/
|
|
796
1324
|
modelCall?: ModelCall
|
|
1325
|
+
/**
|
|
1326
|
+
* **按需解析**模型接缝(0044)。
|
|
1327
|
+
*
|
|
1328
|
+
* 为什么不能只用上面那个 `modelCall`:它是**构造时读一次**的值,而插件 apply 的
|
|
1329
|
+
* 时刻早于 `agent-default-model` 那个 bundle 挂载 —— profile 的组合顺序是
|
|
1330
|
+
* 「package.json 的 bundles → cordis.patch.yml」,而 `agent-default-model` 在 patch 层。
|
|
1331
|
+
* 于是那一次读到的永远是 undefined,而它**再也不会变**,模型抽取一次都不跑。
|
|
1332
|
+
*
|
|
1333
|
+
* 实跑证据(2026-09-27):默认模型配的是 `opencode-go / deepseek-v4.1-flash`,而编译出来的
|
|
1334
|
+
* 卡片 `metadata.extraction` 是 `pass0`(规则路径),正文是整篇文章 —— 也就是用户报的
|
|
1335
|
+
* 「编译只是按目录截取」。静默降级了很久,因为没有任何一处说「模型抽取没生效」。
|
|
1336
|
+
*
|
|
1337
|
+
* 所以两者都给:`modelCall` 供测试注入固定值,`modelSeam` 供插件层延迟解析,
|
|
1338
|
+
* **并且必须能说出拿不到的原因** —— `why` 会原样进 `rebuild` 的 warnings。
|
|
1339
|
+
* 一个说不出理由的降级,等于没有降级信息。
|
|
1340
|
+
*/
|
|
1341
|
+
modelSeam?: () => ModelSeam
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
/** 一次模型接缝解析的结果。两件东西都要有交代:能不能用,以及不能用时为什么。 */
|
|
1345
|
+
export interface ModelSeam {
|
|
1346
|
+
call?: ModelCall
|
|
1347
|
+
/** 拿不到 `call` 的原因,人话。会被原样写进 `rebuild` 的 warnings。 */
|
|
1348
|
+
why?: string
|
|
797
1349
|
}
|
|
798
1350
|
|
|
799
1351
|
export class KnowledgeRuntime {
|
|
@@ -826,6 +1378,8 @@ export class KnowledgeRuntime {
|
|
|
826
1378
|
* 写入与删除后必须失效(见 `addSource` / `deleteDocument`)。
|
|
827
1379
|
*/
|
|
828
1380
|
private sourceIndex: Map<string, string> | undefined
|
|
1381
|
+
/** `path → sourceId`:目标路径被谁占着(0051 的重名判定)。与 `sourceIndex` 同一次遍历建。 */
|
|
1382
|
+
private sourcePathOwner: Map<string, string> | undefined
|
|
829
1383
|
/** 段内布局整理是否已经跑过(每个 runtime 一次)。 */
|
|
830
1384
|
private segmentsOrganized = false
|
|
831
1385
|
/**
|
|
@@ -863,8 +1417,10 @@ export class KnowledgeRuntime {
|
|
|
863
1417
|
* `memoryPath`,守卫换成「不许跑出 memoryDir」。
|
|
864
1418
|
*/
|
|
865
1419
|
private readonly memoryDir: string
|
|
866
|
-
/** #3b 的模型调用接缝;undefined =
|
|
867
|
-
private readonly
|
|
1420
|
+
/** #3b 的模型调用接缝;undefined = 退回纯规则抽取。测试注入的固定值。 */
|
|
1421
|
+
private readonly staticModelCall: ModelCall | undefined
|
|
1422
|
+
/** 每次真正要用模型时问一遍的那个接缝 provider(0044)。为空则退回 `staticModelCall`。 */
|
|
1423
|
+
private readonly modelSeamProvider?: () => ModelSeam
|
|
868
1424
|
|
|
869
1425
|
constructor(rootDir: string, config: Partial<RuntimeKnowledgeConfig> = {}, options: KnowledgeRuntimeOptions = {}) {
|
|
870
1426
|
this.rootDir = resolve(rootDir)
|
|
@@ -875,7 +1431,8 @@ export class KnowledgeRuntime {
|
|
|
875
1431
|
this.sourceTypeProvider = options.sourceTypes
|
|
876
1432
|
this.wikiSchemaProvider = options.wikiSchema
|
|
877
1433
|
this.memoryDir = options.memoryRoot ? resolve(options.memoryRoot) : join(this.rootDir, '04_memory')
|
|
878
|
-
this.
|
|
1434
|
+
this.staticModelCall = options.modelCall
|
|
1435
|
+
this.modelSeamProvider = options.modelSeam
|
|
879
1436
|
const resources = resolveResources(options.resourceRoot)
|
|
880
1437
|
this.pythonPath = config.python?.trim() || resources.python
|
|
881
1438
|
this.embeddingModelPath = config.embeddingModel?.trim() || resources.embeddingModel
|
|
@@ -883,6 +1440,17 @@ export class KnowledgeRuntime {
|
|
|
883
1440
|
this.embeddingModelId = resources.modelId(this.embeddingModelPath)
|
|
884
1441
|
}
|
|
885
1442
|
|
|
1443
|
+
/**
|
|
1444
|
+
* 现在能不能用模型抽实体。
|
|
1445
|
+
*
|
|
1446
|
+
* **每次现问**,不是在构造时定死:插件 apply 的时刻早于 `agent-default-model` 挂载,
|
|
1447
|
+
* 那时问一次只会得到 undefined,而定死之后就再也不会变(见 `modelCallProvider`)。
|
|
1448
|
+
*/
|
|
1449
|
+
private modelSeamNow(): ModelSeam {
|
|
1450
|
+
if (this.modelSeamProvider) return this.modelSeamProvider()
|
|
1451
|
+
return this.staticModelCall ? { call: this.staticModelCall } : { why: '运行时没有注入模型接缝' }
|
|
1452
|
+
}
|
|
1453
|
+
|
|
886
1454
|
/**
|
|
887
1455
|
* 就地应用一份新的运行时配置(0.1.7 的 volatile 改动走这条路)。
|
|
888
1456
|
*
|
|
@@ -935,30 +1503,33 @@ export class KnowledgeRuntime {
|
|
|
935
1503
|
|
|
936
1504
|
/** 当前生效的卡片 schema:类型定义 + 生效方式。 */
|
|
937
1505
|
wikiSchema(): WikiSchemaSnapshot {
|
|
938
|
-
return this.wikiSchemaProvider?.() ?? { types: defaultWikiTypes()
|
|
1506
|
+
return this.wikiSchemaProvider?.() ?? { types: defaultWikiTypes() }
|
|
939
1507
|
}
|
|
940
1508
|
|
|
941
1509
|
/**
|
|
942
|
-
* Agent 与设置面板共用的类型目录:注册表 +
|
|
943
|
-
* 现有卡片的三层校验结果(`report` 模式下这是"待补"清单)+
|
|
1510
|
+
* Agent 与设置面板共用的类型目录:注册表 + 内置只读判定规则 +
|
|
1511
|
+
* 现有卡片的三层校验结果(`report` 模式下这是"待补"清单)+ **review 进度**。
|
|
944
1512
|
*
|
|
945
|
-
*
|
|
946
|
-
*
|
|
1513
|
+
* review 计数挂在这里而不是另开一路,是因为 review 就发生在这一屏(「知识库 → Wiki」)——
|
|
1514
|
+
* 与 lint 待补清单是同一处,不新造第二套界面。
|
|
1515
|
+
*
|
|
1516
|
+
* 0044:`proposals`(待审提议条数)改成 `reviews`。队列退役之后「还有几条等人确认」
|
|
1517
|
+
* 这个问题不再有意义 —— 卡片一编译就入库了,剩下的问题是「入库的这些里,我看过几条」。
|
|
947
1518
|
*/
|
|
948
1519
|
async wikiTypeCatalog(): Promise<Envelope<WikiTypeCatalog>> {
|
|
949
1520
|
const schema = this.wikiSchema()
|
|
950
1521
|
const issues = await this.lintCards()
|
|
951
|
-
const
|
|
1522
|
+
const cards = await this.loadCards()
|
|
952
1523
|
return success({
|
|
953
1524
|
types: schema.types,
|
|
954
|
-
mode: schema.mode,
|
|
955
1525
|
rules: WIKI_TYPE_RULES,
|
|
956
1526
|
issues: issues.slice(0, 50),
|
|
957
1527
|
issueCount: issues.length,
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1528
|
+
reviews: {
|
|
1529
|
+
total: cards.length,
|
|
1530
|
+
unreviewed: cards.filter(card => card.review?.state !== 'reviewed').length,
|
|
1531
|
+
reviewed: cards.filter(card => card.review?.state === 'reviewed').length,
|
|
1532
|
+
anomalies: cards.filter(card => (card.annotations ?? []).some(item => item.kind === 'anomaly')).length,
|
|
962
1533
|
},
|
|
963
1534
|
})
|
|
964
1535
|
}
|
|
@@ -973,42 +1544,34 @@ export class KnowledgeRuntime {
|
|
|
973
1544
|
private diagnosis?: DiagnosisStore
|
|
974
1545
|
|
|
975
1546
|
diagnosisStore(): DiagnosisStore {
|
|
976
|
-
if (!this.diagnosis) this.diagnosis = new DiagnosisStore(this
|
|
1547
|
+
if (!this.diagnosis) this.diagnosis = new DiagnosisStore(this)
|
|
977
1548
|
return this.diagnosis
|
|
978
1549
|
}
|
|
979
1550
|
|
|
980
|
-
|
|
981
|
-
* 卡片提议的存储(0021 §6.1 的那道门)。
|
|
982
|
-
*
|
|
983
|
-
* 它按**记忆段**(`<agentRoot>/memory/<库 id>`)落,不在知识树里 —— 理由写在
|
|
984
|
-
* `proposals.ts` 顶上:提议是「关于 Agent 动作的待办留痕」,删掉只丢待办、
|
|
985
|
-
* 重跑一次提议就能回来,所以它既不属于「源头」(删了就丢数据的那一段),
|
|
986
|
-
* 更不能落 `02_wiki/`(那正是这道门要拦的事)。
|
|
987
|
-
*/
|
|
988
|
-
private proposalStore?: CardProposalStore
|
|
989
|
-
|
|
990
|
-
cardProposals(): CardProposalStore {
|
|
991
|
-
if (!this.proposalStore) {
|
|
992
|
-
this.proposalStore = new CardProposalStore({
|
|
993
|
-
memoryRoot: this.memoryDir,
|
|
994
|
-
schema: () => ({ types: this.wikiSchema().types }),
|
|
995
|
-
})
|
|
996
|
-
}
|
|
997
|
-
return this.proposalStore
|
|
998
|
-
}
|
|
999
|
-
|
|
1000
|
-
/* ────────────────────────────────────── 卡片提议:提议 → 确认 ── */
|
|
1551
|
+
/* ────────────────────────────────────── 卡片入库:直接写,不排队 ── */
|
|
1001
1552
|
|
|
1002
1553
|
/**
|
|
1003
|
-
*
|
|
1554
|
+
* 把一批卡片**直接写进 `02_wiki`**(0044)。
|
|
1004
1555
|
*
|
|
1005
|
-
*
|
|
1006
|
-
*
|
|
1007
|
-
*
|
|
1556
|
+
* 0040 v1.5 的这道门原来是「提一条提议 → 人在面板确认 → 才落卡」,理由是类型层是收敛的
|
|
1557
|
+
* 那一层,错一条所有现象的排序一起错。0044 把方向反过来:**入库在前、review 在后**。
|
|
1558
|
+
* 判据是实践中的两个事实:
|
|
1559
|
+
*
|
|
1560
|
+
* 1. **确认门把知识挡在了门外。** 抽到的东西停在 `memory/card-proposals/` 里等人点,
|
|
1561
|
+
* 而人不会一条条点。用户看到的现象就是「编译没生效」——文档导进去了,wiki 是空的。
|
|
1562
|
+
* 2. **review 的价值不需要排队来实现。** 「这张卡片对不对」可以在它已经可召回之后回答;
|
|
1563
|
+
* 而「未 review 就不许进来」等于把「暂时没人看的正确知识」和「错误知识」一起丢掉。
|
|
1564
|
+
*
|
|
1565
|
+
* 于是现在的形状是:入库 → 召回得到(评分稍低)→ 人 review → 评分抬高 / 备注异常点。
|
|
1566
|
+
* 拒绝某条知识的正确做法是 `set_knowledge_status(deprecated|problem)`,它会留痕,
|
|
1567
|
+
* 也能被撤回 —— 比「从来没见过它」更容易纠正。
|
|
1568
|
+
*
|
|
1569
|
+
* 与编译路径的差别只有一条:这里允许调用方指定 `entityId` 与正文,编译路径由候选推导。
|
|
1570
|
+
* 两者共用 `writeCard`,所以校验、版本快照、备注保留的行为完全一致。
|
|
1008
1571
|
*/
|
|
1009
|
-
async
|
|
1572
|
+
async writeCards(input: {
|
|
1010
1573
|
sourceId: string
|
|
1011
|
-
|
|
1574
|
+
cards: Array<{
|
|
1012
1575
|
action?: 'create' | 'update' | 'merge'
|
|
1013
1576
|
entityType: string
|
|
1014
1577
|
entityId: string
|
|
@@ -1018,132 +1581,210 @@ export class KnowledgeRuntime {
|
|
|
1018
1581
|
sourceRefs?: string[]
|
|
1019
1582
|
metadata?: Record<string, unknown>
|
|
1020
1583
|
}>
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1584
|
+
lineage?: LineageContext
|
|
1585
|
+
}): Promise<Envelope<{ written: number; ids: string[]; rejected: string[]; warnings: string[] }>> {
|
|
1586
|
+
const empty = { written: 0, ids: [] as string[], rejected: [] as string[], warnings: [] as string[] }
|
|
1587
|
+
if (!input.sourceId?.trim()) return failure(empty, 'SOURCE_REF_REQUIRED', '卡片必须挂在一条来源上', false, '先调 add_source 落一条来源,再用它的 source_id 写入卡片。')
|
|
1024
1588
|
const sources = new Set((await this.loadSources()).map(source => source.sourceId))
|
|
1025
1589
|
if (!sources.has(input.sourceId)) {
|
|
1026
1590
|
return failure(empty, 'SOURCE_REF_NOT_FOUND', `source_refs 引用不存在: ${input.sourceId}`, false, '先调用 add_source 写入对应的 SourceTruth')
|
|
1027
1591
|
}
|
|
1028
|
-
if (!Array.isArray(input.
|
|
1029
|
-
return failure(empty, '
|
|
1592
|
+
if (!Array.isArray(input.cards) || input.cards.length === 0) {
|
|
1593
|
+
return failure(empty, 'CARD_EMPTY', '至少要写一张卡片', true)
|
|
1030
1594
|
}
|
|
1031
1595
|
const schema = this.wikiSchema()
|
|
1032
1596
|
const rejected: string[] = []
|
|
1033
|
-
const
|
|
1597
|
+
const warnings: string[] = []
|
|
1598
|
+
const ids: string[] = []
|
|
1599
|
+
/**
|
|
1600
|
+
* 判重索引只读一次。**批量里的后一张要能看到前一张刚写的卡片** —— 同一个实体在
|
|
1601
|
+
* 一批里出现两次时,第二次必须走并入而不是再写一遍,所以写成功后要回填这张表。
|
|
1602
|
+
*/
|
|
1603
|
+
const byKey = new Map((await this.loadCards()).map(card => [`${card.entityType}:${card.entityId}`, card]))
|
|
1604
|
+
for (const item of input.cards) {
|
|
1034
1605
|
const type = schema.types.find(def => def.id === item.entityType)
|
|
1035
|
-
if (!type) { rejected.push(`${item.entityId}:未知卡片类型 ${item.entityType}`);
|
|
1036
|
-
if (!type.enabled) { rejected.push(`${item.entityId}:类型 ${item.entityType} 已停用`);
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1606
|
+
if (!type) { rejected.push(`${item.entityId}:未知卡片类型 ${item.entityType}`); continue }
|
|
1607
|
+
if (!type.enabled) { rejected.push(`${item.entityId}:类型 ${item.entityType} 已停用`); continue }
|
|
1608
|
+
/**
|
|
1609
|
+
* **已有卡片时走「并入」而不是覆盖。**
|
|
1610
|
+
*
|
|
1611
|
+
* 这一步是必需的,不是优化:Agent 按「同一实体跨两份文档」调用两次时,覆盖式写入会让
|
|
1612
|
+
* 第一份来源与它的正文**静默消失**(实测:先 doc:a 再 doc:b,卡片只剩 `["doc:b"]`)。
|
|
1613
|
+
* 退役的提议存储天然按身份去重并累积 `source_refs`,所以少了这一步就是相对旧链路的一处退化。
|
|
1614
|
+
*
|
|
1615
|
+
* 三条合并规则与编译路径完全相同,两处共用 `appendExtracted`:
|
|
1616
|
+
* 正文追加(自带文本判重)、`source_refs` 取并集、`metadata` 合并。
|
|
1617
|
+
* `status` 与人工备注 / review 由 `writeCard` 自己保留,不在这里重复照顾。
|
|
1618
|
+
*/
|
|
1619
|
+
const existing = byKey.get(`${item.entityType}:${item.entityId}`)
|
|
1620
|
+
const requestedRefs = item.sourceRefs ?? [input.sourceId]
|
|
1621
|
+
const sourceRefs = existing ? [...new Set([...existing.sourceRefs, ...requestedRefs])] : requestedRefs
|
|
1622
|
+
const content = existing ? appendExtracted(existing.content, [item.content]) : item.content
|
|
1623
|
+
const written = await this.writeCard({
|
|
1624
|
+
entityType: item.entityType as SchemaType,
|
|
1625
|
+
entityId: item.entityId,
|
|
1626
|
+
title: item.title,
|
|
1627
|
+
content,
|
|
1628
|
+
sourceRefs,
|
|
1629
|
+
...(item.summary ?? existing?.summary ? { summary: item.summary ?? existing?.summary } : {}),
|
|
1630
|
+
...(item.metadata || existing?.metadata ? { metadata: { ...(existing?.metadata ?? {}), ...(item.metadata ?? {}) } } : {}),
|
|
1631
|
+
...(existing ? { status: existing.status } : {}),
|
|
1632
|
+
...(existing?.owner ? { owner: existing.owner } : {}),
|
|
1633
|
+
...(existing?.provenance ? { provenance: existing.provenance } : {}),
|
|
1634
|
+
origin: 'compile',
|
|
1635
|
+
...(input.lineage ? { lineage: input.lineage } : {}),
|
|
1636
|
+
})
|
|
1637
|
+
if (written.ok === false) {
|
|
1638
|
+
rejected.push(`${item.entityId}:${written.error.code} ${written.error.message}`)
|
|
1639
|
+
continue
|
|
1640
|
+
}
|
|
1641
|
+
ids.push(item.entityId)
|
|
1642
|
+
// 回填:这一批里后面那张卡片要按**刚写下的**样子判重,而不是按开批之前的磁盘快照。
|
|
1643
|
+
byKey.set(`${item.entityType}:${item.entityId}`, {
|
|
1644
|
+
entityId: item.entityId, entityType: item.entityType as SchemaType, title: item.title,
|
|
1645
|
+
status: existing?.status ?? 'unreviewed', content, sourceRefs,
|
|
1646
|
+
...(item.summary ?? existing?.summary ? { summary: item.summary ?? existing?.summary } : {}),
|
|
1647
|
+
metadata: { ...(existing?.metadata ?? {}), ...(item.metadata ?? {}) }, updatedAt: new Date().toISOString(),
|
|
1648
|
+
})
|
|
1649
|
+
for (const warning of written.warnings ?? []) warnings.push(`${item.entityId}:${warning}`)
|
|
1650
|
+
}
|
|
1053
1651
|
return success(
|
|
1054
|
-
{
|
|
1055
|
-
rejected.length > 0 ? [`${rejected.length}
|
|
1652
|
+
{ written: ids.length, ids, rejected, warnings },
|
|
1653
|
+
rejected.length > 0 ? [`${rejected.length} 张卡片被拒:${rejected.join(';')}`] : [],
|
|
1056
1654
|
)
|
|
1057
1655
|
}
|
|
1058
1656
|
|
|
1657
|
+
/* ────────────────────────────────────── 入库之后:备注与 review ── */
|
|
1658
|
+
|
|
1059
1659
|
/**
|
|
1060
|
-
*
|
|
1660
|
+
* 给一张卡片加一条**人工备注**,或撤掉一条(0044)。
|
|
1061
1661
|
*
|
|
1062
|
-
*
|
|
1063
|
-
*
|
|
1064
|
-
*
|
|
1065
|
-
*
|
|
1066
|
-
*
|
|
1662
|
+
* 这是「wiki 不可修改」的对偶:正文改不了,但**判断改得了**。「这份文档的数字和卡片
|
|
1663
|
+
* 对不上」「这个口径在另一份文档里是反的」——这些都是关于卡片的判断,不是卡片的内容,
|
|
1664
|
+
* 所以它们落在 frontmatter 的 `annotations` 里,编译永远不碰。
|
|
1665
|
+
*
|
|
1666
|
+
* `kind: 'anomaly'` 会把卡片的有效评分压到 `ANOMALY_SCORE`(见 `effectiveReviewScore`):
|
|
1667
|
+
* 异常卡片照样召回得到,只是排在后面 —— 而不是被藏起来。藏起来的问题在于没人会再看到它,
|
|
1668
|
+
* 于是也就没人会去修。
|
|
1067
1669
|
*/
|
|
1068
|
-
async
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1670
|
+
async annotateCard(input: {
|
|
1671
|
+
entityId: string
|
|
1672
|
+
entityType?: SchemaType
|
|
1673
|
+
/** 不传 `note` 时必须是「撤销一条」:给出 `removeId`。 */
|
|
1674
|
+
kind?: 'note' | 'anomaly'
|
|
1675
|
+
note?: string
|
|
1676
|
+
author?: string
|
|
1677
|
+
sourceRef?: string
|
|
1678
|
+
removeId?: string
|
|
1679
|
+
lineage?: LineageContext
|
|
1680
|
+
}): Promise<Envelope<{ entityId: string; annotations: CardAnnotation[]; reviewScore: number }>> {
|
|
1681
|
+
const card = await this.readCard(input.entityId, input.entityType)
|
|
1682
|
+
if (!card) return failure({ entityId: input.entityId, annotations: [], reviewScore: 0 }, 'CARD_NOT_FOUND', `知识卡片不存在: ${input.entityId}`, true)
|
|
1683
|
+
const current = card.annotations ?? []
|
|
1684
|
+
let next: CardAnnotation[]
|
|
1685
|
+
if (input.removeId?.trim()) {
|
|
1686
|
+
next = current.filter(item => item.id !== input.removeId!.trim())
|
|
1687
|
+
if (next.length === current.length) {
|
|
1688
|
+
return failure({ entityId: input.entityId, annotations: current, reviewScore: effectiveReviewScore(card) }, 'ANNOTATION_NOT_FOUND', `备注不存在: ${input.removeId}`, true)
|
|
1689
|
+
}
|
|
1690
|
+
} else {
|
|
1691
|
+
const note = input.note?.trim() ?? ''
|
|
1692
|
+
if (!note) return failure({ entityId: input.entityId, annotations: current, reviewScore: effectiveReviewScore(card) }, 'ANNOTATION_NOTE_REQUIRED', 'note 不能为空', true)
|
|
1693
|
+
const at = new Date().toISOString()
|
|
1694
|
+
next = [...current, {
|
|
1695
|
+
id: annotationId(note, at),
|
|
1696
|
+
kind: input.kind === 'anomaly' ? 'anomaly' : 'note',
|
|
1697
|
+
note,
|
|
1698
|
+
...(input.author?.trim() ? { author: input.author.trim() } : {}),
|
|
1699
|
+
createdAt: at,
|
|
1700
|
+
...(input.sourceRef?.trim() ? { sourceRef: input.sourceRef.trim() } : {}),
|
|
1701
|
+
}]
|
|
1081
1702
|
}
|
|
1082
|
-
const written = await this.writeCard({
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
sourceRefs: proposal.sourceRefs,
|
|
1088
|
-
...(proposal.summary ? { summary: proposal.summary } : {}),
|
|
1089
|
-
...(proposal.metadata ? { metadata: proposal.metadata } : {}),
|
|
1090
|
-
...(lineage ? { lineage } : {}),
|
|
1091
|
-
})
|
|
1092
|
-
if (written.ok === false) return failure(empty, written.error.code, written.error.message, written.error.retryable, written.error.suggestion)
|
|
1093
|
-
await store.markWritten(proposalId)
|
|
1094
|
-
return success({ entityId: proposal.entityId, path: written.data.path, proposalId })
|
|
1703
|
+
const written = await this.writeCard({ ...card, sourceRefs: card.sourceRefs, annotations: next, origin: 'internal' as const })
|
|
1704
|
+
if (written.ok === false) return failure({ entityId: input.entityId, annotations: current, reviewScore: effectiveReviewScore(card) }, written.error.code, written.error.message, written.error.retryable, written.error.suggestion)
|
|
1705
|
+
await this.audit('wiki.annotate', { entityId: input.entityId, added: input.removeId ? 0 : 1, removed: input.removeId ? 1 : 0, count: next.length })
|
|
1706
|
+
const score = effectiveReviewScore({ annotations: next, ...(card.review ? { review: card.review } : {}) })
|
|
1707
|
+
return success({ entityId: input.entityId, annotations: next, reviewScore: score })
|
|
1095
1708
|
}
|
|
1096
1709
|
|
|
1097
1710
|
/**
|
|
1098
|
-
*
|
|
1711
|
+
* review 一张卡片(0044)。**它不是准入闸门** —— 卡片在 review 之前就已经入库、可召回了。
|
|
1099
1712
|
*
|
|
1100
|
-
*
|
|
1101
|
-
*
|
|
1713
|
+
* `review` 把评分抬到 `REVIEWED_SCORE_FLOOR`,`unreview` 放回出厂分。两个动作都可逆:
|
|
1714
|
+
* review 是一个可以被推翻的判断,不是一次性的凭证。
|
|
1102
1715
|
*/
|
|
1103
|
-
async
|
|
1104
|
-
|
|
1105
|
-
|
|
1716
|
+
async reviewCard(input: {
|
|
1717
|
+
entityId: string
|
|
1718
|
+
entityType?: SchemaType
|
|
1719
|
+
action: 'review' | 'unreview'
|
|
1106
1720
|
reviewer?: string
|
|
1107
|
-
reason?: string
|
|
1108
|
-
patch?: { title?: string; summary?: string; content?: string }
|
|
1109
1721
|
lineage?: LineageContext
|
|
1110
|
-
}): Promise<Envelope<{
|
|
1111
|
-
const
|
|
1722
|
+
}): Promise<Envelope<{ entityId: string; review: CardReview; reviewScore: number; status: KnowledgeStatus }>> {
|
|
1723
|
+
const card = await this.readCard(input.entityId, input.entityType)
|
|
1724
|
+
if (!card) return failure({ entityId: input.entityId, review: { state: 'unreviewed', score: CARD_REVIEW_BASE_SCORE }, reviewScore: CARD_REVIEW_BASE_SCORE, status: 'unreviewed' as KnowledgeStatus }, 'CARD_NOT_FOUND', `知识卡片不存在: ${input.entityId}`, true)
|
|
1725
|
+
const at = new Date().toISOString()
|
|
1112
1726
|
const reviewer = input.reviewer?.trim() || 'user:panel'
|
|
1113
|
-
const
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1727
|
+
const review: CardReview = input.action === 'review'
|
|
1728
|
+
? { state: 'reviewed', score: Math.max(card.review?.score ?? 0, REVIEWED_SCORE_FLOOR), reviewedAt: at, reviewedBy: reviewer }
|
|
1729
|
+
: { state: 'unreviewed', score: CARD_REVIEW_BASE_SCORE }
|
|
1730
|
+
/**
|
|
1731
|
+
* 状态跟着一起走(0044 实机验证第七批)。
|
|
1732
|
+
*
|
|
1733
|
+
* 卡片有两个字段:`status` 是**生命周期**(编译器写它,人在 `set_knowledge_status`
|
|
1734
|
+
* 里改),`review` 是**人的判断 + 评分**。它们本来描述不同的事,但「已经 review 过」
|
|
1735
|
+
* 的卡片在预览里显示 `状态 unreviewed` 会让人以为刚才那一下没生效 —— 实机验证时
|
|
1736
|
+
* 看到的就是这个组合。
|
|
1737
|
+
*
|
|
1738
|
+
* 所以:
|
|
1739
|
+
* - `review` 通过时,`unreviewed` / `draft` 推进到 `reviewed`(这个词本来就在
|
|
1740
|
+
* 状态词表里,含义就是这个);
|
|
1741
|
+
* - 撤销 review 时只把 `reviewed` 退回 `unreviewed`,**不动** `active` / `deprecated` /
|
|
1742
|
+
* `problem` —— 那些是人另外做过的决定,不该被一次撤销顺手抹掉。
|
|
1743
|
+
*/
|
|
1744
|
+
const nextStatus: KnowledgeStatus = input.action === 'review'
|
|
1745
|
+
? (card.status === 'unreviewed' || card.status === 'draft' ? 'reviewed' : card.status)
|
|
1746
|
+
: (card.status === 'reviewed' ? 'unreviewed' : card.status)
|
|
1747
|
+
const written = await this.writeCard({ ...card, status: nextStatus, sourceRefs: card.sourceRefs, review, origin: 'review' as const })
|
|
1748
|
+
if (written.ok === false) {
|
|
1749
|
+
const current: CardReview = card.review ?? { state: 'unreviewed', score: CARD_REVIEW_BASE_SCORE }
|
|
1750
|
+
return failure({ entityId: input.entityId, review: current, reviewScore: effectiveReviewScore(card), status: card.status }, written.error.code, written.error.message, written.error.retryable, written.error.suggestion)
|
|
1118
1751
|
}
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
if (input.action === 'confirm') return success({ proposalId: confirmed.id, status: confirmed.status, written: false })
|
|
1122
|
-
const applied = await this.applyProposal(confirmed.id, input.lineage)
|
|
1123
|
-
if (applied.ok === false) return failure(empty, applied.error.code, applied.error.message, applied.error.retryable, applied.error.suggestion)
|
|
1124
|
-
return success({ proposalId: confirmed.id, status: 'confirmed', written: true })
|
|
1752
|
+
await this.audit('wiki.review', { entityId: input.entityId, action: input.action, reviewer, score: review.score })
|
|
1753
|
+
return success({ entityId: input.entityId, review, reviewScore: effectiveReviewScore({ ...card, review }), status: nextStatus })
|
|
1125
1754
|
}
|
|
1126
1755
|
|
|
1127
|
-
/**
|
|
1756
|
+
/**
|
|
1757
|
+
* 对已落库的卡片跑三层校验。不写盘,只报告。
|
|
1758
|
+
*
|
|
1759
|
+
* 0049 补齐了 `reference` 那一层:正文里的 `[[类型/名称]]` 指向一张不存在的卡片,
|
|
1760
|
+
* 就是一条断链。此前这一层只有类型定义里的名字,没有任何产生者 —— 而「引用错说明
|
|
1761
|
+
* 图断了」正是它要回答的问题(format 错说明没填完,语义错说明这条知识没用)。
|
|
1762
|
+
*/
|
|
1128
1763
|
async lintCards(): Promise<Array<CardSchemaIssue & { entityId: string; entityType: string }>> {
|
|
1129
1764
|
const schema = this.wikiSchema()
|
|
1130
1765
|
const byId = new Map(schema.types.map(def => [def.id, def]))
|
|
1131
1766
|
const result: Array<CardSchemaIssue & { entityId: string; entityType: string }> = []
|
|
1132
|
-
|
|
1767
|
+
const cards = await this.loadCards()
|
|
1768
|
+
for (const card of cards) {
|
|
1133
1769
|
const type = byId.get(card.entityType)
|
|
1134
1770
|
if (!type) continue
|
|
1135
1771
|
for (const issue of validateCardAgainstType(type, card)) {
|
|
1136
1772
|
result.push({ ...issue, entityId: card.entityId, entityType: card.entityType })
|
|
1137
1773
|
}
|
|
1138
1774
|
}
|
|
1775
|
+
// 断链与类型无关(一张类型没登记的卡片照样能有链接),所以单独扫一遍全部卡片。
|
|
1776
|
+
for (const link of deadLinksOf(cards)) {
|
|
1777
|
+
result.push({ ...deadLinkIssue(link.target), entityId: link.entityId, entityType: link.entityType })
|
|
1778
|
+
}
|
|
1139
1779
|
return result
|
|
1140
1780
|
}
|
|
1141
1781
|
|
|
1142
1782
|
/**
|
|
1143
|
-
*
|
|
1783
|
+
* 类型闸门 + 三层校验。
|
|
1144
1784
|
*
|
|
1145
|
-
* 返回 `{ type, issues }`
|
|
1146
|
-
*
|
|
1785
|
+
* 返回 `{ type, issues }` 或一个失败信封(只可能因为**未知类型** —— 没有类型定义就连
|
|
1786
|
+
* 「缺什么」都说不出来)。issues **不决定写不写**:0044 起卡片一律直接入库,问题写进
|
|
1787
|
+
* `metadata.schema_issues` 由面板显示成「待补」。原来的 `strict` / `report` 开关一并撤掉了。
|
|
1147
1788
|
*/
|
|
1148
1789
|
private cardSchemaGate(input: { entityType: string; title: string; content: string; sourceRefs: string[]; metadata: Record<string, unknown>; owner?: string; summary?: string }):
|
|
1149
1790
|
{ type: WikiTypeDef; issues: CardSchemaIssue[] } | { error: KnowledgeError } {
|
|
@@ -1215,6 +1856,13 @@ export class KnowledgeRuntime {
|
|
|
1215
1856
|
}
|
|
1216
1857
|
|
|
1217
1858
|
directory(name: typeof LAYOUT[number]): string { return join(this.rootDir, name) }
|
|
1859
|
+
/**
|
|
1860
|
+
* 0051 之前的段目录路径。
|
|
1861
|
+
*
|
|
1862
|
+
* 它**不在 `LAYOUT` 里**:`LAYOUT` 是「应该建出来的目录」清单,而老段名只是要读、
|
|
1863
|
+
* 要搬走的东西 —— 把它加进 `LAYOUT`,`ensureLayout` 每次启动都会把那个空目录建回来。
|
|
1864
|
+
*/
|
|
1865
|
+
private legacySourceDir(): string { return join(this.rootDir, LEGACY_SOURCE_TRUTH) }
|
|
1218
1866
|
async ensureLayout(): Promise<void> {
|
|
1219
1867
|
await Promise.all(LAYOUT.map(name => mkdir(this.directory(name), { recursive: true })))
|
|
1220
1868
|
// 记忆段在知识树**外面**(0032),所以要单独建。它与 LAYOUT 一起建的理由相同:
|
|
@@ -1225,13 +1873,109 @@ export class KnowledgeRuntime {
|
|
|
1225
1873
|
// 写入与统计的入口。实例级标志保证每个 runtime 只扫一次。
|
|
1226
1874
|
if (!this.segmentsOrganized) {
|
|
1227
1875
|
this.segmentsOrganized = true
|
|
1876
|
+
// 0051 的改名排在最前:后面几个迁移都按 `SOURCE_TRUTH` 扫目录,先把树搬过来,
|
|
1877
|
+
// 它们就只需要认一个名字(而老段名只留兼容解析那一处)。
|
|
1878
|
+
await this.migrateSourceTree()
|
|
1228
1879
|
await this.migrateSourceTruthLayout()
|
|
1229
1880
|
await this.dropRetiredSegments()
|
|
1230
1881
|
await this.migrateLegacySourceTypeRename()
|
|
1231
1882
|
await this.migrateLegacyDiagnosisCards()
|
|
1883
|
+
await this.migrateSourceFrontMatter()
|
|
1884
|
+
await this.migrateSourceFileNames()
|
|
1885
|
+
}
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1888
|
+
/**
|
|
1889
|
+
* 0051:`01_source_truth/` → `01_source_raw/`。
|
|
1890
|
+
*
|
|
1891
|
+
* **整棵树一次 `rename`**,不逐文件搬:`_assets/` 就在那个目录里,逐文件搬会把
|
|
1892
|
+
* 图片落下的概率变成「每个文件各一次」。一次 rename 是原子的,要么全过来要么全没动。
|
|
1893
|
+
*
|
|
1894
|
+
* 两种中间态都要处理:
|
|
1895
|
+
* - 目标不存在 → 直接 rename;
|
|
1896
|
+
* - 两个目录都在(上一次搬到一半)→ 逐项合并:目标里没有的搬过去,两边都有的
|
|
1897
|
+
* (`_assets` 与分类目录)递归合并,最后删掉空壳。
|
|
1898
|
+
*
|
|
1899
|
+
* 失败不回滚、不抛:`loadSources` 两个段名都扫,搬到一半的库照常可读,
|
|
1900
|
+
* 下一次启动接着搬。
|
|
1901
|
+
*/
|
|
1902
|
+
private async migrateSourceTree(): Promise<void> {
|
|
1903
|
+
const from = this.legacySourceDir()
|
|
1904
|
+
const to = this.directory(SOURCE_TRUTH)
|
|
1905
|
+
if (!existsSync(from)) return
|
|
1906
|
+
try {
|
|
1907
|
+
if (!existsSync(to)) {
|
|
1908
|
+
await rename(from, to)
|
|
1909
|
+
} else {
|
|
1910
|
+
await mergeDir(from, to)
|
|
1911
|
+
await rmdir(from).catch(() => { /* 还有搬不动的,下次再来 */ })
|
|
1912
|
+
}
|
|
1913
|
+
this.sourceIndex = undefined
|
|
1914
|
+
this.sourcePathOwner = undefined
|
|
1915
|
+
this.migrationProblems.push('段目录已改名为 01_source_raw(原 01_source_truth)')
|
|
1916
|
+
} catch (cause) {
|
|
1917
|
+
this.migrationProblems.push(`段目录改名未完成,下次启动继续:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
1232
1918
|
}
|
|
1233
1919
|
}
|
|
1234
1920
|
|
|
1921
|
+
/**
|
|
1922
|
+
* 0051:文件名改成标题。
|
|
1923
|
+
*
|
|
1924
|
+
* 只动**名字不对**的那些(可重入),而且**只 rename,不改内容** —— 正文里那些
|
|
1925
|
+
* `../../01_source_truth/_assets/…` 的老引用一个字节都不动,解析器认两个段名
|
|
1926
|
+
* (见 `assetRefCandidates`)。改内容会造内容版本、让卡片重编重编码,代价远大于
|
|
1927
|
+
* 一个字符串好不好看。
|
|
1928
|
+
*
|
|
1929
|
+
* 目标路径被别的来源占着时**不覆盖**:加内容哈希后缀并记一条,让人去改标题
|
|
1930
|
+
* (与 `addSource` 的默认拒收同一条纪律,只是这里不能拒 —— 那是历史数据)。
|
|
1931
|
+
*/
|
|
1932
|
+
private async migrateSourceFileNames(): Promise<void> {
|
|
1933
|
+
const sources = await this.loadSources()
|
|
1934
|
+
const index = await this.sourceIndexMap()
|
|
1935
|
+
const taken = new Map<string, string>()
|
|
1936
|
+
for (const source of sources) {
|
|
1937
|
+
const path = index.get(slug(source.sourceId))
|
|
1938
|
+
if (path === undefined) continue
|
|
1939
|
+
taken.set(path, source.sourceId)
|
|
1940
|
+
}
|
|
1941
|
+
for (const source of sources) {
|
|
1942
|
+
const path = index.get(slug(source.sourceId))
|
|
1943
|
+
if (path === undefined) continue
|
|
1944
|
+
/**
|
|
1945
|
+
* 没有合法 `type` 的文件**留在原地**(与 0030 同一条纪律:宁可少搬,不可搬错)。
|
|
1946
|
+
*
|
|
1947
|
+
* `loadSources` 把读不出分类的来源当成 `document`(读得到总比丢掉好),所以
|
|
1948
|
+
* 「默认成了 document」与「真的写着 document」在这里长得一样 —— 必须回文件里
|
|
1949
|
+
* 看那一行,否则一个没有 front matter 的散文件会被搬进 `document/`。
|
|
1950
|
+
*/
|
|
1951
|
+
if (!(await this.hasDeclaredSourceType(path))) continue
|
|
1952
|
+
let target = this.sourcePath(source.sourceId, source.sourceType, source.title)
|
|
1953
|
+
if (target === path) continue
|
|
1954
|
+
const occupant = taken.get(target)
|
|
1955
|
+
if (occupant !== undefined && occupant !== source.sourceId) {
|
|
1956
|
+
target = this.safePath(SOURCE_TRUTH, source.sourceType, `${sourceFileBase(source.title, source.sourceId)}-${hash(source.sourceId).slice(0, 6)}.md`)
|
|
1957
|
+
this.migrationProblems.push(`来源 ${source.sourceId} 与 ${occupant} 标题相同(「${source.title}」),已落成 ${basename(target)};建议改标题`)
|
|
1958
|
+
}
|
|
1959
|
+
try {
|
|
1960
|
+
await mkdir(dirname(target), { recursive: true })
|
|
1961
|
+
await rename(path, target)
|
|
1962
|
+
taken.delete(path)
|
|
1963
|
+
taken.set(target, source.sourceId)
|
|
1964
|
+
} catch (cause) {
|
|
1965
|
+
this.migrationProblems.push(`来源 ${source.sourceId} 改名失败,保持原名:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
this.sourceIndex = undefined
|
|
1969
|
+
this.sourcePathOwner = undefined
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
/** 这份文件自己声明了合法分类吗(`migrateSourceFileNames` 只搬声明过的)。 */
|
|
1973
|
+
private async hasDeclaredSourceType(path: string): Promise<boolean> {
|
|
1974
|
+
try {
|
|
1975
|
+
return SOURCE_TYPE_ID_PATTERN.test(sourceTypeOf(parseMarkdown(await readFile(path, 'utf8')).frontmatter))
|
|
1976
|
+
} catch { return false }
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1235
1979
|
/**
|
|
1236
1980
|
* 0030:把平铺的 `<段>/*.md` 搬进 `<段>/<source_type>/*.md`。
|
|
1237
1981
|
*
|
|
@@ -1249,7 +1993,7 @@ export class KnowledgeRuntime {
|
|
|
1249
1993
|
const from = join(root, entry.name)
|
|
1250
1994
|
let sourceType = ''
|
|
1251
1995
|
try {
|
|
1252
|
-
sourceType =
|
|
1996
|
+
sourceType = sourceTypeOf(parseMarkdown(await readFile(from, 'utf8')).frontmatter)
|
|
1253
1997
|
} catch { continue }
|
|
1254
1998
|
if (!SOURCE_TYPE_ID_PATTERN.test(sourceType)) continue
|
|
1255
1999
|
const target = join(root, sourceType, entry.name)
|
|
@@ -1262,6 +2006,64 @@ export class KnowledgeRuntime {
|
|
|
1262
2006
|
this.sourceIndex = undefined
|
|
1263
2007
|
}
|
|
1264
2008
|
|
|
2009
|
+
/**
|
|
2010
|
+
* 0048:把存量 SourceRaw 的 front matter 改成 OKF v0.2 的形状。
|
|
2011
|
+
*
|
|
2012
|
+
* 改三件事,都只搬**已经存在**的值,不猜、不造:
|
|
2013
|
+
*
|
|
2014
|
+
* 1. `source_type` → `type`(OKF 唯一必填键)。旧键删掉 —— 同一个事实留两份,
|
|
2015
|
+
* 迟早会出现「改了这份、那份没改」。读取侧两个键都认(见 `sourceTypeOf`),
|
|
2016
|
+
* 所以迁移没跑到、或别的库还是旧形状,都不影响可用性。
|
|
2017
|
+
* 2. `metadata.source_url` → 顶层 `resource`(OKF 的「底层资产地址」)。
|
|
2018
|
+
* 3. 补 `generated{by,at}` 与 `sources[]`:`by` 取自 `metadata.parser`(谁解析的),
|
|
2019
|
+
* `at` 取自 `created_at`(这份文件什么时候写出来的)。**作者与发布时间补不出来**
|
|
2020
|
+
* —— 0048 之前适配器把它们抽出来就扔了,这里如实留空,不编一个。
|
|
2021
|
+
*
|
|
2022
|
+
* 正文一个字节都不动,也不制造内容版本(版本只认正文变化)。整个迁移可重入:
|
|
2023
|
+
* 已经有 `type` 的文件直接跳过。
|
|
2024
|
+
*/
|
|
2025
|
+
private async migrateSourceFrontMatter(): Promise<void> {
|
|
2026
|
+
for (const path of await filesUnder(this.directory(SOURCE_TRUTH))) {
|
|
2027
|
+
let parsed
|
|
2028
|
+
try { parsed = parseMarkdown(await readFile(path, 'utf8')) } catch { continue }
|
|
2029
|
+
const frontmatter = parsed.frontmatter
|
|
2030
|
+
if (typeof frontmatter.type === 'string' && frontmatter.type.trim()) continue
|
|
2031
|
+
const sourceType = sourceTypeOf(frontmatter) || 'document'
|
|
2032
|
+
const metadata = plainRecord(frontmatter.metadata)
|
|
2033
|
+
const resource = typeof metadata.source_url === 'string' && metadata.source_url.trim() ? metadata.source_url.trim() : undefined
|
|
2034
|
+
const parser = typeof metadata.parser === 'string' && metadata.parser.trim() ? metadata.parser.trim() : undefined
|
|
2035
|
+
const createdAt = typeof frontmatter.created_at === 'string' ? frontmatter.created_at : undefined
|
|
2036
|
+
const title = typeof frontmatter.title === 'string' ? frontmatter.title : undefined
|
|
2037
|
+
/**
|
|
2038
|
+
* `metadata.source_url` 与顶层 `resource` 是同一个事实:搬上去之后就把旧的删掉
|
|
2039
|
+
* —— 两份键迟早出现「改了这份、那份没改」。`source_path` 同理:它指的是暂存目录里
|
|
2040
|
+
* 那个**已经被删掉**的文件,留着它只会让人以为还能找回来(底稿现在由交付目录的
|
|
2041
|
+
* `source/` 承担,0048 起才有)。
|
|
2042
|
+
*/
|
|
2043
|
+
const { source_url: _promoted, source_path: _stale, ...restMetadata } = metadata
|
|
2044
|
+
const { source_type: _legacy, ...rest } = frontmatter
|
|
2045
|
+
const nextMetadata = Object.keys(restMetadata).length > 0 ? restMetadata : undefined
|
|
2046
|
+
const next: Record<string, unknown> = {
|
|
2047
|
+
type: sourceType,
|
|
2048
|
+
...(title !== undefined ? { title } : {}),
|
|
2049
|
+
...(resource !== undefined ? { resource } : {}),
|
|
2050
|
+
...(parser !== undefined || createdAt !== undefined
|
|
2051
|
+
? { generated: { ...(parser !== undefined ? { by: `convert/${parser}` } : {}), ...(createdAt !== undefined ? { at: createdAt } : {}) } }
|
|
2052
|
+
: {}),
|
|
2053
|
+
...(resource !== undefined ? { sources: [{ id: 'origin', resource, ...(title !== undefined ? { title } : {}) }] } : {}),
|
|
2054
|
+
...rest,
|
|
2055
|
+
...(nextMetadata !== undefined ? { metadata: nextMetadata } : {}),
|
|
2056
|
+
}
|
|
2057
|
+
try {
|
|
2058
|
+
await writeTextAtomic(path, markdown(next, parsed.content))
|
|
2059
|
+
} catch (cause) {
|
|
2060
|
+
this.migrationProblems.push(`来源 ${basename(path)} 的 front matter 迁移失败:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
2061
|
+
}
|
|
2062
|
+
}
|
|
2063
|
+
this.sourceIndex = undefined
|
|
2064
|
+
this.sourceCorpusCache = undefined
|
|
2065
|
+
}
|
|
2066
|
+
|
|
1265
2067
|
/**
|
|
1266
2068
|
* 0030:删掉已经被撤掉的 `00_schema` 段。
|
|
1267
2069
|
*
|
|
@@ -1309,9 +2111,9 @@ export class KnowledgeRuntime {
|
|
|
1309
2111
|
}
|
|
1310
2112
|
|
|
1311
2113
|
/**
|
|
1312
|
-
* 0040 §3.2 的一次性迁移:`
|
|
2114
|
+
* 0040 §3.2 的一次性迁移:`01_source_raw/ai_chat_message/` → `agent_chat_message/`。
|
|
1313
2115
|
*
|
|
1314
|
-
* 改名不能只改注册表:`
|
|
2116
|
+
* 改名不能只改注册表:`01_source_raw/` 是**按分类分目录**的,存量来源的
|
|
1315
2117
|
* frontmatter 里写着 `source_type: ai_chat_message`。只改注册表会有两个后果 ——
|
|
1316
2118
|
* 旧目录里那些来源变成一个「注册表里没有的分类」(界面上显示成未知分类),
|
|
1317
2119
|
* 而旧分类因为还有来源在,永远删不掉。
|
|
@@ -1345,9 +2147,9 @@ export class KnowledgeRuntime {
|
|
|
1345
2147
|
await unlinkIfExists(source)
|
|
1346
2148
|
continue
|
|
1347
2149
|
}
|
|
1348
|
-
await writeTextAtomic(target, markdown({ ...parsed.frontmatter, source_type: 'agent_chat_message' }, parsed.content))
|
|
2150
|
+
await writeTextAtomic(target, markdown({ ...parsed.frontmatter, type: 'agent_chat_message', source_type: 'agent_chat_message' }, parsed.content))
|
|
1349
2151
|
const roundTrip = parseMarkdown(await readFile(target, 'utf8'))
|
|
1350
|
-
if (roundTrip.content.trim() !== parsed.content.trim() || roundTrip.frontmatter
|
|
2152
|
+
if (roundTrip.content.trim() !== parsed.content.trim() || sourceTypeOf(roundTrip.frontmatter) !== 'agent_chat_message') {
|
|
1351
2153
|
// 读回来对不上:把目标撤掉,旧的那份留着 —— 宁可迁移没生效,也不要留半份。
|
|
1352
2154
|
await unlinkIfExists(target)
|
|
1353
2155
|
conflicts.push(`${sourceId}(迁移后比对不一致,已放弃)`)
|
|
@@ -1357,7 +2159,7 @@ export class KnowledgeRuntime {
|
|
|
1357
2159
|
}
|
|
1358
2160
|
try { await rmdir(from) } catch { /* 还有别的东西在里面 */ }
|
|
1359
2161
|
if (conflicts.length > 0) {
|
|
1360
|
-
this.migrationProblems.push(`来源分类改名迁移有 ${conflicts.length} 处冲突,旧文件保留在
|
|
2162
|
+
this.migrationProblems.push(`来源分类改名迁移有 ${conflicts.length} 处冲突,旧文件保留在 01_source_raw/ai_chat_message/:${conflicts.join('、')}`)
|
|
1361
2163
|
}
|
|
1362
2164
|
}
|
|
1363
2165
|
|
|
@@ -1716,10 +2518,9 @@ export class KnowledgeRuntime {
|
|
|
1716
2518
|
only?: string; incremental?: boolean; skipped?: boolean
|
|
1717
2519
|
cardsReused?: number; cardsRecompiled?: number; cardsAdded?: number; cardsRemoved?: number
|
|
1718
2520
|
notesReused?: number; notesRecompiled?: number
|
|
1719
|
-
// Pass 0
|
|
1720
|
-
// v1.5 把 `cardsCreated` / `cardsMerged` 改成提议计数 —— Pass 0 不再落卡。
|
|
2521
|
+
// Pass 0 的计数:面板据此显示「这一轮抽了几个候选、新建/更新了几张卡片」。
|
|
1721
2522
|
candidatesExtracted?: number; candidatesRejected?: number
|
|
1722
|
-
|
|
2523
|
+
cardsCreated?: number; cardsUpdated?: number
|
|
1723
2524
|
relationsExtracted?: number; relationsDropped?: number
|
|
1724
2525
|
sourcesExtracted?: number; sourcesSkipped?: number
|
|
1725
2526
|
}
|
|
@@ -1740,7 +2541,7 @@ export class KnowledgeRuntime {
|
|
|
1740
2541
|
memoryFiles: await fileCount(this.memoryDir),
|
|
1741
2542
|
/**
|
|
1742
2543
|
* 按**段名**取,不按位置取。这里原先是 `counts[1]`,而它之所以曾经是对的,
|
|
1743
|
-
* 只是因为 `LAYOUT[1]` 恰好是 `
|
|
2544
|
+
* 只是因为 `LAYOUT[1]` 恰好是 `01_source_raw` —— 0030 撤掉 `00_schema`
|
|
1744
2545
|
* 之后同一条表达式就静默变成了「wiki 卡片数」,界面上的来源数直接归零。
|
|
1745
2546
|
* 布局的索引是实现的偶然,段名才是契约。
|
|
1746
2547
|
*/
|
|
@@ -1779,7 +2580,133 @@ export class KnowledgeRuntime {
|
|
|
1779
2580
|
}
|
|
1780
2581
|
}
|
|
1781
2582
|
|
|
1782
|
-
|
|
2583
|
+
/**
|
|
2584
|
+
* 批量写入来源 + **一次**编译(0045)。
|
|
2585
|
+
*
|
|
2586
|
+
* ## 为什么值得单独一个方法,而不是让调用方循环 `add_source` + `rebuild_knowledge`
|
|
2587
|
+
*
|
|
2588
|
+
* 三件事只有放在一起才对:
|
|
2589
|
+
*
|
|
2590
|
+
* 1. **一次审批。** Agent 侧写入都要过 DSH 的审批闸门(`tools/pre-execute` 里那几个
|
|
2591
|
+
* 写工具)。十份文档 = 十次弹窗,而人真正要同意的是「把这十份导进去」这一件事。
|
|
2592
|
+
* 2. **编译只跑一次。** 每写一份就 `rebuild` 一次,第 10 份文档要付 10 次抽取的钱,
|
|
2593
|
+
* 而前 9 次的结果当场就被下一次覆盖。
|
|
2594
|
+
* 3. **失败要局部化。** 第 7 份文档格式不对,不该让前 6 份白写、后 3 份不写。
|
|
2595
|
+
* 所以逐条记录成败,最后一次性汇报 —— 成功的已经在库里,失败的带着原因回来,
|
|
2596
|
+
* 调用方可以只重试那几条。
|
|
2597
|
+
*
|
|
2598
|
+
* ## 为什么全部失败时返回 failure 而不是带 warning 的 success
|
|
2599
|
+
*
|
|
2600
|
+
* 「10 份全没进去」与「10 份都进去了,其中 1 份顺带报了个警告」是两件完全不同的事,
|
|
2601
|
+
* 用同一种信封表达,模型会把前一种当成后一种继续往下走。所以一份都没写成时如实失败,
|
|
2602
|
+
* 每条的原因仍然在 `sources[].message` 里。
|
|
2603
|
+
*/
|
|
2604
|
+
async importSources(input: {
|
|
2605
|
+
sources: Array<{ sourceType: SourceType; title?: string; content: string; externalId?: string; metadata?: Record<string, unknown>; allowDuplicateTitle?: boolean }>
|
|
2606
|
+
/** 默认编译(`only: 'wiki'`)。关掉它就只做写入,留给调用方自己决定何时编译。 */
|
|
2607
|
+
compile?: boolean
|
|
2608
|
+
lineage?: LineageContext
|
|
2609
|
+
}): Promise<Envelope<ImportSourcesResult>> {
|
|
2610
|
+
const empty: ImportSourcesResult = { written: [], failed: [], attempted: 0, compile: null }
|
|
2611
|
+
const items = Array.isArray(input.sources) ? input.sources : []
|
|
2612
|
+
if (items.length === 0) return failure(empty, 'SOURCES_REQUIRED', 'sources 不能为空:至少要给一份来源', true)
|
|
2613
|
+
/**
|
|
2614
|
+
* 上限只防「一次调用把整台机器按住」:编译是逐份来源串行抽实体的,50 份已经要跑很久。
|
|
2615
|
+
* 它**不是**产品配额 —— 超过就分批调,每批各自编译。
|
|
2616
|
+
*/
|
|
2617
|
+
if (items.length > MAX_IMPORT_SOURCES) {
|
|
2618
|
+
return failure(empty, 'IMPORT_TOO_MANY', `一次最多导入 ${MAX_IMPORT_SOURCES} 份来源,收到 ${items.length} 份;请分批调用`, true)
|
|
2619
|
+
}
|
|
2620
|
+
|
|
2621
|
+
const written: ImportedSourceReport[] = []
|
|
2622
|
+
const failed: ImportFailure[] = []
|
|
2623
|
+
const warnings: string[] = []
|
|
2624
|
+
for (const [index, item] of items.entries()) {
|
|
2625
|
+
const title = item?.title?.trim() || undefined
|
|
2626
|
+
const result = await this.addSource({
|
|
2627
|
+
sourceType: item?.sourceType,
|
|
2628
|
+
...(title ? { title } : {}),
|
|
2629
|
+
content: item?.content,
|
|
2630
|
+
...(item?.externalId ? { externalId: item.externalId } : {}),
|
|
2631
|
+
...(item?.metadata ? { metadata: item.metadata } : {}),
|
|
2632
|
+
...(item?.allowDuplicateTitle === true ? { allowDuplicateTitle: true } : {}),
|
|
2633
|
+
...(input.lineage ? { lineage: input.lineage } : {}),
|
|
2634
|
+
})
|
|
2635
|
+
if (result.ok === false) {
|
|
2636
|
+
const message = result.error?.message ?? '写入失败'
|
|
2637
|
+
failed.push({ index, ...(title ? { title } : {}), code: result.error?.code ?? 'SOURCE_WRITE_FAILED', message })
|
|
2638
|
+
// 逐条失败也要出现在 warnings 里:`ImportSourcesResult.failed` 是结构化账本,
|
|
2639
|
+
// 而 warnings 是不管谁在读都必须看见的那一句。
|
|
2640
|
+
warnings.push(`第 ${index + 1} 份来源${title ? `「${title}」` : ''}没有写入:${message}`)
|
|
2641
|
+
continue
|
|
2642
|
+
}
|
|
2643
|
+
written.push({
|
|
2644
|
+
index,
|
|
2645
|
+
sourceId: result.data.sourceId,
|
|
2646
|
+
...(title ? { title } : {}),
|
|
2647
|
+
path: result.data.path,
|
|
2648
|
+
bytes: typeof item?.content === 'string' ? item.content.length : 0,
|
|
2649
|
+
})
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
if (written.length === 0) {
|
|
2653
|
+
return failure({ ...empty, failed, attempted: items.length }, 'IMPORT_ALL_FAILED', `${items.length} 份来源全部写入失败,没有任何东西进入知识库`, true,
|
|
2654
|
+
failed[0] ? `第一份的原因:${failed[0].message}` : undefined)
|
|
2655
|
+
}
|
|
2656
|
+
|
|
2657
|
+
/**
|
|
2658
|
+
* 编译。**只要有一份写成就编**:成功的那几份已经是库里的知识,
|
|
2659
|
+
* 「因为同批里有一份失败就不编」等于把已经进来的知识又挡在门外一次。
|
|
2660
|
+
*/
|
|
2661
|
+
let compile: ImportCompileReport | null = null
|
|
2662
|
+
if (input.compile !== false) {
|
|
2663
|
+
const rebuilt = await this.rebuild('wiki', input.lineage)
|
|
2664
|
+
if (rebuilt.ok === false) {
|
|
2665
|
+
warnings.push(`来源已写入,但编译失败:${rebuilt.error?.message ?? '未知原因'}`)
|
|
2666
|
+
} else {
|
|
2667
|
+
const data = rebuilt.data
|
|
2668
|
+
compile = {
|
|
2669
|
+
cardsCreated: data.cardsCreated,
|
|
2670
|
+
cardsUpdated: data.cardsUpdated,
|
|
2671
|
+
cardsReused: data.cardsReused,
|
|
2672
|
+
notesReused: data.notesReused,
|
|
2673
|
+
notesRecompiled: data.notesRecompiled,
|
|
2674
|
+
sourcesExtracted: data.sourcesExtracted,
|
|
2675
|
+
sourcesSkipped: data.sourcesSkipped,
|
|
2676
|
+
cardsRemoved: data.cardsRemoved,
|
|
2677
|
+
embedding: data.embedding,
|
|
2678
|
+
version: data.version,
|
|
2679
|
+
}
|
|
2680
|
+
warnings.push(...(rebuilt.warnings ?? []))
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2684
|
+
return success({ written, failed, attempted: items.length, compile }, warnings)
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
/**
|
|
2688
|
+
* 写入一份 SourceRaw。
|
|
2689
|
+
*
|
|
2690
|
+
* `provenance` 是 OKF v0.2 那一组(`description` / `resource` / `generated` /
|
|
2691
|
+
* `sources[]` / `tags`),它们**写成真正的顶层键**而不是塞进 `metadata` 那个 JSON
|
|
2692
|
+
* 字符串里(0048):元数据要能被人读、被 `grep` 到、被外部工具按公开规范读走。
|
|
2693
|
+
* `metadata` 只留类型专属的扩展键。
|
|
2694
|
+
*/
|
|
2695
|
+
async addSource(input: {
|
|
2696
|
+
sourceType: SourceType
|
|
2697
|
+
title?: string
|
|
2698
|
+
content: string
|
|
2699
|
+
externalId?: string
|
|
2700
|
+
metadata?: Record<string, unknown>
|
|
2701
|
+
sourceId?: string
|
|
2702
|
+
lineage?: LineageContext
|
|
2703
|
+
provenance?: SourceProvenance
|
|
2704
|
+
/**
|
|
2705
|
+
* 允许与已有的来源同标题(0051)。默认 `false`:重名是错误,拒收并要求改标题。
|
|
2706
|
+
* 传 `true` 时落成 `<标题>-<内容哈希前 6 位>.md`,并回一条 warning。
|
|
2707
|
+
*/
|
|
2708
|
+
allowDuplicateTitle?: boolean
|
|
2709
|
+
}): Promise<Envelope<{ sourceId: string; path: string; lineage?: LineageContext }>> {
|
|
1783
2710
|
if (!this.config.allowSourceTruth) return failure({ sourceId: '', path: '' }, 'SOURCE_TRUTH_DISABLED', '当前配置禁止写入 SourceTruth', false)
|
|
1784
2711
|
// 分类闸门走注册表(0013),不再是硬编码枚举:自定义分类建完即可用。
|
|
1785
2712
|
const gate = this.sourceTypeGate(input.sourceType)
|
|
@@ -1800,7 +2727,39 @@ export class KnowledgeRuntime {
|
|
|
1800
2727
|
await this.ensureLayout()
|
|
1801
2728
|
const sourceId = input.sourceId?.trim() || `${input.sourceType}:${hash(input.externalId ?? input.content)}`
|
|
1802
2729
|
if (!SAFE_ID.test(sourceId)) return failure({ sourceId: '', path: '' }, 'SOURCE_ID_INVALID', 'source_id 含有非法字符', false)
|
|
1803
|
-
const
|
|
2730
|
+
const title = input.title?.trim() || sourceId
|
|
2731
|
+
/**
|
|
2732
|
+
* 重名(0051):**默认拒收**。
|
|
2733
|
+
*
|
|
2734
|
+
* 文件名就是标题,所以「两份来源同一个标题」不再是看不见的事 —— 它们会争同一个
|
|
2735
|
+
* 路径。判据落在**目标路径**上而不是标题字符串上:标题里被换掉的非法字符
|
|
2736
|
+
* (`A/B` 与 `A-B`)也会撞路径,按字符串比会漏。
|
|
2737
|
+
*
|
|
2738
|
+
* 三条出路,覆盖三种真实情形:
|
|
2739
|
+
* 1. 同一个 `sourceId`(同一份东西重复导入)→ 目标是它自己现在的位置,不冲突;
|
|
2740
|
+
* 2. 另一个 `sourceId` 占着这个路径 → **拒收**,报错里带上「被谁占着」,让人改标题;
|
|
2741
|
+
* 3. 调用方明确要留两份同标题材料 → 传 `allowDuplicateTitle`,落成
|
|
2742
|
+
* `<标题>-<内容哈希前 6 位>.md`,并回一条 warning。
|
|
2743
|
+
*
|
|
2744
|
+
* 为什么不静默加后缀:那样人永远不知道库里有两份「复盘」—— 而「有两份」这件事本身
|
|
2745
|
+
* 就是要人处理的信号。也不静默覆盖:那等于直接丢一份知识。
|
|
2746
|
+
*/
|
|
2747
|
+
let path = this.sourcePath(sourceId, input.sourceType, title)
|
|
2748
|
+
let duplicateNote: string | undefined
|
|
2749
|
+
const owner = await this.sourceIdAtPath(path)
|
|
2750
|
+
if (owner !== undefined && owner !== sourceId) {
|
|
2751
|
+
if (input.allowDuplicateTitle !== true) {
|
|
2752
|
+
return failure(
|
|
2753
|
+
{ sourceId: '', path },
|
|
2754
|
+
'SOURCE_TITLE_DUPLICATED',
|
|
2755
|
+
`标题「${title}」已被来源 ${owner} 占用(它写在这个路径上),改一个标题再来`,
|
|
2756
|
+
true,
|
|
2757
|
+
'换一个有区分度的标题(例如带上日期或版本),或显式传 allow_duplicate_title: true 让两份并存。',
|
|
2758
|
+
)
|
|
2759
|
+
}
|
|
2760
|
+
path = this.safePath(SOURCE_TRUTH, input.sourceType, `${sourceFileBase(title, sourceId)}-${hash(sourceId).slice(0, 6)}.md`)
|
|
2761
|
+
duplicateNote = `标题「${title}」与来源 ${owner} 重复,已按 allow_duplicate_title 落成 ${basename(path)};建议改成有区分度的标题。`
|
|
2762
|
+
}
|
|
1804
2763
|
/**
|
|
1805
2764
|
* 这一份文档**可能已经以另一个分类存在**。0030 起分类是目录名,所以
|
|
1806
2765
|
* 「改一个来源的分类」不再是原地覆盖,而是换目录 —— 必须先解析它现在落在哪,
|
|
@@ -1825,17 +2784,46 @@ export class KnowledgeRuntime {
|
|
|
1825
2784
|
await this.snapshotSource(sourceId, previousRaw, previous.content)
|
|
1826
2785
|
}
|
|
1827
2786
|
}
|
|
1828
|
-
const record: SourceRaw = { sourceId, sourceType: input.sourceType, title
|
|
1829
|
-
|
|
2787
|
+
const record: SourceRaw = { sourceId, sourceType: input.sourceType, title, content: input.content, ...(input.externalId ? { externalId: input.externalId } : {}), metadata: input.metadata ?? {}, ...(input.lineage ? { lineage: input.lineage } : {}), createdAt, updatedAt: new Date().toISOString() }
|
|
2788
|
+
const frontMatter = sourceFrontMatter(record, input.provenance)
|
|
2789
|
+
await writeTextAtomic(path, markdown(frontMatter, record.content))
|
|
2790
|
+
/**
|
|
2791
|
+
* 分类声明的必填元数据缺了就**如实报出来**,不拒收(0048)。
|
|
2792
|
+
*
|
|
2793
|
+
* 与「缺必填小节的卡片照常入库」同一条纪律:元数据缺失是**信息不全**,不是
|
|
2794
|
+
* 「这份知识不存在」。拒收的代价是知识直接丢了,而报出来的代价只是界面上多一句
|
|
2795
|
+
* 提示 —— 后者可以补,前者补不回来。
|
|
2796
|
+
*/
|
|
2797
|
+
const declared = this.sourceTypes().find(def => def.id === record.sourceType)?.metadataFields
|
|
2798
|
+
const missing = missingMetadataFields(declared, frontMatter)
|
|
1830
2799
|
// 换了分类的那一次:新文件已经落好,旧目录里那份才删 —— 顺序反过来的话,写入
|
|
1831
2800
|
// 失败就同时丢了旧的那一份。
|
|
1832
2801
|
if (previousPath !== undefined && previousPath !== path) {
|
|
1833
2802
|
try { await unlink(previousPath) } catch { /* already moved or removed */ }
|
|
1834
2803
|
}
|
|
1835
|
-
|
|
2804
|
+
/**
|
|
2805
|
+
* 索引与归属表都要跟着**新路径**走(0051)。
|
|
2806
|
+
*
|
|
2807
|
+
* 归属表(`path → sourceId`)是重名判定的依据,漏更新会留下一个幽灵归属:改了标题
|
|
2808
|
+
* 之后,旧路径上明明没有文件,另一个来源想用那个标题却会被判成「重名」而拒收。
|
|
2809
|
+
*/
|
|
2810
|
+
if (this.sourceIndex) {
|
|
2811
|
+
if (previousPath !== undefined && previousPath !== path) {
|
|
2812
|
+
this.sourceIndex.delete(previousPath)
|
|
2813
|
+
this.sourcePathOwner?.delete(previousPath)
|
|
2814
|
+
}
|
|
2815
|
+
this.sourceIndex.set(slug(sourceId), path)
|
|
2816
|
+
this.sourcePathOwner?.set(path, sourceId)
|
|
2817
|
+
}
|
|
1836
2818
|
// 证据片段取自 SourceRaw 正文,写入后那份缓存必须失效(否则上下文会引用旧原文)。
|
|
1837
2819
|
this.sourceCorpusCache = undefined
|
|
1838
|
-
return success(
|
|
2820
|
+
return success(
|
|
2821
|
+
{ sourceId, path, ...(record.lineage ? { lineage: record.lineage } : {}) },
|
|
2822
|
+
[
|
|
2823
|
+
...(duplicateNote ? [duplicateNote] : []),
|
|
2824
|
+
...missing.map(field => `来源缺少「${field.label}」(${field.key}):分类「${record.sourceType}」声明它是必填的。`),
|
|
2825
|
+
],
|
|
2826
|
+
)
|
|
1839
2827
|
}
|
|
1840
2828
|
|
|
1841
2829
|
/** Directory holding one document's version snapshots. */
|
|
@@ -1861,14 +2849,30 @@ export class KnowledgeRuntime {
|
|
|
1861
2849
|
|
|
1862
2850
|
async loadSources(): Promise<SourceRaw[]> {
|
|
1863
2851
|
const result: SourceRaw[] = []
|
|
2852
|
+
/**
|
|
2853
|
+
* 索引在**同一次遍历**里建起来(0051)。
|
|
2854
|
+
*
|
|
2855
|
+
* 文件名从 `source_id` 改成标题之后,「按 id 找文件」不再能从文件名推出来 ——
|
|
2856
|
+
* 权威身份在 front matter 里。与其再扫一遍磁盘、再读一遍文件,不如在唯一一次
|
|
2857
|
+
* 遍历里同时建两张表:`sourceId → path`(`sourcePathOf` 用)与 `path → sourceId`
|
|
2858
|
+
* (`addSource` 判断目标路径被谁占着)。
|
|
2859
|
+
*/
|
|
2860
|
+
const byId = new Map<string, string>()
|
|
2861
|
+
const byPath = new Map<string, string>()
|
|
1864
2862
|
for (const path of await filesUnder(this.directory(SOURCE_TRUTH))) {
|
|
1865
2863
|
const parsed = parseMarkdown(await readFile(path, 'utf8'))
|
|
1866
2864
|
const sourceId = String(parsed.frontmatter.source_id ?? basename(path, '.md'))
|
|
1867
|
-
|
|
2865
|
+
byId.set(slug(sourceId), path)
|
|
2866
|
+
// 文件名也可能是旧布局的 id(迁移中间态):两个键都登记,谁在都不算丢。
|
|
2867
|
+
byId.set(basename(path, '.md'), path)
|
|
2868
|
+
byPath.set(path, sourceId)
|
|
2869
|
+
const sourceType = (sourceTypeOf(parsed.frontmatter) || 'document') as SourceType
|
|
1868
2870
|
// 0013:未知分类**不再静默丢弃**。分类只是元数据,内容仍然可召回;
|
|
1869
2871
|
// 丢掉文档才是真正的数据损失。未知分类在列表里以「未知分类」呈现。
|
|
1870
2872
|
result.push({ sourceId, sourceType, title: String(parsed.frontmatter.title ?? sourceId), content: parsed.content, externalId: typeof parsed.frontmatter.external_id === 'string' && parsed.frontmatter.external_id ? parsed.frontmatter.external_id : undefined, metadata: typeof parsed.frontmatter.metadata === 'object' && parsed.frontmatter.metadata ? parsed.frontmatter.metadata as Record<string, unknown> : {}, ...(parseLineage(parsed.frontmatter.lineage) ? { lineage: parseLineage(parsed.frontmatter.lineage) } : {}), createdAt: String(parsed.frontmatter.created_at ?? ''), updatedAt: String(parsed.frontmatter.updated_at ?? '') })
|
|
1871
2873
|
}
|
|
2874
|
+
this.sourceIndex = byId
|
|
2875
|
+
this.sourcePathOwner = byPath
|
|
1872
2876
|
return result
|
|
1873
2877
|
}
|
|
1874
2878
|
|
|
@@ -1881,17 +2885,26 @@ export class KnowledgeRuntime {
|
|
|
1881
2885
|
* (`addSource` 的 `sourceTypeGate`),所以它一定匹配 `SOURCE_TYPE_ID_PATTERN`,
|
|
1882
2886
|
* 不会引入路径穿越。
|
|
1883
2887
|
*/
|
|
1884
|
-
private sourcePath(sourceId: string, sourceType: string): string {
|
|
1885
|
-
return this.safePath(SOURCE_TRUTH, sourceType, `${
|
|
2888
|
+
private sourcePath(sourceId: string, sourceType: string, title: string): string {
|
|
2889
|
+
return this.safePath(SOURCE_TRUTH, sourceType, `${sourceFileBase(title, sourceId)}.md`)
|
|
1886
2890
|
}
|
|
1887
2891
|
|
|
1888
|
-
/**
|
|
2892
|
+
/**
|
|
2893
|
+
* `docSlug → path` 索引;两种布局都覆盖。
|
|
2894
|
+
*
|
|
2895
|
+
* 0051 起它**由 `loadSources` 建**(那一次遍历本来就要读每个文件的 front matter,
|
|
2896
|
+
* 而文件名不再是身份,只能从 front matter 里拿)。这里只是把「还没建」补上。
|
|
2897
|
+
*/
|
|
1889
2898
|
private async sourceIndexMap(): Promise<Map<string, string>> {
|
|
1890
2899
|
if (this.sourceIndex) return this.sourceIndex
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
2900
|
+
await this.loadSources()
|
|
2901
|
+
return this.sourceIndex ?? new Map()
|
|
2902
|
+
}
|
|
2903
|
+
|
|
2904
|
+
/** 这个路径现在被哪一份来源占着(0051:重名判定的依据)。 */
|
|
2905
|
+
private async sourceIdAtPath(path: string): Promise<string | undefined> {
|
|
2906
|
+
if (!this.sourcePathOwner) await this.loadSources()
|
|
2907
|
+
return this.sourcePathOwner?.get(path)
|
|
1895
2908
|
}
|
|
1896
2909
|
|
|
1897
2910
|
/** 已知文档 id 的落盘位置;不存在时 `undefined`(调用方各自决定怎么报错)。 */
|
|
@@ -1902,13 +2915,12 @@ export class KnowledgeRuntime {
|
|
|
1902
2915
|
/**
|
|
1903
2916
|
* Assets a document really has on disk.
|
|
1904
2917
|
*
|
|
1905
|
-
* Preferred source of truth is `metadata.assets` (
|
|
1906
|
-
*
|
|
1907
|
-
*
|
|
1908
|
-
*
|
|
1909
|
-
*
|
|
1910
|
-
*
|
|
1911
|
-
* returned, so `assetCount` is never an over-count.
|
|
2918
|
+
* Preferred source of truth is `metadata.assets` (paths written by
|
|
2919
|
+
* `convert/ingest.ts`). Documents written by hand fall back to parsing local
|
|
2920
|
+
* markdown image links. Both go through {@link assetRefCandidates}, so a
|
|
2921
|
+
* document's asset list and a card's `imagePaths` answer the same question the
|
|
2922
|
+
* same way. Only existing files are returned, so `assetCount` is never an
|
|
2923
|
+
* over-count.
|
|
1912
2924
|
*/
|
|
1913
2925
|
private async assetsOf(path: string, content: string, metadata: Record<string, unknown>): Promise<KnowledgeAssetRef[]> {
|
|
1914
2926
|
const refs = new Set<string>()
|
|
@@ -1919,7 +2931,7 @@ export class KnowledgeRuntime {
|
|
|
1919
2931
|
const assets: KnowledgeAssetRef[] = []
|
|
1920
2932
|
const seen = new Set<string>()
|
|
1921
2933
|
for (const ref of refs) {
|
|
1922
|
-
const candidates =
|
|
2934
|
+
const candidates = assetRefCandidates(ref, dirname(path), root)
|
|
1923
2935
|
for (const absolute of candidates) {
|
|
1924
2936
|
if (!insideRoot(absolute)) continue
|
|
1925
2937
|
let details
|
|
@@ -1928,14 +2940,49 @@ export class KnowledgeRuntime {
|
|
|
1928
2940
|
const rel = relative(root, absolute).split(sep).join('/')
|
|
1929
2941
|
if (seen.has(rel)) break
|
|
1930
2942
|
seen.add(rel)
|
|
1931
|
-
|
|
1932
|
-
assets.push({ rel, bytes: details.size, url: `/knowledge/assets/${encodeURIComponent(this.baseId)}/${encoded}` })
|
|
2943
|
+
assets.push({ rel, bytes: details.size, url: this.assetUrlOf(rel) })
|
|
1933
2944
|
break
|
|
1934
2945
|
}
|
|
1935
2946
|
}
|
|
1936
2947
|
return assets.sort((left, right) => left.rel.localeCompare(right.rel))
|
|
1937
2948
|
}
|
|
1938
2949
|
|
|
2950
|
+
/**
|
|
2951
|
+
* 图片素材的浏览器地址。面板与整页都用它加载图,所以只能有一个拼法。
|
|
2952
|
+
*/
|
|
2953
|
+
private assetUrlOf(rel: string): string {
|
|
2954
|
+
const encoded = rel.split('/').map(segment => encodeURIComponent(segment)).join('/')
|
|
2955
|
+
return `/knowledge/assets/${encodeURIComponent(this.baseId)}/${encoded}`
|
|
2956
|
+
}
|
|
2957
|
+
|
|
2958
|
+
/**
|
|
2959
|
+
* 一张 wiki 卡片的图片素材(0047)。
|
|
2960
|
+
*
|
|
2961
|
+
* 卡片详情原先只回 `imagePaths` —— **磁盘绝对路径**。客户端拿它加载不了图(浏览器
|
|
2962
|
+
* 只认 URL),所以预览抽屉里卡片的图一直不显示;而 SourceRaw 那边早就有 `assets`
|
|
2963
|
+
* (`rel` + `url` + `bytes`)。这里把同一种东西补齐,两个预览走同一条路。
|
|
2964
|
+
*
|
|
2965
|
+
* 只认知识库根目录以内的文件:卡片正文里可能留着别的绝对路径(人手工写的),
|
|
2966
|
+
* 那种不该变成一个可下载的 URL。
|
|
2967
|
+
*/
|
|
2968
|
+
async assetsOfCard(card: KnowledgeCard): Promise<KnowledgeAssetRef[]> {
|
|
2969
|
+
const root = this.rootDir
|
|
2970
|
+
const insideRoot = (value: string): boolean => value === root || value.startsWith(root.endsWith(sep) ? root : `${root}${sep}`)
|
|
2971
|
+
const assets: KnowledgeAssetRef[] = []
|
|
2972
|
+
const seen = new Set<string>()
|
|
2973
|
+
for (const absolute of card.imagePaths ?? []) {
|
|
2974
|
+
if (!isAbsolute(absolute) || !insideRoot(absolute)) continue
|
|
2975
|
+
let details
|
|
2976
|
+
try { details = await stat(absolute) } catch { continue }
|
|
2977
|
+
if (!details.isFile()) continue
|
|
2978
|
+
const rel = relative(root, absolute).split(sep).join('/')
|
|
2979
|
+
if (seen.has(rel)) continue
|
|
2980
|
+
seen.add(rel)
|
|
2981
|
+
assets.push({ rel, bytes: details.size, url: this.assetUrlOf(rel) })
|
|
2982
|
+
}
|
|
2983
|
+
return assets.sort((left, right) => left.rel.localeCompare(right.rel))
|
|
2984
|
+
}
|
|
2985
|
+
|
|
1939
2986
|
/**
|
|
1940
2987
|
* Parse one on-disk SourceRaw file into the detail projection.
|
|
1941
2988
|
*
|
|
@@ -1950,7 +2997,7 @@ export class KnowledgeRuntime {
|
|
|
1950
2997
|
const contentChars = parsed.content.length
|
|
1951
2998
|
const content = previewChars !== undefined && previewChars < contentChars ? parsed.content.slice(0, previewChars) : parsed.content
|
|
1952
2999
|
const sourceId = String(parsed.frontmatter.source_id ?? basename(path, '.md'))
|
|
1953
|
-
const sourceType =
|
|
3000
|
+
const sourceType = sourceTypeOf(parsed.frontmatter) || 'document'
|
|
1954
3001
|
// 未知分类照样投影出来(0013);前端按 sourceTypeKnown 渲染「未知分类」徽标。
|
|
1955
3002
|
const sourceTypeKnown = this.sourceTypes().some(def => def.id === sourceType)
|
|
1956
3003
|
const metadata = plainRecord(parsed.frontmatter.metadata)
|
|
@@ -2095,7 +3142,7 @@ export class KnowledgeRuntime {
|
|
|
2095
3142
|
if (!existsSync(snapshotPath)) return failure(head, 'VERSION_NOT_FOUND', `版本不存在: ${target}`, true)
|
|
2096
3143
|
const parsed = parseMarkdown(await readFile(snapshotPath, 'utf8'))
|
|
2097
3144
|
const restored = await this.addSource({
|
|
2098
|
-
sourceType: (
|
|
3145
|
+
sourceType: (sourceTypeOf(parsed.frontmatter) || 'document') as SourceType,
|
|
2099
3146
|
title: typeof parsed.frontmatter.title === 'string' ? parsed.frontmatter.title : undefined,
|
|
2100
3147
|
content: parsed.content,
|
|
2101
3148
|
externalId: typeof parsed.frontmatter.external_id === 'string' && parsed.frontmatter.external_id ? parsed.frontmatter.external_id : undefined,
|
|
@@ -2104,7 +3151,13 @@ export class KnowledgeRuntime {
|
|
|
2104
3151
|
lineage: parseLineage(parsed.frontmatter.lineage),
|
|
2105
3152
|
})
|
|
2106
3153
|
if (!restored.ok) return failure(head, restored.error?.code ?? 'REVERT_FAILED', restored.error?.message ?? '回滚写入失败', restored.error?.retryable ?? false, restored.error?.suggestion)
|
|
2107
|
-
|
|
3154
|
+
/**
|
|
3155
|
+
* 读的是**回滚写完之后**的路径,不是开头那个。
|
|
3156
|
+
*
|
|
3157
|
+
* 0051 起文件名跟着标题走:回滚到一版标题不同的历史,文件会被改名 —— 开头取到的
|
|
3158
|
+
* 路径当场过期,再按它读就是「回滚后文档不可读」,而文件其实好好的。
|
|
3159
|
+
*/
|
|
3160
|
+
const document = await this.readDocumentFile(restored.data.path, SOURCE_PREVIEW_CHARS)
|
|
2108
3161
|
if (!document) return failure(head, 'DOCUMENT_NOT_FOUND', `回滚后文档不可读: ${docId}`, true)
|
|
2109
3162
|
await this.audit('source_truth.revert', { docId, versionId: target })
|
|
2110
3163
|
return success(document)
|
|
@@ -2126,7 +3179,24 @@ export class KnowledgeRuntime {
|
|
|
2126
3179
|
* 负责人。它属于 0021 §6.4 的卡片信封(智能归因的五类卡片里 `check.owner` 是必填 ——
|
|
2127
3180
|
* 「这一步该找谁」和成本、判定阈值一样是执行信息);其余 entity_type 不使用它。
|
|
2128
3181
|
*/
|
|
2129
|
-
owner?: string; metadata?: Record<string, unknown>; lineage?: LineageContext; provenance?: LineageRef[]
|
|
3182
|
+
owner?: string; metadata?: Record<string, unknown>; lineage?: LineageContext; provenance?: LineageRef[]; /**
|
|
3183
|
+
* 额外的校验问题(0049)。编译收尾段算出「正文里的 wiki 链接指向不存在的卡片」,
|
|
3184
|
+
* 它是 `reference` 那一层的问题 —— 与 format / semantic 一起写进 `metadata.schema_issues`,
|
|
3185
|
+
* 卡片抽屉上就会出现一枚「引用」徽标。收尾段不能自己拼 metadata:那样会把写入时
|
|
3186
|
+
* 重算的 format / semantic 覆盖掉。
|
|
3187
|
+
*/
|
|
3188
|
+
extraIssues?: CardSchemaIssue[]; /**
|
|
3189
|
+
* 人工备注(0044)。**不传 = 保留磁盘上已有的那些**,不是清空 —— 编译路径不传它,
|
|
3190
|
+
* 所以「重新编译一次不会抹掉备注」是结构上的保证,而不是靠每个调用方记得带上。
|
|
3191
|
+
*/
|
|
3192
|
+
annotations?: CardAnnotation[]; /** review 结论与评分(0044)。同样:不传 = 保留已有的。 */ review?: CardReview; /**
|
|
3193
|
+
* 谁在写这张卡片。
|
|
3194
|
+
*
|
|
3195
|
+
* `'manual'` 是**唯一会被正文不可变规则拦下**的来源:已有卡片的正文字节不同 →
|
|
3196
|
+
* `WIKI_CARD_IMMUTABLE`。编译(`'compile'`)追加正文、review 与备注写信封字段,
|
|
3197
|
+
* 两者都不受这条规则约束 —— 把它们也拦掉,卡片就再也不会长了。
|
|
3198
|
+
*/
|
|
3199
|
+
origin?: 'compile' | 'review' | 'manual' | 'internal' }): Promise<Envelope<{ entityId: string; path: string; lineage?: LineageContext }>> {
|
|
2130
3200
|
if (!SAFE_ID.test(input.entityId)) return failure({ entityId: input.entityId, path: '' }, 'ENTITY_ID_INVALID', 'entity_id 必须是稳定且安全的标识符', false)
|
|
2131
3201
|
if (!input.title?.trim() || !input.content?.trim()) return failure({ entityId: input.entityId, path: '' }, 'CARD_CONTENT_REQUIRED', 'title 和 content 不能为空', true)
|
|
2132
3202
|
if (!Array.isArray(input.sourceRefs) || input.sourceRefs.length === 0) return failure({ entityId: input.entityId, path: '' }, 'SOURCE_REFS_REQUIRED', '知识卡片必须引用 SourceTruth', false)
|
|
@@ -2135,7 +3205,18 @@ export class KnowledgeRuntime {
|
|
|
2135
3205
|
if (missingRefs.length > 0) return failure({ entityId: input.entityId, path: '' }, 'SOURCE_REF_NOT_FOUND', `source_refs 引用不存在: ${missingRefs.join(', ')}`, false, '先调用 add_source 写入对应的 SourceTruth')
|
|
2136
3206
|
const status = input.status ?? 'unreviewed'
|
|
2137
3207
|
if (!STATUS.includes(status)) return failure({ entityId: input.entityId, path: '' }, 'STATUS_INVALID', 'status 不合法', false)
|
|
2138
|
-
|
|
3208
|
+
/**
|
|
3209
|
+
* 0044:缺信息的卡片**照常入库**。
|
|
3210
|
+
*
|
|
3211
|
+
* 0040 时期这里有两道拒绝:类型不在注册表里直接拒、三层校验不通过时 `strict` 模式拒。
|
|
3212
|
+
* 两条都撤掉了 —— 编译的价值在于「记下来,后面有涉及到的文档再慢慢补」,而拒绝
|
|
3213
|
+
* 恰好把这个机会丢掉了。缺什么由 `schema_issues` 如实标出,人在 review 时补。
|
|
3214
|
+
*
|
|
3215
|
+
* 唯一保留的拦截是**未知类型**:没有类型定义就没有小节骨架,连「缺什么」都说不出来。
|
|
3216
|
+
* 编译侧在调用这里之前已经把未登记的类型**退回兜底类型**(`compileSource`),
|
|
3217
|
+
* 所以这条路只在别的调用方拿着一个没登记的类型直接写盘时才会走到 —— 那时拒绝是对的,
|
|
3218
|
+
* 而提示会给出可用类型的清单。
|
|
3219
|
+
*/
|
|
2139
3220
|
const gate = this.cardSchemaGate({
|
|
2140
3221
|
entityType: input.entityType, title: input.title.trim(), content: input.content,
|
|
2141
3222
|
sourceRefs: input.sourceRefs, metadata: input.metadata ?? {}, ...(input.owner ? { owner: input.owner } : {}), ...(input.summary ? { summary: input.summary } : {}),
|
|
@@ -2143,15 +3224,6 @@ export class KnowledgeRuntime {
|
|
|
2143
3224
|
if ('error' in gate) return failure({ entityId: input.entityId, path: '' }, gate.error.code, gate.error.message, gate.error.retryable, gate.error.suggestion)
|
|
2144
3225
|
const { issues } = gate
|
|
2145
3226
|
const warnings = issues.map(issue => `${issue.level}: ${issue.message}`)
|
|
2146
|
-
if (issues.length > 0 && this.wikiSchema().mode === 'strict') {
|
|
2147
|
-
return failure(
|
|
2148
|
-
{ entityId: input.entityId, path: '' },
|
|
2149
|
-
'CARD_SCHEMA_INVALID',
|
|
2150
|
-
`卡片不符合「${gate.type.label}」的 schema:${issues.map(issue => issue.message).join(';')}`,
|
|
2151
|
-
false,
|
|
2152
|
-
'补齐必填小节 / 字段后重试;或在「知识库设置 → Wiki 类型」里把生效方式改为 report(只标记不拦截)。',
|
|
2153
|
-
)
|
|
2154
|
-
}
|
|
2155
3227
|
await this.ensureLayout()
|
|
2156
3228
|
const path = this.safePath(WIKI, input.entityType, `${slug(input.entityId)}.md`)
|
|
2157
3229
|
await mkdir(dirname(path), { recursive: true })
|
|
@@ -2161,18 +3233,72 @@ export class KnowledgeRuntime {
|
|
|
2161
3233
|
// 「审核一次 = 多一个版本」,版本列表会被状态流转灌满。
|
|
2162
3234
|
let createdAt = updatedAt
|
|
2163
3235
|
let previousRaw: string | undefined
|
|
3236
|
+
let previousAnnotations: CardAnnotation[] | undefined
|
|
3237
|
+
let previousReview: CardReview | undefined
|
|
2164
3238
|
try { previousRaw = await readFile(path, 'utf8') } catch { /* 首次写入这张卡片 */ }
|
|
2165
3239
|
if (previousRaw !== undefined) {
|
|
2166
3240
|
const previous = parseMarkdown(previousRaw)
|
|
2167
3241
|
createdAt = String(previous.frontmatter.created_at ?? createdAt)
|
|
3242
|
+
previousAnnotations = parseAnnotations(previous.frontmatter.annotations)
|
|
3243
|
+
previousReview = parseReview(previous.frontmatter.review)
|
|
2168
3244
|
if (previous.content.trim() !== input.content.trim()) {
|
|
3245
|
+
if (input.origin === 'manual') {
|
|
3246
|
+
return failure(
|
|
3247
|
+
{ entityId: input.entityId, path },
|
|
3248
|
+
'WIKI_CARD_IMMUTABLE',
|
|
3249
|
+
`wiki 正文不可手工修改: ${input.entityId}`,
|
|
3250
|
+
false,
|
|
3251
|
+
'正文由编译产出(新文档会追加进来)。要指出问题请加备注:annotate_wiki_card 或面板上的「备注 / 异常点」。',
|
|
3252
|
+
)
|
|
3253
|
+
}
|
|
2169
3254
|
await this.snapshotCard(input.entityType, input.entityId, previousRaw, previous.content)
|
|
2170
3255
|
}
|
|
2171
3256
|
}
|
|
2172
3257
|
const provenance = [...(input.provenance ?? []), ...(input.lineage ? [{ ...input.lineage, role: 'created' as const, refType: 'knowledge_card', refId: input.entityId }] : [])]
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
3258
|
+
/**
|
|
3259
|
+
* report 模式:把问题留在卡片上,界面据此显示"待补",而不是静默通过。
|
|
3260
|
+
*
|
|
3261
|
+
* `schema_issues` **一律由这里重算**(先摘掉入参里可能带着的旧值)。入参 metadata
|
|
3262
|
+
* 常常是「从磁盘上读回来的那一份」,里面就有上一次的 `schema_issues` —— 只做覆盖
|
|
3263
|
+
* 不摘旧值,一张补好了的卡片会一直挂着旧问题;而问题清空时那个键根本没被覆盖,
|
|
3264
|
+
* 于是「补好了」这件事在界面上永远不生效。
|
|
3265
|
+
*/
|
|
3266
|
+
const { schema_issues: _staleIssues, ...inputMetadata } = input.metadata ?? {}
|
|
3267
|
+
void _staleIssues
|
|
3268
|
+
const allIssues = [...issues, ...(input.extraIssues ?? [])]
|
|
3269
|
+
const metadata = { ...inputMetadata, ...(allIssues.length > 0 ? { schema_issues: allIssues } : {}) }
|
|
3270
|
+
// 备注与 review **默认继承磁盘上已有的**:不传不等于清空。见入参注释。
|
|
3271
|
+
const annotations = input.annotations ?? previousAnnotations ?? []
|
|
3272
|
+
const review = input.review ?? previousReview
|
|
3273
|
+
await writeTextAtomic(path, markdown({
|
|
3274
|
+
entity_id: input.entityId, entity_type: input.entityType, title: input.title.trim(), status, summary: input.summary ?? '',
|
|
3275
|
+
...(input.owner ? { owner: input.owner } : {}), source_refs: input.sourceRefs, created_at: createdAt, updated_at: updatedAt, metadata,
|
|
3276
|
+
...(annotations.length > 0 ? { annotations } : {}),
|
|
3277
|
+
...(review ? { review } : {}),
|
|
3278
|
+
...(provenance.length > 0 ? { provenance } : {}),
|
|
3279
|
+
}, input.content))
|
|
3280
|
+
/**
|
|
3281
|
+
* 同步内存里的卡片清单。
|
|
3282
|
+
*
|
|
3283
|
+
* `this.cards` 是召回组装 `context[].wiki` 时读的那一份。写盘不同步它,会出现
|
|
3284
|
+
* 「刚加的备注,在召回结果里看不到」—— 而人正是在召回结果里核对备注的。
|
|
3285
|
+
*
|
|
3286
|
+
* 重新 `loadCards()` 一遍是 O(整个 wiki),而一次编译会连着写几十张卡;所以这里
|
|
3287
|
+
* 就地替换那一条。缓存为空时什么都不做:那表示还没人读过卡片清单,下次读会拿到全量。
|
|
3288
|
+
*/
|
|
3289
|
+
if (this.cards.length > 0) {
|
|
3290
|
+
const next: KnowledgeCard = {
|
|
3291
|
+
entityId: input.entityId, entityType: input.entityType, title: input.title.trim(), status,
|
|
3292
|
+
...(input.summary ? { summary: input.summary } : {}),
|
|
3293
|
+
content: input.content, sourceRefs: input.sourceRefs, metadata, updatedAt,
|
|
3294
|
+
...(annotations.length > 0 ? { annotations } : {}),
|
|
3295
|
+
...(review ? { review } : {}),
|
|
3296
|
+
...(provenance.length > 0 ? { provenance } : {}),
|
|
3297
|
+
}
|
|
3298
|
+
const index = this.cards.findIndex(card => card.entityId === input.entityId && card.entityType === input.entityType)
|
|
3299
|
+
if (index >= 0) this.cards[index] = next
|
|
3300
|
+
else this.cards.push(next)
|
|
3301
|
+
}
|
|
2176
3302
|
return success({ entityId: input.entityId, path, ...(input.lineage ? { lineage: input.lineage } : {}) }, warnings)
|
|
2177
3303
|
}
|
|
2178
3304
|
|
|
@@ -2285,7 +3411,21 @@ export class KnowledgeRuntime {
|
|
|
2285
3411
|
// 0024:未知类型**不再静默丢弃**。类型只是元数据,丢掉卡片才是真正的数据损失;
|
|
2286
3412
|
// 未知类型在界面上显示成「未登记类型」,由人决定是补登记还是改卡片。
|
|
2287
3413
|
const refs = Array.isArray(parsed.frontmatter.source_refs) ? parsed.frontmatter.source_refs.map(String) : []
|
|
2288
|
-
|
|
3414
|
+
const annotations = parseAnnotations(parsed.frontmatter.annotations)
|
|
3415
|
+
const review = parseReview(parsed.frontmatter.review)
|
|
3416
|
+
result.push({
|
|
3417
|
+
entityId, entityType, title: String(parsed.frontmatter.title ?? entityId),
|
|
3418
|
+
status: STATUS.includes(String(parsed.frontmatter.status) as KnowledgeStatus) ? String(parsed.frontmatter.status) as KnowledgeStatus : 'draft',
|
|
3419
|
+
summary: typeof parsed.frontmatter.summary === 'string' ? parsed.frontmatter.summary : undefined,
|
|
3420
|
+
content: parsed.content, sourceRefs: refs,
|
|
3421
|
+
...(typeof parsed.frontmatter.owner === 'string' && parsed.frontmatter.owner ? { owner: parsed.frontmatter.owner } : {}),
|
|
3422
|
+
...(parseProvenance(parsed.frontmatter.provenance) ? { provenance: parseProvenance(parsed.frontmatter.provenance) } : {}),
|
|
3423
|
+
metadata: typeof parsed.frontmatter.metadata === 'object' && parsed.frontmatter.metadata ? parsed.frontmatter.metadata as Record<string, unknown> : {},
|
|
3424
|
+
updatedAt: String(parsed.frontmatter.updated_at ?? ''),
|
|
3425
|
+
imagePaths: markdownImagePaths(parsed.content, dirname(path), this.rootDir),
|
|
3426
|
+
...(annotations.length > 0 ? { annotations } : {}),
|
|
3427
|
+
...(review ? { review } : {}),
|
|
3428
|
+
})
|
|
2289
3429
|
}
|
|
2290
3430
|
return result
|
|
2291
3431
|
}
|
|
@@ -2322,9 +3462,34 @@ export class KnowledgeRuntime {
|
|
|
2322
3462
|
if (card.status === 'problem') return []
|
|
2323
3463
|
const notes: WikiNote[] = []
|
|
2324
3464
|
const sourceTypesFor = card.sourceRefs.flatMap(ref => { const sourceType = sourceTypes.get(ref); return sourceType ? [sourceType] : [] })
|
|
2325
|
-
|
|
3465
|
+
/**
|
|
3466
|
+
* 卡片文件所在目录:正文里的图片引用是相对它写的(`../../01_source_raw/_assets/…`),
|
|
3467
|
+
* 和 `loadCards` 用的是同一个基准。
|
|
3468
|
+
*/
|
|
3469
|
+
const cardDir = join(this.directory(WIKI), card.entityType)
|
|
3470
|
+
// 0044:note 带上父级 wiki 的**有效评分**。它不进正文、不参与向量指纹,所以
|
|
3471
|
+
// review 一下不会让整个语料重新编码,只会改变召回时的加权。
|
|
3472
|
+
const reviewScore = effectiveReviewScore(card)
|
|
3473
|
+
/**
|
|
3474
|
+
* 召回单元里**不带相关块**(0049)。
|
|
3475
|
+
*
|
|
3476
|
+
* 相关块是导航,不是知识:它写的是别的卡片的名字。切进 wikiNotes 之后,一句
|
|
3477
|
+
* 「毛利率怎么算」会同时命中真正讲毛利率的那张卡和所有**提到**它的卡 —— 而后者
|
|
3478
|
+
* 除了一个标题什么都没有。文件里留着它(人 grep 得到、编辑器里点得开),
|
|
3479
|
+
* 召回不吃它。
|
|
3480
|
+
*/
|
|
3481
|
+
const recallBody = stripRelatedSection(card.content)
|
|
3482
|
+
for (const [index, chunk] of splitWikiNotes({ ...card, content: recallBody }).entries()) {
|
|
2326
3483
|
const baseId = `${card.entityType}:${card.entityId}`
|
|
2327
3484
|
const unitId = index === 0 ? baseId : `${baseId}#${index}`
|
|
3485
|
+
/**
|
|
3486
|
+
* 图按**这一节自己**的引用算,不是把整张卡片的图摊给每一节(0046)。
|
|
3487
|
+
*
|
|
3488
|
+
* 多模态编码只吃每张 note 的**第一张**图,所以「这一节的第一张图」必须是这一节
|
|
3489
|
+
* 自己的图 —— 否则「景气投资」那一节编码时会拿到「低估值」那节的图,而卡片
|
|
3490
|
+
* 正文里图片的落点本来就是按节算好的。
|
|
3491
|
+
*/
|
|
3492
|
+
const chunkImages = markdownImagePaths(chunk.content, cardDir, this.rootDir)
|
|
2328
3493
|
notes.push({
|
|
2329
3494
|
unitId,
|
|
2330
3495
|
noteId: unitId,
|
|
@@ -2338,7 +3503,8 @@ export class KnowledgeRuntime {
|
|
|
2338
3503
|
sourceTypes: sourceTypesFor,
|
|
2339
3504
|
...(card.provenance ? { provenance: card.provenance } : {}),
|
|
2340
3505
|
status: card.status,
|
|
2341
|
-
...(
|
|
3506
|
+
...(chunkImages.length > 0 ? { imagePaths: chunkImages } : {}),
|
|
3507
|
+
reviewScore,
|
|
2342
3508
|
})
|
|
2343
3509
|
}
|
|
2344
3510
|
return notes
|
|
@@ -2374,7 +3540,7 @@ export class KnowledgeRuntime {
|
|
|
2374
3540
|
* sigmoid 之后的展示值,两者单调等价,排序不受影响。刻度换算集中在
|
|
2375
3541
|
* `applyRerank`(纯函数,有测试),这里不再自己乘除。
|
|
2376
3542
|
*/
|
|
2377
|
-
private async rerankNotes(query: string, hits: RecallHit[], threshold: number): Promise<{ hits: RecallHit[]; result: RerankResult; warning?: string }> {
|
|
3543
|
+
private async rerankNotes(query: string, hits: RecallHit[], threshold: number): Promise<{ hits: RecallHit[]; result: RerankResult; scored?: Array<{ unitId: string; logit: number; rerankScore: number; kept: boolean }>; warning?: string }> {
|
|
2378
3544
|
const provider = this.config.rerankProvider
|
|
2379
3545
|
if (provider === 'none') return { hits, result: { status: 'skipped', provider, threshold } }
|
|
2380
3546
|
if (hits.length < 2) return { hits, result: { status: 'skipped', provider, model: this.rerankModelPath, scored: hits.length, threshold } }
|
|
@@ -2393,6 +3559,7 @@ export class KnowledgeRuntime {
|
|
|
2393
3559
|
const vocabSize = Number(response?.vocabSize)
|
|
2394
3560
|
return {
|
|
2395
3561
|
hits: outcome.kept,
|
|
3562
|
+
scored: outcome.scored,
|
|
2396
3563
|
result: {
|
|
2397
3564
|
status: 'ready', provider, model: this.rerankModelPath,
|
|
2398
3565
|
scored: outcome.kept.length, threshold, dropped: outcome.dropped,
|
|
@@ -2415,13 +3582,28 @@ export class KnowledgeRuntime {
|
|
|
2415
3582
|
* Fingerprint of everything a wikiNote is compiled *from*.
|
|
2416
3583
|
*
|
|
2417
3584
|
* Deliberately excludes the fields that are cheap to refresh on a reused note
|
|
2418
|
-
* (`provenance`, `imagePaths`, `metadata`) and includes the ones
|
|
2419
|
-
* what the note says (`title`, `status`, `content`, `sourceRefs
|
|
2420
|
-
* fingerprint is unchanged produces byte-identical
|
|
2421
|
-
* not a shortcut — it is the same answer without the
|
|
3585
|
+
* (`provenance`, `imagePaths`, `metadata`, `annotations`) and includes the ones
|
|
3586
|
+
* that change what the note says (`title`, `status`, `content`, `sourceRefs`,
|
|
3587
|
+
* `review`). A card whose fingerprint is unchanged produces byte-identical
|
|
3588
|
+
* notes, so reusing them is not a shortcut — it is the same answer without the
|
|
3589
|
+
* churn.
|
|
3590
|
+
*
|
|
3591
|
+
* 0044 把**有效评分**算进来,因为 note 上确实带着它。少了这一条会踩一个很隐蔽的坑:
|
|
3592
|
+
* review 不改 `title` / `content` / `sourceRefs`,于是增量编译判定「语料没变」直接
|
|
3593
|
+
* 跳过,`wiki_notes.json` 停在旧评分上 —— 表现就是「review 了,但召回排序一动不动」。
|
|
3594
|
+
*
|
|
3595
|
+
* 代价可控:评分变了会重编这条 note,但**不会重新编码**。向量指纹(`vectorInputs`)
|
|
3596
|
+
* 只哈希 `unitId + text + sourceRefs + images`,而 review 一个字都不改这些 ——
|
|
3597
|
+
* 所以重编拿到的还是同一条文本、同一个向量。
|
|
3598
|
+
*
|
|
3599
|
+
* `annotations` 刻意不进指纹:备注只出现在召回结果附带的 wiki 卡片上,不进 note。
|
|
3600
|
+
* 把它算进来会让「加一条备注」触发一次没必要的版本升级。
|
|
2422
3601
|
*/
|
|
2423
3602
|
private cardHashOf(card: KnowledgeCard): string {
|
|
2424
|
-
return contentHashOf(JSON.stringify({
|
|
3603
|
+
return contentHashOf(JSON.stringify({
|
|
3604
|
+
title: card.title, status: card.status, content: card.content, sourceRefs: card.sourceRefs,
|
|
3605
|
+
review: effectiveReviewScore(card),
|
|
3606
|
+
}))
|
|
2425
3607
|
}
|
|
2426
3608
|
|
|
2427
3609
|
/** Previous `wiki_notes.json` (or the pre-0014 mirror), without touching `this.units`. */
|
|
@@ -2463,19 +3645,19 @@ export class KnowledgeRuntime {
|
|
|
2463
3645
|
private emptyExtractionOutcome(extractionHashes: Record<string, string>): ExtractionOutcome {
|
|
2464
3646
|
return {
|
|
2465
3647
|
extractionHashes,
|
|
2466
|
-
candidatesExtracted: 0, candidatesRejected: 0,
|
|
3648
|
+
candidatesExtracted: 0, candidatesRejected: 0, cardsCreated: 0, cardsUpdated: 0,
|
|
2467
3649
|
sourcesExtracted: 0, sourcesSkipped: 0, relations: [], relationsBySource: {}, warnings: [],
|
|
2468
3650
|
}
|
|
2469
3651
|
}
|
|
2470
3652
|
|
|
2471
3653
|
/**
|
|
2472
|
-
* Pass 0:对指纹变了的来源做候选抽取 +
|
|
3654
|
+
* Pass 0:对指纹变了的来源做候选抽取 + 归一,并把结果**直接落成 wiki 卡片**。
|
|
2473
3655
|
*
|
|
2474
3656
|
* 三条硬约束:
|
|
2475
3657
|
* 1. **增量**:指纹没变的来源一次都不抽(这是 0025 增量承诺的一部分);
|
|
2476
3658
|
* 2. **不重写没变的卡片**:合并只追加 `source_refs`,正文不动,所以不会凭空造
|
|
2477
3659
|
* 版本快照(`snapshotCard` 只认正文变化);
|
|
2478
|
-
* 3. **fail-soft
|
|
3660
|
+
* 3. **fail-soft**:一个候选写不进去、甚至一份来源整个抛错,都只记一笔,
|
|
2479
3661
|
* 绝不带走整次重建 —— 重建失败比少一张卡片严重得多。
|
|
2480
3662
|
*/
|
|
2481
3663
|
private async runExtractionStage(sources: SourceRaw[], previousHashes: Record<string, string>): Promise<ExtractionOutcome> {
|
|
@@ -2483,6 +3665,21 @@ export class KnowledgeRuntime {
|
|
|
2483
3665
|
const enabledTypes = schema.types.filter(def => def.enabled)
|
|
2484
3666
|
const registryHash = contentHashOf(JSON.stringify(enabledTypes.map(def => ({ id: def.id, description: def.description, examples: def.examples }))))
|
|
2485
3667
|
const outcome = this.emptyExtractionOutcome({})
|
|
3668
|
+
/**
|
|
3669
|
+
* 模型调用**每轮解析一次**,而且解析不到时**如实报出来**(0044)。
|
|
3670
|
+
*
|
|
3671
|
+
* 静默降级的代价实测过:默认模型明明配着,接缝却永远是空的,于是编译一路按
|
|
3672
|
+
* `##` 小节切了好几个月,而界面上没有任何一处说「模型抽取没生效」。人只会得出
|
|
3673
|
+
* 一个结论 ——「这个编译没用」。降级可以是诚实的,前提是它说话。
|
|
3674
|
+
*/
|
|
3675
|
+
const seam = this.config.extractWithModel ? this.modelSeamNow() : {}
|
|
3676
|
+
const call = seam.call
|
|
3677
|
+
if (this.config.extractWithModel && !call) {
|
|
3678
|
+
outcome.warnings.push(
|
|
3679
|
+
`模型抽取已开启,但取不到可用的模型调用:${seam.why ?? '原因未知'} —— 本次退回规则抽取:`
|
|
3680
|
+
+ '卡片正文是按 `##` 小节切下来的原文,判重也只能比字面,同一实体换个标题写法会落成两张卡。',
|
|
3681
|
+
)
|
|
3682
|
+
}
|
|
2486
3683
|
/**
|
|
2487
3684
|
* 上一轮落下的关系产物。**跳过的来源从这里读回自己的关系** —— 没有这一步,
|
|
2488
3685
|
* 一次全量重编就会把图上所有 `relates_to` 边抹掉(见 `EXTRACTED_RELATIONS`)。
|
|
@@ -2491,12 +3688,12 @@ export class KnowledgeRuntime {
|
|
|
2491
3688
|
// 计数器与产物不能由 readdir 的顺序决定,所以先按 sourceId 排序。
|
|
2492
3689
|
const ordered = [...sources].sort((left, right) => left.sourceId.localeCompare(right.sourceId))
|
|
2493
3690
|
/**
|
|
2494
|
-
*
|
|
2495
|
-
*
|
|
2496
|
-
* 正文累积(而不是产生 N 条互相打架的提议)。
|
|
3691
|
+
* 库里的卡片是判重的参照物。整轮结束才一次性落盘 —— 同一张卡片被多份来源命中时
|
|
3692
|
+
* 只写一次,正文在这一轮内累积(而不是产生 N 次互相覆盖的写入)。
|
|
2497
3693
|
*/
|
|
2498
3694
|
const cards = await this.loadCards()
|
|
2499
|
-
const
|
|
3695
|
+
const byKey = new Map(cards.map(card => [`${card.entityType}:${card.entityId}`, card]))
|
|
3696
|
+
const pending = new Map<string, PendingCard>()
|
|
2500
3697
|
for (const source of ordered) {
|
|
2501
3698
|
const fingerprint = this.extractionFingerprint(source, registryHash)
|
|
2502
3699
|
outcome.extractionHashes[source.sourceId] = fingerprint
|
|
@@ -2511,33 +3708,120 @@ export class KnowledgeRuntime {
|
|
|
2511
3708
|
}
|
|
2512
3709
|
outcome.sourcesExtracted += 1
|
|
2513
3710
|
try {
|
|
2514
|
-
|
|
3711
|
+
// 同一轮里后一份来源应当能看到前一份来源刚抽出来的实体,所以判重索引
|
|
3712
|
+
// 每份来源都用「磁盘上的卡片 + 这一轮 pending 的卡片」重建。
|
|
3713
|
+
await this.compileSource(source, byKey, pending, enabledTypes, outcome, call)
|
|
2515
3714
|
} catch (cause) {
|
|
2516
3715
|
outcome.warnings.push(`来源 ${source.sourceId} 的候选抽取失败,已跳过该来源:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
2517
3716
|
}
|
|
2518
3717
|
}
|
|
2519
3718
|
/**
|
|
2520
|
-
*
|
|
3719
|
+
* 落**卡片**,不是提议(0044)。
|
|
3720
|
+
*
|
|
3721
|
+
* 每个计数都对应一次真实的 `writeCard`:只有真的写成功了才算「新建 / 更新」。
|
|
3722
|
+
* 写失败的进 `warnings` 而不是静默消失 —— 少一张卡片是一个需要被看见的事件。
|
|
3723
|
+
*/
|
|
3724
|
+
/**
|
|
3725
|
+
* 图落位(0046)。
|
|
2521
3726
|
*
|
|
2522
|
-
*
|
|
2523
|
-
*
|
|
2524
|
-
*
|
|
3727
|
+
* 抽取那一段交给模型的是文字,模型给回来的正文里没有图,于是来源里 22 张图表
|
|
3728
|
+
* 一张也进不了卡片 —— 而多模态编码读的正是 `imagePaths[0]`。图是**确定性**的
|
|
3729
|
+
* (来源里有哪张图、它跟在哪段话后面,都不需要模型判断),所以在落盘前单独算一遍。
|
|
3730
|
+
*
|
|
3731
|
+
* 跨卡片一起算:一篇方法论文章编出 8 张卡,同一张图必须只落一张。已经引用过这张图
|
|
3732
|
+
* 的卡片不算「变了」,所以重复编译不会反复写盘。
|
|
2525
3733
|
*/
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
3734
|
+
const folded = new Map<string, string>()
|
|
3735
|
+
for (const [key, item] of pending) folded.set(key, appendExtracted(item.existing?.content ?? '', [item.content]))
|
|
3736
|
+
const view: Array<{ key: string; content: string; sourceRefs: string[] }> = []
|
|
3737
|
+
for (const [key, card] of byKey) view.push({ key, content: card.content, sourceRefs: card.sourceRefs })
|
|
3738
|
+
for (const [key, item] of pending) view.push({ key, content: folded.get(key) ?? '', sourceRefs: item.sourceRefs })
|
|
3739
|
+
const imagePlan = planSourceImages({ cards: view, sources: ordered.map(source => ({ sourceId: source.sourceId, content: source.content })) })
|
|
3740
|
+
for (const [key, item] of pending) {
|
|
3741
|
+
let content = folded.get(key) ?? ''
|
|
3742
|
+
for (const image of imagePlan.byCard.get(key) ?? []) content = insertImage(content, image)
|
|
3743
|
+
folded.set(key, content)
|
|
3744
|
+
}
|
|
3745
|
+
for (const [sourceId, count] of countBySource(imagePlan.unplaced)) {
|
|
3746
|
+
outcome.warnings.push(`来源 ${sourceId} 有 ${count} 张图没找到落点(既不在任何卡片正文里,也不属于图所在的来源小节),仍留在来源里`)
|
|
3747
|
+
}
|
|
3748
|
+
const writtenKeys = new Set<string>()
|
|
3749
|
+
for (const item of pending.values()) {
|
|
3750
|
+
const key = `${item.entityType}:${item.entityId}`
|
|
3751
|
+
const existing = item.existing
|
|
3752
|
+
const metadata = { ...item.metadata, ...(item.merges.length > 0 ? { extraction_merges: mergeHistoryOf(item.metadata, item.merges) } : {}) }
|
|
3753
|
+
/**
|
|
3754
|
+
* 已经一模一样的卡片不重写。
|
|
3755
|
+
*
|
|
3756
|
+
* 判据只有两条:**正文没有新增内容**,而且**所有来源都已在 `source_refs` 里**。
|
|
3757
|
+
* 少了这条,重建一次就会把每张卡片重写一遍 —— 而增量编译的意义正在于
|
|
3758
|
+
* 「重编一次,没变的卡片不该产生任何写入」。
|
|
3759
|
+
*
|
|
3760
|
+
* 比较的是「追加之后是否还是原样」,而不是逐字比较:`item.content` 是从已有正文
|
|
3761
|
+
* 出发追加出来的,两者相等就说明这次抽到的东西一个字都不新。图也算在内:
|
|
3762
|
+
* 计划里带图而正文里没有,就是「变了」。
|
|
3763
|
+
*/
|
|
3764
|
+
const content = folded.get(key) ?? item.content
|
|
3765
|
+
writtenKeys.add(key)
|
|
3766
|
+
const unchanged = existing !== undefined
|
|
3767
|
+
&& !item.fieldChanged
|
|
3768
|
+
&& content === existing.content
|
|
3769
|
+
&& item.sourceRefs.every(ref => existing.sourceRefs.includes(ref))
|
|
3770
|
+
if (unchanged) continue
|
|
3771
|
+
const written = await this.writeCard({
|
|
2531
3772
|
entityType: item.entityType,
|
|
2532
3773
|
entityId: item.entityId,
|
|
2533
3774
|
title: item.title,
|
|
2534
|
-
content
|
|
3775
|
+
content,
|
|
2535
3776
|
sourceRefs: item.sourceRefs,
|
|
2536
|
-
...(item.
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
3777
|
+
...(item.summary ? { summary: item.summary } : {}),
|
|
3778
|
+
status: existing?.status ?? 'unreviewed',
|
|
3779
|
+
...(existing?.owner ? { owner: existing.owner } : {}),
|
|
3780
|
+
...(existing?.provenance ? { provenance: existing.provenance } : {}),
|
|
3781
|
+
metadata,
|
|
3782
|
+
origin: 'compile',
|
|
3783
|
+
})
|
|
3784
|
+
if (written.ok === false) {
|
|
3785
|
+
outcome.warnings.push(`卡片 ${item.entityId} 写入失败(${written.error.code}):${written.error.message}`)
|
|
3786
|
+
continue
|
|
3787
|
+
}
|
|
3788
|
+
if (item.action === 'create') outcome.cardsCreated += 1
|
|
3789
|
+
else outcome.cardsUpdated += 1
|
|
3790
|
+
for (const warning of written.warnings ?? []) outcome.warnings.push(`卡片 ${item.entityId}:${warning}`)
|
|
3791
|
+
}
|
|
3792
|
+
/**
|
|
3793
|
+
* 只补图的卡片:指纹没变的来源这一轮**不重抽**(`sourcesSkipped`),所以它的卡片
|
|
3794
|
+
* 压根不在 `pending` 里 —— 而「卡片正文里有没有图」不影响抽取指纹。
|
|
3795
|
+
*
|
|
3796
|
+
* 少了这一段,0046 之前编好的库永远拿不到图:重编一次,来源被跳过,图还是不在。
|
|
3797
|
+
* 这段只做一件事:把算好的图插进去,正文别的部分一个字都不动。
|
|
3798
|
+
*/
|
|
3799
|
+
for (const [key, images] of imagePlan.byCard) {
|
|
3800
|
+
if (writtenKeys.has(key)) continue
|
|
3801
|
+
const card = byKey.get(key)
|
|
3802
|
+
if (!card) continue
|
|
3803
|
+
let content = card.content
|
|
3804
|
+
for (const image of images) content = insertImage(content, image)
|
|
3805
|
+
if (content === card.content) continue
|
|
3806
|
+
const written = await this.writeCard({
|
|
3807
|
+
entityType: card.entityType,
|
|
3808
|
+
entityId: card.entityId,
|
|
3809
|
+
title: card.title,
|
|
3810
|
+
content,
|
|
3811
|
+
sourceRefs: card.sourceRefs,
|
|
3812
|
+
...(card.summary ? { summary: card.summary } : {}),
|
|
3813
|
+
status: card.status,
|
|
3814
|
+
...(card.owner ? { owner: card.owner } : {}),
|
|
3815
|
+
...(card.provenance ? { provenance: card.provenance } : {}),
|
|
3816
|
+
metadata: card.metadata,
|
|
3817
|
+
origin: 'compile',
|
|
3818
|
+
})
|
|
3819
|
+
if (written.ok === false) {
|
|
3820
|
+
outcome.warnings.push(`卡片 ${card.entityId} 补图失败(${written.error.code}):${written.error.message}`)
|
|
3821
|
+
continue
|
|
3822
|
+
}
|
|
3823
|
+
outcome.cardsUpdated += 1
|
|
3824
|
+
for (const warning of written.warnings ?? []) outcome.warnings.push(`卡片 ${card.entityId}:${warning}`)
|
|
2541
3825
|
}
|
|
2542
3826
|
// 只留这一轮真实存在的来源:删掉的文档不该在图上留下幽灵关系。
|
|
2543
3827
|
const alive = new Set(ordered.map(source => source.sourceId))
|
|
@@ -2581,76 +3865,127 @@ export class KnowledgeRuntime {
|
|
|
2581
3865
|
}
|
|
2582
3866
|
|
|
2583
3867
|
/**
|
|
2584
|
-
*
|
|
3868
|
+
* 一份来源的编译结果:**按 wiki schema 抽实体**(新建 / 更新),并收集关系候选。
|
|
2585
3869
|
*
|
|
2586
|
-
*
|
|
2587
|
-
*
|
|
2588
|
-
*
|
|
2589
|
-
*
|
|
2590
|
-
*
|
|
3870
|
+
* ## 模型先抽实体,规则补关系
|
|
3871
|
+
*
|
|
3872
|
+
* 规则抽取把「小节标题」当实体身份,实测下来的后果是同一件东西横跨两份文档就
|
|
3873
|
+
* 变成两张卡片(`新建 2 / 并入 0`),而且卡片正文是按 `##` 切下来的原文片段 ——
|
|
3874
|
+
* 也就是「编译只是按目录截取」。所以实体这一层交给模型:它按类型的必填小节与字段
|
|
3875
|
+
* 写正文,并回答「这个实体是不是已经在下表里了」。
|
|
3876
|
+
*
|
|
3877
|
+
* 但仍然**照跑一遍规则抽取**,理由有两个:关系候选只有规则会抽(模型这一层不产
|
|
3878
|
+
* 关系),而且模型不可用时它就是那条降级路径。两者不是二选一。
|
|
3879
|
+
*
|
|
3880
|
+
* ## 缺信息也抽
|
|
3881
|
+
*
|
|
3882
|
+
* 0040 那一版在这里拒绝「没有正文」的候选、拒绝「类型未登记」的候选。两条都撤掉了:
|
|
3883
|
+
* 编译的价值在于**记下来,后面有涉及到的文档再慢慢补**,而拒绝恰好把这个机会丢掉。
|
|
3884
|
+
* 只剩一条硬要求 —— 正文不能是空的(一条什么都说不出的卡片没有东西可召回);
|
|
3885
|
+
* 模型路径已经保证只要给了 summary 就有骨架,规则路径的候选天然带正文。
|
|
2591
3886
|
*/
|
|
2592
|
-
private async
|
|
3887
|
+
private async compileSource(
|
|
2593
3888
|
source: SourceRaw,
|
|
2594
|
-
|
|
3889
|
+
byKey: Map<string, KnowledgeCard>,
|
|
3890
|
+
pending: Map<string, PendingCard>,
|
|
2595
3891
|
enabledTypes: WikiTypeDef[],
|
|
2596
3892
|
outcome: ExtractionOutcome,
|
|
2597
|
-
|
|
3893
|
+
/** 这一轮可用的模型调用;由 `runExtractionStage` 解析一次后传进来。 */
|
|
3894
|
+
call: ModelCall | undefined,
|
|
2598
3895
|
): Promise<void> {
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
*
|
|
2606
|
-
* 但仍然**照跑一遍规则抽取**,理由有两个:关系候选只有规则会抽(模型这一层不产
|
|
2607
|
-
* 关系),而且模型不可用时它就是那条降级路径。两者不是二选一。
|
|
2608
|
-
*/
|
|
3896
|
+
// 判重索引要看到「这一轮前面几份来源刚抽出来的卡片」,否则同一实体跨两份文档
|
|
3897
|
+
// 会各建一张 —— 而那正是这次改造要修掉的那个 bug。
|
|
3898
|
+
const cards = [...byKey.values()]
|
|
3899
|
+
// 同一份来源里的两个候选指向同一张卡片时,只该合并一次。
|
|
3900
|
+
const merges = new Map<string, NormalizedCandidates['merges'][number]>()
|
|
3901
|
+
|
|
2609
3902
|
const ruleCandidates = extractCandidates(source, enabledTypes)
|
|
2610
3903
|
outcome.candidatesExtracted += ruleCandidates.length
|
|
2611
3904
|
const ruleNormalized = normalizeCandidates(ruleCandidates, buildAliasIndex(cards))
|
|
2612
3905
|
|
|
2613
|
-
let
|
|
2614
|
-
|
|
2615
|
-
|
|
3906
|
+
let fresh = ruleNormalized.fresh
|
|
3907
|
+
for (const merge of ruleNormalized.merges) if (!merges.has(merge.key)) merges.set(merge.key, merge)
|
|
3908
|
+
|
|
3909
|
+
/**
|
|
3910
|
+
* 这一份来源**实际用了哪条路**(0044 实机验证修的第二个诚实问题)。
|
|
3911
|
+
*
|
|
3912
|
+
* 原来这里写的是 `extraction: call ? 'model' : 'pass0'` —— 判据是「有没有接缝」,
|
|
3913
|
+
* 不是「模型给的东西有没有被用上」。实机上模型确实被调用了,但输出不是 JSON,
|
|
3914
|
+
* 于是候选整批来自规则路径,卡片却标着 `extraction: model`:**脏的是标签,不是数据**。
|
|
3915
|
+
* 下一次有人排查「为什么正文还是按 `##` 切的」,会盯着这个字段得出相反结论。
|
|
3916
|
+
*
|
|
3917
|
+
* 现在的判据是产物:模型给出至少一条实体(新建或并入)才算 `model`。
|
|
3918
|
+
*/
|
|
3919
|
+
let extractionPath: 'model' | 'pass0' = 'pass0'
|
|
3920
|
+
if (call) {
|
|
3921
|
+
const modelled = await extractWithModel({
|
|
3922
|
+
source,
|
|
3923
|
+
wikiTypes: enabledTypes,
|
|
3924
|
+
existing: cards,
|
|
3925
|
+
call,
|
|
3926
|
+
/**
|
|
3927
|
+
* 路由与缓存亲和的会话身份,**按知识库稳定**(0044 实机验证第五批)。
|
|
3928
|
+
*
|
|
3929
|
+
* 没有它,OpenCode 系路由直接回 400 `MissingSessionID` —— 那件事在界面上长得
|
|
3930
|
+
* 像「模型抽取不可用」。用 `knowledge:<库 id>` 而不是随机 UUID:同一个库反复
|
|
3931
|
+
* 编译时提示词高度重复(类型定义 + 已有卡片清单),稳定的值把请求钉在同一个
|
|
3932
|
+
* 上游后端,命中缓存的概率才成立。
|
|
3933
|
+
*/
|
|
3934
|
+
sessionId: `knowledge:${this.baseId || 'default'}`,
|
|
3935
|
+
})
|
|
2616
3936
|
for (const warning of modelled.warnings) outcome.warnings.push(warning)
|
|
2617
3937
|
// 模型一条实体都没给(或者不可用)时,完整退回规则结果,而不是丢掉整轮抽取。
|
|
2618
3938
|
if (modelled.fresh.length > 0 || modelled.merges.length > 0) {
|
|
2619
|
-
|
|
3939
|
+
extractionPath = 'model'
|
|
3940
|
+
fresh = modelled.fresh
|
|
3941
|
+
merges.clear()
|
|
3942
|
+
for (const merge of modelled.merges) if (!merges.has(merge.key)) merges.set(merge.key, merge)
|
|
3943
|
+
} else if (modelled.warnings.length === 0) {
|
|
3944
|
+
// 模型正常回答、只是没给实体 —— 这也是一次实际的降级,不该只体现在卡片标签上。
|
|
3945
|
+
outcome.warnings.push('模型没有给出任何实体,本次退回规则抽取(正文按 `##` 小节切)。')
|
|
2620
3946
|
}
|
|
2621
3947
|
}
|
|
2622
3948
|
|
|
2623
|
-
|
|
3949
|
+
const at = new Date().toISOString()
|
|
3950
|
+
for (const candidate of fresh) {
|
|
2624
3951
|
/**
|
|
2625
|
-
*
|
|
3952
|
+
* 类型:未登记就**退回一个已启用的兜底类型**,而不是丢掉这条候选。
|
|
2626
3953
|
*
|
|
2627
|
-
*
|
|
2628
|
-
*
|
|
2629
|
-
*
|
|
2630
|
-
*
|
|
2631
|
-
*
|
|
2632
|
-
*
|
|
3954
|
+
* 0040 时期这里是 `candidatesRejected += 1` 然后 continue。那条规则与
|
|
3955
|
+
* 「缺信息也编译进去」直接冲突:类型没登记是**配置问题**,不是「这条知识不存在」——
|
|
3956
|
+
* 而配置问题该由配置界面回答,不该让文档里的知识消失。退回时记一条 warning,
|
|
3957
|
+
* 把原始类型名带上,人是能顺藤摸瓜去登记它的。
|
|
3958
|
+
*
|
|
3959
|
+
* 模型路径(`extract-model.ts`)早就是这么做的,这里补齐规则路径 —— 两条路径对
|
|
3960
|
+
* 同一种情况给出不同结论,比任选一种都糟。
|
|
2633
3961
|
*/
|
|
2634
|
-
const
|
|
2635
|
-
|
|
3962
|
+
const wanted = enabledTypes.find(def => def.id === candidate.type)
|
|
3963
|
+
const resolvedType = wanted ?? enabledTypes.find(def => def.id === 'concept') ?? enabledTypes[0]
|
|
3964
|
+
if (!resolvedType) {
|
|
2636
3965
|
outcome.candidatesRejected += 1
|
|
2637
|
-
outcome.warnings.push(`候选「${candidate.name}
|
|
3966
|
+
outcome.warnings.push(`候选「${candidate.name}」未入库:没有任何已启用的 wiki 类型可落(先在「知识库设置 → Wiki 类型」里登记一个)`)
|
|
2638
3967
|
continue
|
|
2639
3968
|
}
|
|
3969
|
+
if (!wanted) {
|
|
3970
|
+
outcome.warnings.push(`候选「${candidate.name}」的类型 ${candidate.type} 未登记或已停用,已按「${resolvedType.id}」入库`)
|
|
3971
|
+
}
|
|
2640
3972
|
/**
|
|
2641
|
-
*
|
|
2642
|
-
*
|
|
2643
|
-
*
|
|
3973
|
+
* 正文:候选自带的正文优先,没有就用 summary 撑一个带必填小节骨架的最小正文。
|
|
3974
|
+
*
|
|
3975
|
+
* 两条路径(模型 / 规则)走同一个 `skeletonBody`,所以「只说了半句」的候选在两处
|
|
3976
|
+
* 得到同一种对待。撑完还是空 → 这是**唯一保留的拒绝**:一条什么都不说的卡片没有
|
|
3977
|
+
* 东西可召回,而它占的 id 会让下一份真正说了点什么的文档并进这张空卡里。
|
|
2644
3978
|
*/
|
|
2645
|
-
|
|
3979
|
+
const body = candidate.body?.trim() || skeletonBody(resolvedType, candidate.summary?.trim() ?? '')
|
|
3980
|
+
if (!body) {
|
|
2646
3981
|
outcome.candidatesRejected += 1
|
|
2647
|
-
outcome.warnings.push(`候选「${candidate.name}
|
|
3982
|
+
outcome.warnings.push(`候选「${candidate.name}」没有任何正文与摘要(只在别处被提到,所在小节也没取到),未入库`)
|
|
2648
3983
|
continue
|
|
2649
3984
|
}
|
|
2650
3985
|
const entityId = candidateEntityId(candidate.name)
|
|
2651
|
-
const key = `${
|
|
3986
|
+
const key = `${resolvedType.id}:${entityId}`
|
|
2652
3987
|
const metadata: Record<string, unknown> = {
|
|
2653
|
-
extraction:
|
|
3988
|
+
extraction: extractionPath,
|
|
2654
3989
|
candidate_kind: candidate.kind,
|
|
2655
3990
|
confidence: candidate.confidence,
|
|
2656
3991
|
evidence: {
|
|
@@ -2659,45 +3994,62 @@ export class KnowledgeRuntime {
|
|
|
2659
3994
|
},
|
|
2660
3995
|
// 重复的写法都进 aliases:判重认的是名字,不是卡片 id(中文名会退化成哈希 id)。
|
|
2661
3996
|
...(aliasesOf(candidate).length > 0 ? { aliases: aliasesOf(candidate) } : {}),
|
|
3997
|
+
// 类型定义里的字段(`metric.formula` 之类)落在 metadata 上,不占正文。
|
|
3998
|
+
...(candidate.fields ?? {}),
|
|
2662
3999
|
}
|
|
2663
4000
|
// 库里已经有这张卡片 → 不是新建,而是「把这次抽到的正文并进去」。
|
|
2664
|
-
const existing =
|
|
4001
|
+
const existing = byKey.get(key)
|
|
2665
4002
|
if (existing) {
|
|
2666
|
-
|
|
4003
|
+
mergeIntoPending(pending, existing, candidate.evidence.sourceId, [body], [candidate.name], at, candidate.fields, candidate.summary)
|
|
2667
4004
|
continue
|
|
2668
4005
|
}
|
|
2669
4006
|
const previous = pending.get(key)
|
|
2670
4007
|
if (previous) {
|
|
2671
4008
|
previous.content = appendExtracted(previous.content, [body])
|
|
2672
4009
|
if (!previous.sourceRefs.includes(candidate.evidence.sourceId)) previous.sourceRefs.push(candidate.evidence.sourceId)
|
|
4010
|
+
if (!previous.summary && candidate.summary) previous.summary = candidate.summary
|
|
4011
|
+
previous.metadata = { ...previous.metadata, ...(candidate.fields ?? {}) }
|
|
4012
|
+
previous.merges.push({ sourceId: candidate.evidence.sourceId, keys: [candidate.name], at })
|
|
2673
4013
|
continue
|
|
2674
4014
|
}
|
|
2675
4015
|
pending.set(key, {
|
|
2676
4016
|
action: 'create',
|
|
2677
|
-
entityType:
|
|
4017
|
+
entityType: resolvedType.id as SchemaType,
|
|
2678
4018
|
entityId,
|
|
2679
4019
|
title: candidate.name,
|
|
2680
4020
|
content: body,
|
|
2681
4021
|
sourceRefs: [candidate.evidence.sourceId],
|
|
2682
4022
|
metadata,
|
|
4023
|
+
...(candidate.summary ? { summary: candidate.summary } : {}),
|
|
4024
|
+
merges: [{ sourceId: candidate.evidence.sourceId, keys: [candidate.name], at }],
|
|
2683
4025
|
})
|
|
2684
4026
|
}
|
|
2685
4027
|
|
|
2686
|
-
//
|
|
2687
|
-
for (const merge of
|
|
2688
|
-
const
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
4028
|
+
// 归一:命中的候选把证据来源并进已有卡片。同一张卡片在这一轮里的多次命中合并成一条。
|
|
4029
|
+
for (const merge of merges.values()) {
|
|
4030
|
+
const key = `${merge.entityType}:${merge.entityId}`
|
|
4031
|
+
// 这一轮刚新建的卡片也要认:它此刻还在 `pending` 里,不在磁盘上。
|
|
4032
|
+
const existing = byKey.get(key)
|
|
4033
|
+
if (existing) {
|
|
4034
|
+
// 「只在没有的时候追加」:来源已在 source_refs 里、正文也已在卡片里时什么都不做 ——
|
|
4035
|
+
// 空写一次也会让「更新了几张卡」这个计数变成谎话。
|
|
4036
|
+
if (existing.sourceRefs.includes(merge.sourceId) && appendExtracted(existing.content, [merge.body]) === existing.content) continue
|
|
4037
|
+
mergeIntoPending(pending, existing, merge.sourceId, [merge.body], [merge.key], at, merge.fields, merge.summary)
|
|
4038
|
+
continue
|
|
4039
|
+
}
|
|
4040
|
+
const previous = pending.get(key)
|
|
4041
|
+
if (!previous) continue
|
|
4042
|
+
previous.content = appendExtracted(previous.content, [merge.body])
|
|
4043
|
+
if (!previous.sourceRefs.includes(merge.sourceId)) previous.sourceRefs.push(merge.sourceId)
|
|
4044
|
+
previous.metadata = { ...previous.metadata, ...(merge.fields ?? {}) }
|
|
4045
|
+
previous.merges.push({ sourceId: merge.sourceId, keys: [merge.key], at })
|
|
2694
4046
|
}
|
|
2695
4047
|
|
|
2696
|
-
outcome.relations.push(...
|
|
4048
|
+
outcome.relations.push(...ruleNormalized.relations)
|
|
2697
4049
|
// 按来源存一份:跳过的来源下次重建要靠它把关系读回来(见 EXTRACTED_RELATIONS)。
|
|
2698
4050
|
outcome.relationsBySource[source.sourceId] = [
|
|
2699
4051
|
...(outcome.relationsBySource[source.sourceId] ?? []),
|
|
2700
|
-
...
|
|
4052
|
+
...ruleNormalized.relations,
|
|
2701
4053
|
]
|
|
2702
4054
|
}
|
|
2703
4055
|
|
|
@@ -2729,6 +4081,326 @@ export class KnowledgeRuntime {
|
|
|
2729
4081
|
return { extracted, dropped }
|
|
2730
4082
|
}
|
|
2731
4083
|
|
|
4084
|
+
// ------------------------------------------- Finalize · 索引 / 断链 / 相关(0049)
|
|
4085
|
+
|
|
4086
|
+
/**
|
|
4087
|
+
* 收尾段的**正文层**:给卡片补相关块、把断链写成 `reference` 问题。
|
|
4088
|
+
*
|
|
4089
|
+
* 它必须在 stage 2(切 wikiNotes)**之前**跑:相关块是正文的一部分,晚于切分写进去,
|
|
4090
|
+
* 这一轮编出来的召回单元就与卡片正文对不上,而下次重建又会因为正文变了重切一遍 ——
|
|
4091
|
+
* 每次编译都白编一次。
|
|
4092
|
+
*
|
|
4093
|
+
* 三段都是确定性的,所以这个函数可以重复跑:没变化的卡片一次都不写盘(判据是
|
|
4094
|
+
* 「正文 + 问题清单逐字相同」)。这也让「重编一次,卡片不该有任何写入」这条承诺
|
|
4095
|
+
* 在收尾段照样成立。
|
|
4096
|
+
*/
|
|
4097
|
+
private async runFinalizeStage(
|
|
4098
|
+
cards: KnowledgeCard[],
|
|
4099
|
+
relations: Array<{ fromKey: string; toKey: string }>,
|
|
4100
|
+
warnings: string[],
|
|
4101
|
+
): Promise<FinalizeOutcome> {
|
|
4102
|
+
const related = planRelated(cards, relations)
|
|
4103
|
+
/**
|
|
4104
|
+
* 断链扫的是**写完相关块之后的正文**,不是读进来的那一份。
|
|
4105
|
+
*
|
|
4106
|
+
* 相关块自己就是一堆 `[[类型/标题]]`:扫旧正文等于这一轮看不见它们,而它们马上
|
|
4107
|
+
* 要写进文件 —— 于是「引用是否成立」这件事被推迟到下一轮,而下一轮又会因为
|
|
4108
|
+
* 问题清单变了再写一遍。扫新正文,一轮就说清楚。
|
|
4109
|
+
*/
|
|
4110
|
+
const nextContent = new Map(cards.map(card => [cardKey(card), applyRelatedSection(card.content, related.get(cardKey(card)) ?? [])]))
|
|
4111
|
+
const deadLinks = deadLinksOf(cards.map(card => ({ ...card, content: nextContent.get(cardKey(card)) ?? card.content })))
|
|
4112
|
+
const deadByKey = new Map<string, string[]>()
|
|
4113
|
+
for (const link of deadLinks) {
|
|
4114
|
+
const bucket = deadByKey.get(link.key) ?? []
|
|
4115
|
+
bucket.push(link.target)
|
|
4116
|
+
deadByKey.set(link.key, bucket)
|
|
4117
|
+
}
|
|
4118
|
+
let rewritten = 0
|
|
4119
|
+
for (const card of cards) {
|
|
4120
|
+
const key = cardKey(card)
|
|
4121
|
+
const content = nextContent.get(key) ?? card.content
|
|
4122
|
+
// 旧的问题清单里 `reference` 那几条是上一轮算的,先摘掉再按这一轮重算 ——
|
|
4123
|
+
// 断链修好之后那条问题必须消失,否则「待补」清单会越用越长。
|
|
4124
|
+
const kept = cardIssuesOf(card).filter(issue => issue.level !== 'reference')
|
|
4125
|
+
const reference = (deadByKey.get(key) ?? []).map(deadLinkIssue)
|
|
4126
|
+
const issues = [...kept, ...reference]
|
|
4127
|
+
if (content === card.content && JSON.stringify(issues) === JSON.stringify(cardIssuesOf(card))) continue
|
|
4128
|
+
/**
|
|
4129
|
+
* 没有来源的卡片写不进去(`writeCard` 要求 `source_refs` 非空),而这是**唯一**
|
|
4130
|
+
* 一种收尾段碰不到的卡片:它已经是一张不合法的卡片了。如实报出来,不静默跳过 ——
|
|
4131
|
+
* 这种卡片只能由人补来源,或者删掉。
|
|
4132
|
+
*/
|
|
4133
|
+
if (card.sourceRefs.length === 0) {
|
|
4134
|
+
warnings.push(`卡片 ${card.entityId} 没有来源,无法补相关块:正文只读,来源只能由编译或人工重新写入`)
|
|
4135
|
+
continue
|
|
4136
|
+
}
|
|
4137
|
+
const written = await this.writeCard({
|
|
4138
|
+
entityType: card.entityType,
|
|
4139
|
+
entityId: card.entityId,
|
|
4140
|
+
title: card.title,
|
|
4141
|
+
content,
|
|
4142
|
+
sourceRefs: card.sourceRefs,
|
|
4143
|
+
...(card.summary ? { summary: card.summary } : {}),
|
|
4144
|
+
status: card.status,
|
|
4145
|
+
...(card.owner ? { owner: card.owner } : {}),
|
|
4146
|
+
...(card.provenance ? { provenance: card.provenance } : {}),
|
|
4147
|
+
metadata: card.metadata,
|
|
4148
|
+
extraIssues: reference,
|
|
4149
|
+
origin: 'compile',
|
|
4150
|
+
})
|
|
4151
|
+
if (written.ok === false) {
|
|
4152
|
+
warnings.push(`卡片 ${card.entityId} 的相关块 / 断链未写入(${written.error.code}):${written.error.message}`)
|
|
4153
|
+
continue
|
|
4154
|
+
}
|
|
4155
|
+
rewritten += 1
|
|
4156
|
+
}
|
|
4157
|
+
return {
|
|
4158
|
+
related, deadLinks, rewritten,
|
|
4159
|
+
relatedLinks: [...related.values()].reduce((sum, entries) => sum + entries.length, 0),
|
|
4160
|
+
relatedCards: [...related.keys()].length,
|
|
4161
|
+
}
|
|
4162
|
+
}
|
|
4163
|
+
|
|
4164
|
+
/** 收尾段的**产物层**:写索引页。它读的是正文层算完之后的卡片清单。 */
|
|
4165
|
+
private async writeWikiIndex(input: {
|
|
4166
|
+
cards: KnowledgeCard[]
|
|
4167
|
+
related: Map<string, RelatedEntry[]>
|
|
4168
|
+
deadLinks: DeadLink[]
|
|
4169
|
+
}): Promise<void> {
|
|
4170
|
+
await writeTextAtomic(this.safePath('03_compiled', WIKI_INDEX), buildWikiIndex(input))
|
|
4171
|
+
}
|
|
4172
|
+
|
|
4173
|
+
/** 一条路的候选 → 账本行(0050)。名次按这一路自己的顺序,没过阈值的也在。 */
|
|
4174
|
+
private ledgerRows(rows: Array<RouteRanking & { kept: boolean }> | undefined, unitById: Map<string, WikiNote>): RecallLedgerRow[] {
|
|
4175
|
+
return (rows ?? []).flatMap((row, index) => {
|
|
4176
|
+
const note = unitById.get(row.unitId)
|
|
4177
|
+
if (note === undefined) return []
|
|
4178
|
+
return [{
|
|
4179
|
+
unitId: row.unitId,
|
|
4180
|
+
title: note.title,
|
|
4181
|
+
entityType: note.entityType,
|
|
4182
|
+
entityId: note.entityId,
|
|
4183
|
+
score: row.score,
|
|
4184
|
+
rank: index + 1,
|
|
4185
|
+
kept: row.kept,
|
|
4186
|
+
}]
|
|
4187
|
+
})
|
|
4188
|
+
}
|
|
4189
|
+
|
|
4190
|
+
// ------------------------------------------- 召回组装:去重 / 扩展 / 预算(0050)
|
|
4191
|
+
|
|
4192
|
+
/**
|
|
4193
|
+
* **1 跳扩展**:从命中的卡片出发,按图上的边拉邻居。
|
|
4194
|
+
*
|
|
4195
|
+
* 两个来源,优先级相同(都记进 `via`,一条邻居可以有多条来路):
|
|
4196
|
+
* 1. **图上的边**(`relates_to` 这类卡片之间的关系)—— `supported_by` 不算:
|
|
4197
|
+
* 它连的是「卡片 → 来源」,而来源不是 wiki 卡片,进不了上下文;
|
|
4198
|
+
* 2. **共同来源**:同一份 SourceRaw 编译出来的卡片也是邻居(这是最实用的一类,
|
|
4199
|
+
* 新库里往往一条 `relates_to` 都没有)。
|
|
4200
|
+
*
|
|
4201
|
+
* 调用点必须在**按卡片去重之后**:先扩展再去重会把邻居与命中混在一起,
|
|
4202
|
+
* 而上下文里必须分得清「这是召回来的」还是「这是延伸出来的」。
|
|
4203
|
+
*/
|
|
4204
|
+
private expandWiki(seedKeys: string[], cardByKey: Map<string, KnowledgeCard>, includeProblems: boolean): RecallExpansion {
|
|
4205
|
+
const seeds = new Set(seedKeys)
|
|
4206
|
+
const visible = (card: KnowledgeCard): boolean => includeProblems || card.status !== 'problem'
|
|
4207
|
+
const via = new Map<string, string[]>()
|
|
4208
|
+
const remember = (key: string, from: string): void => {
|
|
4209
|
+
const bucket = via.get(key) ?? []
|
|
4210
|
+
if (!bucket.includes(from)) bucket.push(from)
|
|
4211
|
+
via.set(key, bucket)
|
|
4212
|
+
}
|
|
4213
|
+
const edges: Array<Record<string, unknown>> = []
|
|
4214
|
+
for (const edge of this.graph.edges) {
|
|
4215
|
+
const type = String(edge.type)
|
|
4216
|
+
if (type === 'supported_by') continue
|
|
4217
|
+
const from = String(edge.from)
|
|
4218
|
+
const to = String(edge.to)
|
|
4219
|
+
const fromSeed = seeds.has(from)
|
|
4220
|
+
const toSeed = seeds.has(to)
|
|
4221
|
+
// 两端都在种子里(种子里的一条内部边)或都不在(与这次召回无关):不是扩展。
|
|
4222
|
+
if (fromSeed === toSeed) continue
|
|
4223
|
+
const neighbor = fromSeed ? to : from
|
|
4224
|
+
const card = cardByKey.get(neighbor)
|
|
4225
|
+
if (card === undefined || !visible(card)) continue
|
|
4226
|
+
remember(neighbor, `${fromSeed ? from : to}|${type}`)
|
|
4227
|
+
edges.push(edge)
|
|
4228
|
+
}
|
|
4229
|
+
// 共同来源:反向走 `supported_by`(来源 → 它编译出的卡片)。
|
|
4230
|
+
const bySource = new Map<string, string[]>()
|
|
4231
|
+
for (const key of seeds) {
|
|
4232
|
+
for (const ref of cardByKey.get(key)?.sourceRefs ?? []) {
|
|
4233
|
+
const bucket = bySource.get(ref) ?? []
|
|
4234
|
+
bucket.push(key)
|
|
4235
|
+
bySource.set(ref, bucket)
|
|
4236
|
+
}
|
|
4237
|
+
}
|
|
4238
|
+
for (const card of cardByKey.values()) {
|
|
4239
|
+
const key = cardKey(card)
|
|
4240
|
+
if (seeds.has(key) || !visible(card)) continue
|
|
4241
|
+
for (const ref of card.sourceRefs) {
|
|
4242
|
+
const owners = bySource.get(ref)
|
|
4243
|
+
if (owners === undefined || owners.length === 0) continue
|
|
4244
|
+
remember(key, `${owners[0]}|shared_source`)
|
|
4245
|
+
}
|
|
4246
|
+
}
|
|
4247
|
+
const nodes: RecallExpansion['nodes'] = []
|
|
4248
|
+
for (const [key, sources] of via) {
|
|
4249
|
+
const card = cardByKey.get(key)
|
|
4250
|
+
if (card === undefined) continue
|
|
4251
|
+
nodes.push({ entityType: card.entityType, entityId: card.entityId, title: card.title, via: sources })
|
|
4252
|
+
}
|
|
4253
|
+
// 排序:来路多的在前(更可能是真的相关),同数按身份,保证确定性。
|
|
4254
|
+
nodes.sort((left, right) => right.via.length - left.via.length || `${left.entityType}:${left.entityId}`.localeCompare(`${right.entityType}:${right.entityId}`))
|
|
4255
|
+
const capped = nodes.slice(0, MAX_EXPANSION_CARDS)
|
|
4256
|
+
const kept = new Set(capped.map(node => `${node.entityType}:${node.entityId}`))
|
|
4257
|
+
return {
|
|
4258
|
+
nodes: capped,
|
|
4259
|
+
// 只留两端都在扩展集里的边:一端是命中卡片的边已经写在命中卡片自己的子图里了。
|
|
4260
|
+
edges: edges.filter(edge => kept.has(String(edge.from)) && kept.has(String(edge.to))),
|
|
4261
|
+
included: [],
|
|
4262
|
+
}
|
|
4263
|
+
}
|
|
4264
|
+
|
|
4265
|
+
/** 一条上下文条目「值多少字符」:命中条目算它那些 note 的正文,扩展条目算标题 + 摘要(+ 正文)。 */
|
|
4266
|
+
private contextCost(item: ContextItem, full: boolean): number {
|
|
4267
|
+
if (item.reason === 'hit') return item.notes.reduce((sum, note) => sum + note.title.length + note.content.length, 0)
|
|
4268
|
+
const card = item.wiki
|
|
4269
|
+
if (card === null) return 0
|
|
4270
|
+
const shallow = card.title.length + (card.summary?.length ?? 0)
|
|
4271
|
+
return full ? shallow + card.content.length : shallow
|
|
4272
|
+
}
|
|
4273
|
+
|
|
4274
|
+
/** 溢出目录:**系统临时目录**,不随知识库走(0050 的取舍,见 issue)。 */
|
|
4275
|
+
private recallOverflowDir(): string {
|
|
4276
|
+
return join(tmpdir(), 'dsh-knowledge-recall', this.baseId || 'default')
|
|
4277
|
+
}
|
|
4278
|
+
|
|
4279
|
+
/**
|
|
4280
|
+
* 上下文预算:装箱 + 溢出(0050)。
|
|
4281
|
+
*
|
|
4282
|
+
* 让位顺序(写死在这里,不是启发式):
|
|
4283
|
+
* 1. **命中的 note 正文优先**:它是证据,装不下就该少给几张卡片,而不是把证据截断;
|
|
4284
|
+
* 2. 扩展卡片先给**标题 + 摘要 + 来路**(便宜,且足以让模型知道「还有这么一张卡」);
|
|
4285
|
+
* 3. 还有余量才给扩展卡片的正文。
|
|
4286
|
+
*
|
|
4287
|
+
* 溢出的内容落**系统临时目录**(`<os.tmpdir()>/dsh-knowledge-recall/<库>/<recallId>.md`),
|
|
4288
|
+
* 不随库走:它是一次召回的临时产物,下一次召回就是另一份。代价是系统会清它 ——
|
|
4289
|
+
* 所以指路行必须说清「本次会话内有效」,并给出「读不到就重新召回」这条出路。
|
|
4290
|
+
*/
|
|
4291
|
+
private async packContext(input: { items: ContextItem[]; maxChars: number; recallId: string }): Promise<{
|
|
4292
|
+
items: ContextItem[]
|
|
4293
|
+
report: ContextBudgetReport
|
|
4294
|
+
notice?: string
|
|
4295
|
+
}> {
|
|
4296
|
+
const { items, maxChars, recallId } = input
|
|
4297
|
+
const hitCards = items.filter(item => item.reason === 'hit').length
|
|
4298
|
+
const expandedCards = items.length - hitCards
|
|
4299
|
+
const totalChars = items.reduce((sum, item) => sum + this.contextCost(item, true), 0)
|
|
4300
|
+
if (!Number.isFinite(maxChars) || maxChars <= 0) {
|
|
4301
|
+
return { items, report: { maxChars: 0, usedChars: totalChars, omittedCards: 0, omittedChars: 0, hitCards, expandedCards } }
|
|
4302
|
+
}
|
|
4303
|
+
const keptItems: ContextItem[] = []
|
|
4304
|
+
let used = 0
|
|
4305
|
+
for (const item of items) {
|
|
4306
|
+
const full = this.contextCost(item, true)
|
|
4307
|
+
const shallow = this.contextCost(item, false)
|
|
4308
|
+
// 命中条目按整条算(它的正文就是证据);扩展条目可以先按「浅」那一档进。
|
|
4309
|
+
const cost = item.reason === 'hit' ? full : shallow
|
|
4310
|
+
if (used + cost > maxChars) continue
|
|
4311
|
+
used += cost
|
|
4312
|
+
keptItems.push(item)
|
|
4313
|
+
}
|
|
4314
|
+
const keptKeys = new Set(keptItems.map(item => item.wiki === null ? '' : `${item.wiki.entityType}:${item.wiki.entityId}`))
|
|
4315
|
+
const omitted = items.filter(item => !keptKeys.has(item.wiki === null ? '' : `${item.wiki.entityType}:${item.wiki.entityId}`))
|
|
4316
|
+
const report: ContextBudgetReport = {
|
|
4317
|
+
maxChars,
|
|
4318
|
+
usedChars: used,
|
|
4319
|
+
omittedCards: omitted.length,
|
|
4320
|
+
omittedChars: omitted.reduce((sum, item) => sum + this.contextCost(item, true), 0),
|
|
4321
|
+
hitCards: keptItems.filter(item => item.reason === 'hit').length,
|
|
4322
|
+
expandedCards: keptItems.filter(item => item.reason === 'expansion').length,
|
|
4323
|
+
}
|
|
4324
|
+
if (omitted.length === 0) return { items: keptItems, report }
|
|
4325
|
+
const overflowPath = await this.writeRecallOverflow({ items, recallId, usedChars: used, maxChars })
|
|
4326
|
+
report.overflowPath = overflowPath
|
|
4327
|
+
return {
|
|
4328
|
+
items: keptItems,
|
|
4329
|
+
report,
|
|
4330
|
+
notice: `还有 ${report.omittedCards} 张卡片共 ${report.omittedChars} 字符没有放进上下文,完整内容在 ${overflowPath};需要更多信息时直接读这个文件(它是本次召回的临时产物,不保证长期存在)。`,
|
|
4331
|
+
}
|
|
4332
|
+
}
|
|
4333
|
+
|
|
4334
|
+
/** 把**完整**的上下文写进溢出文件(含被预算挡在外面的那些)。 */
|
|
4335
|
+
private async writeRecallOverflow(input: { items: ContextItem[]; recallId: string; usedChars: number; maxChars: number }): Promise<string> {
|
|
4336
|
+
const dir = this.recallOverflowDir()
|
|
4337
|
+
await mkdir(dir, { recursive: true })
|
|
4338
|
+
await this.pruneRecallOverflow(dir)
|
|
4339
|
+
const path = join(dir, `${input.recallId}.md`)
|
|
4340
|
+
const lines: string[] = [
|
|
4341
|
+
'# 这次召回的完整上下文',
|
|
4342
|
+
'',
|
|
4343
|
+
`上限 ${input.maxChars} 字符,上下文里装了 ${input.usedChars} 字符;这里是**全部**内容。`,
|
|
4344
|
+
'它是临时产物,读不到时重新召回一次即可。',
|
|
4345
|
+
'',
|
|
4346
|
+
]
|
|
4347
|
+
for (const item of input.items) {
|
|
4348
|
+
const card = item.wiki
|
|
4349
|
+
lines.push(`## ${card?.title ?? '(未找到卡片)'} · ${card?.entityType ?? ''}/${card?.entityId ?? ''}${item.reason === 'expansion' ? '(1 跳扩展)' : ''}`, '')
|
|
4350
|
+
if (item.via) lines.push(`来路:${item.via.entityType}/${item.via.entityId}(${item.via.relation})`, '')
|
|
4351
|
+
if (card?.summary) lines.push(`> ${card.summary}`, '')
|
|
4352
|
+
for (const note of item.notes) lines.push(`### ${note.title}`, '', note.content, '')
|
|
4353
|
+
if (item.reason === 'expansion' && card) lines.push(card.content, '')
|
|
4354
|
+
}
|
|
4355
|
+
await writeTextAtomic(path, `${lines.join('\n').replace(/\n+$/, '')}\n`)
|
|
4356
|
+
return path
|
|
4357
|
+
}
|
|
4358
|
+
|
|
4359
|
+
/** 清掉本库前缀下超过保留时长的溢出文件(自清理,不依赖系统清 tmp)。 */
|
|
4360
|
+
private async pruneRecallOverflow(dir: string): Promise<void> {
|
|
4361
|
+
try {
|
|
4362
|
+
const cutoff = Date.now() - RECALL_OVERFLOW_TTL_MS
|
|
4363
|
+
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
4364
|
+
if (!entry.isFile() || !entry.name.endsWith('.md')) continue
|
|
4365
|
+
const path = join(dir, entry.name)
|
|
4366
|
+
try {
|
|
4367
|
+
const info = await stat(path)
|
|
4368
|
+
if (info.mtimeMs < cutoff) await unlink(path)
|
|
4369
|
+
} catch { /* 并发删掉了就算了 */ }
|
|
4370
|
+
}
|
|
4371
|
+
} catch { /* 目录还不存在:没有什么要清 */ }
|
|
4372
|
+
}
|
|
4373
|
+
|
|
4374
|
+
/**
|
|
4375
|
+
* 一张卡片的出链与入链(面板抽屉里的「相关」分块)。
|
|
4376
|
+
*
|
|
4377
|
+
* 与写进正文的那一块**同一份计划**:两处都调 `planRelated`,所以界面上看到的关系
|
|
4378
|
+
* 与文件里写的不会漂移。入链要单独算 —— 关系是双向记的,但同源那一类是「谁和我
|
|
4379
|
+
* 同源」,只能扫一遍别人的出链。
|
|
4380
|
+
*/
|
|
4381
|
+
async relatedOfCard(card: KnowledgeCard): Promise<{ outgoing: RelatedEntry[]; incoming: RelatedEntry[] }> {
|
|
4382
|
+
/**
|
|
4383
|
+
* 图上那几条关系边住在 `knowledge_graph.json`,进程刚起来时内存里是空的。
|
|
4384
|
+
* 少了这一次读取,「关系」那一类相关会**静默缺失**(只剩同源那几条),而界面上
|
|
4385
|
+
* 看不出少了什么 —— 卡片抽屉是本插件里第一个在召回之外读图的地方。
|
|
4386
|
+
*/
|
|
4387
|
+
if (this.graph.edges.length === 0) await this.loadCompiledState()
|
|
4388
|
+
const cards = await this.loadCards()
|
|
4389
|
+
const graphRelations = this.graph.edges
|
|
4390
|
+
.filter(edge => String(edge.type) !== 'supported_by')
|
|
4391
|
+
.map(edge => ({ fromKey: String(edge.from), toKey: String(edge.to) }))
|
|
4392
|
+
const related = planRelated(cards, graphRelations)
|
|
4393
|
+
const key = cardKey(card)
|
|
4394
|
+
const outgoing = related.get(key) ?? []
|
|
4395
|
+
const incoming: RelatedEntry[] = []
|
|
4396
|
+
for (const [fromKey, entries] of related) {
|
|
4397
|
+
if (fromKey === key) continue
|
|
4398
|
+
const entry = entries.find(item => item.key === key)
|
|
4399
|
+
if (entry) incoming.push({ ...entry, key: fromKey })
|
|
4400
|
+
}
|
|
4401
|
+
return { outgoing, incoming }
|
|
4402
|
+
}
|
|
4403
|
+
|
|
2732
4404
|
async rebuild(only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
2733
4405
|
await this.ensureLayout()
|
|
2734
4406
|
const startedAt = new Date().toISOString()
|
|
@@ -2737,7 +4409,7 @@ export class KnowledgeRuntime {
|
|
|
2737
4409
|
const sourceTypes = new Map(sources.map(source => [source.sourceId, source.sourceType]))
|
|
2738
4410
|
|
|
2739
4411
|
// ------------------------------------------------ previous build state
|
|
2740
|
-
let previousManifest: { version?: number; builtAt?: string; cardHashes?: Record<string, string>; extractionHashes?: Record<string, string>; embedding?: unknown; diagnosisHash?: string } = {}
|
|
4412
|
+
let previousManifest: { version?: number; builtAt?: string; cardHashes?: Record<string, string>; extractionHashes?: Record<string, string>; embedding?: unknown; diagnosisHash?: string; indexHash?: string } = {}
|
|
2741
4413
|
try {
|
|
2742
4414
|
previousManifest = JSON.parse(await readFile(this.safePath('03_compiled', 'build-manifest.json'), 'utf8')) as typeof previousManifest
|
|
2743
4415
|
} catch { /* first build */ }
|
|
@@ -2755,13 +4427,20 @@ export class KnowledgeRuntime {
|
|
|
2755
4427
|
const extraction = extractEnabled
|
|
2756
4428
|
? await this.runExtractionStage(sources, previousExtractionHashes)
|
|
2757
4429
|
: this.emptyExtractionOutcome(previousExtractionHashes)
|
|
2758
|
-
|
|
2759
|
-
|
|
4430
|
+
/**
|
|
4431
|
+
* 抽取刚落过盘:stage 2 必须看到新的卡片清单,否则这一轮建的卡片要等下一次
|
|
4432
|
+
* 重建才进 wikiNotes,用户点了编译却召不回刚抽出来的东西。
|
|
4433
|
+
*
|
|
4434
|
+
* 0044 之前这一行是多余的(Pass 0 只写提议,卡片清单根本不会变),所以它一直是
|
|
4435
|
+
* 一个**没被触发过的**承诺;现在 Pass 0 直接落卡,漏掉它就会表现成「第一次编译
|
|
4436
|
+
* 什么都没建出来,第二次才有」—— 一个看起来像缓存没刷新的 bug。
|
|
4437
|
+
*/
|
|
4438
|
+
if (extractEnabled) this.cards = await this.loadCards()
|
|
2760
4439
|
const extractionCounts = {
|
|
2761
4440
|
candidatesExtracted: extraction.candidatesExtracted,
|
|
2762
4441
|
candidatesRejected: extraction.candidatesRejected,
|
|
2763
|
-
|
|
2764
|
-
|
|
4442
|
+
cardsCreated: extraction.cardsCreated,
|
|
4443
|
+
cardsUpdated: extraction.cardsUpdated,
|
|
2765
4444
|
sourcesExtracted: extraction.sourcesExtracted,
|
|
2766
4445
|
sourcesSkipped: extraction.sourcesSkipped,
|
|
2767
4446
|
}
|
|
@@ -2769,11 +4448,54 @@ export class KnowledgeRuntime {
|
|
|
2769
4448
|
let relationsExtracted = 0
|
|
2770
4449
|
let relationsDropped = 0
|
|
2771
4450
|
|
|
4451
|
+
// --------------------------------- Finalize · 相关块与断链(正文层,先于切分)
|
|
4452
|
+
/**
|
|
4453
|
+
* 收尾段在 stage 2 **之前**跑,理由有两条,都不是风格问题:
|
|
4454
|
+
*
|
|
4455
|
+
* 1. 相关块是正文的一部分。晚于切分写进去,这一轮编出的召回单元就与卡片正文对不上,
|
|
4456
|
+
* 而下一轮又会因为正文变了重切一遍 —— 每次编译都白编一次;
|
|
4457
|
+
* 2. 它必须参与「语料变了没有」的判定。0047 的图片那次踩过同一个坑:派生内容写盘
|
|
4458
|
+
* 晚于判定,增量编译判定「没有变化」,刚写进去的东西连同整个产物一起被丢掉。
|
|
4459
|
+
*
|
|
4460
|
+
* `extractEnabled` 为假(`only: 'embeddings'`)时整段不跑:它不碰 wiki 层。
|
|
4461
|
+
*/
|
|
4462
|
+
const finalize: FinalizeOutcome = extractEnabled
|
|
4463
|
+
? await this.runFinalizeStage(this.cards, extraction.relations, warnings)
|
|
4464
|
+
: { related: new Map(), relatedLinks: 0, relatedCards: 0, deadLinks: [], rewritten: 0 }
|
|
4465
|
+
// 正文层刚落过盘:stage 2 必须看到写完相关块之后的卡片。
|
|
4466
|
+
if (finalize.rewritten > 0) this.cards = await this.loadCards()
|
|
4467
|
+
/**
|
|
4468
|
+
* 索引页也是这次编译的产物,所以它变了也算「语料变了」。
|
|
4469
|
+
*
|
|
4470
|
+
* 少了这一条会踩一个很隐蔽的坑:一个**本来就没有任何关系**的知识库,收尾段一条
|
|
4471
|
+
* 卡片都不改,增量编译于是判定「没有变化」直接跳过 —— 索引页永远不会被写出来,
|
|
4472
|
+
* 而界面上没有任何一处说「这个文件没有生成」。
|
|
4473
|
+
*/
|
|
4474
|
+
const indexText = buildWikiIndex({ cards: this.cards, related: finalize.related, deadLinks: finalize.deadLinks })
|
|
4475
|
+
const indexHash = contentHashOf(indexText)
|
|
4476
|
+
// 文件被手工删掉时也算「变了」:它是产物,不该因为「内容没变」就永远回不来。
|
|
4477
|
+
const indexChanged = indexHash !== previousManifest.indexHash || !existsSync(this.safePath('03_compiled', WIKI_INDEX))
|
|
4478
|
+
const finalizeCounts = {
|
|
4479
|
+
relatedLinks: finalize.relatedLinks,
|
|
4480
|
+
relatedCards: finalize.relatedCards,
|
|
4481
|
+
deadLinks: finalize.deadLinks.length,
|
|
4482
|
+
finalizeRewritten: finalize.rewritten,
|
|
4483
|
+
}
|
|
4484
|
+
|
|
2772
4485
|
// ------------------------------------------------ stage 2 · compile notes
|
|
2773
4486
|
let cardsReused = 0
|
|
2774
4487
|
let cardsRecompiled = 0
|
|
2775
4488
|
let cardsAdded = 0
|
|
2776
4489
|
let notesReused = 0
|
|
4490
|
+
/**
|
|
4491
|
+
* 复用 note 时**派生字段**(图片路径、来源分类、溯源链)被刷新过。
|
|
4492
|
+
*
|
|
4493
|
+
* 它必须参与「语料变了没有」的判定:这些字段不在 `cardHashOf` 里(它们能从正文
|
|
4494
|
+
* 重算出来),所以一张卡片可以「一个字都没变」而编译产物确实不同 —— 0047 实机
|
|
4495
|
+
* 踩到的就是这个:解析规则修好了,增量编译却判定「没有变化」,直接走空操作返回,
|
|
4496
|
+
* 刷新过的路径连同整个 `wiki_notes.json` 一起被丢掉。
|
|
4497
|
+
*/
|
|
4498
|
+
let derivedRefreshed = false
|
|
2777
4499
|
let notesRecompiled = 0
|
|
2778
4500
|
const cardHashes: Record<string, string> = {}
|
|
2779
4501
|
/**
|
|
@@ -2804,16 +4526,33 @@ export class KnowledgeRuntime {
|
|
|
2804
4526
|
if (!previousHashes[key]) cardsAdded += 1
|
|
2805
4527
|
const kept = incremental && previousHashes[key] === fingerprint ? previousByWiki.get(key) : undefined
|
|
2806
4528
|
if (kept && kept.length > 0) {
|
|
2807
|
-
// Reuse the notes verbatim, but refresh the
|
|
2808
|
-
// change *without* the card changing: a SourceRaw's business type
|
|
2809
|
-
//
|
|
2810
|
-
// a source_type-filtered recall silently miss
|
|
4529
|
+
// Reuse the notes verbatim, but refresh the derived fields that can
|
|
4530
|
+
// change *without* the card changing: a SourceRaw's business type, the
|
|
4531
|
+
// card's provenance chain, and the image paths. Keeping a stale
|
|
4532
|
+
// `sourceTypes` would make a source_type-filtered recall silently miss
|
|
4533
|
+
// this card; keeping stale `imagePaths` would keep the multimodal
|
|
4534
|
+
// encoder pointed at a path that no longer resolves (0047: the first
|
|
4535
|
+
// live rebuild reused notes whose paths were resolved by the old rule,
|
|
4536
|
+
// so the encoder still got `02_wiki/concept/_assets/…`).
|
|
4537
|
+
//
|
|
4538
|
+
// 评分不在这里刷新:它已经是 `cardHashOf` 的一部分,所以「评分变了」走的是
|
|
4539
|
+
// 重编那条分支,不会走到这里。
|
|
2811
4540
|
const sourceTypesFor = card.sourceRefs.flatMap(ref => { const sourceType = sourceTypes.get(ref); return sourceType ? [sourceType] : [] })
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
4541
|
+
const cardDir = join(this.directory(WIKI), card.entityType)
|
|
4542
|
+
notes.push(...kept.map(note => {
|
|
4543
|
+
// 图按 note **自己那一节**的引用算 —— 与 `compileCardNotes` 同一条规则,
|
|
4544
|
+
// 否则复用与重编会给出两套答案。
|
|
4545
|
+
const images = markdownImagePaths(note.content, cardDir, this.rootDir)
|
|
4546
|
+
if (JSON.stringify(note.imagePaths ?? []) !== JSON.stringify(images)) derivedRefreshed = true
|
|
4547
|
+
if (JSON.stringify(note.sourceTypes ?? []) !== JSON.stringify(sourceTypesFor)) derivedRefreshed = true
|
|
4548
|
+
const { imagePaths: _stale, ...rest } = note
|
|
4549
|
+
return {
|
|
4550
|
+
...rest,
|
|
4551
|
+
...(images.length > 0 ? { imagePaths: images } : {}),
|
|
4552
|
+
sourceTypes: sourceTypesFor,
|
|
4553
|
+
...(card.provenance ? { provenance: card.provenance } : {}),
|
|
4554
|
+
}
|
|
4555
|
+
}))
|
|
2817
4556
|
cardsReused += 1
|
|
2818
4557
|
notesReused += kept.length
|
|
2819
4558
|
} else {
|
|
@@ -2843,7 +4582,7 @@ export class KnowledgeRuntime {
|
|
|
2843
4582
|
const diagnosis = await this.diagnosisStore().readGraph()
|
|
2844
4583
|
const diagnosisHash = contentHashOf(JSON.stringify({ nodes: diagnosis.nodes, edges: diagnosis.edges }))
|
|
2845
4584
|
const diagnosisChanged = diagnosisHash !== previousManifest.diagnosisHash
|
|
2846
|
-
const corpusChanged = !incremental || previousVersion === 0 || cardsRecompiled > 0 || cardsRemoved > 0 || diagnosisChanged
|
|
4585
|
+
const corpusChanged = !incremental || previousVersion === 0 || cardsRecompiled > 0 || cardsRemoved > 0 || diagnosisChanged || derivedRefreshed || indexChanged
|
|
2847
4586
|
if (!corpusChanged) {
|
|
2848
4587
|
const compileRunId = `rebuild_${hash(`${startedAt}:${this.cards.length}`)}`
|
|
2849
4588
|
const report: RebuildReport = {
|
|
@@ -2854,7 +4593,7 @@ export class KnowledgeRuntime {
|
|
|
2854
4593
|
cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0,
|
|
2855
4594
|
notesReused, notesRecompiled: 0,
|
|
2856
4595
|
incremental: true, skipped: true,
|
|
2857
|
-
...extractionCounts,
|
|
4596
|
+
...extractionCounts, ...finalizeCounts,
|
|
2858
4597
|
// 这一轮没有重装图,所以没有一条边是「这次连上的」或「这次丢掉的」。
|
|
2859
4598
|
// 报 0 而不是把图里已有的边算进来:那会把历史成果记到这次空操作头上。
|
|
2860
4599
|
relationsExtracted: 0, relationsDropped: 0,
|
|
@@ -2864,7 +4603,7 @@ export class KnowledgeRuntime {
|
|
|
2864
4603
|
// 用户刚点了编译却看到上次的统计,会以为按钮没生效。version 不变,所以
|
|
2865
4604
|
// 这个写入既不改索引也不让向量缓存失效。extractionHashes 必须跟着更新:
|
|
2866
4605
|
// 来源变了但没抽出卡片时,不落新指纹会让下一次又把它重抽一遍。
|
|
2867
|
-
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version: previousVersion, builtAt: previousManifest.builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: previousManifest.embedding, diagnosisHash, compileRunId, lastBuild: { only, incremental: true, skipped: true, cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0, notesReused, notesRecompiled: 0, ...extractionCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
4606
|
+
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version: previousVersion, builtAt: previousManifest.builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: previousManifest.embedding, diagnosisHash, indexHash, compileRunId, lastBuild: { only, incremental: true, skipped: true, cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0, notesReused, notesRecompiled: 0, ...extractionCounts, ...finalizeCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
2868
4607
|
return success(report, ['知识库没有变化:已复用全部 wikiNotes,未生成新的索引版本。', ...extraction.warnings])
|
|
2869
4608
|
}
|
|
2870
4609
|
|
|
@@ -2897,6 +4636,15 @@ export class KnowledgeRuntime {
|
|
|
2897
4636
|
*/
|
|
2898
4637
|
this.diagnosisGraph = { nodes: diagnosis.nodes as unknown as Array<Record<string, unknown>>, edges: diagnosis.edges as unknown as Array<Record<string, unknown>> }
|
|
2899
4638
|
await writeTextAtomic(this.safePath('03_compiled', DIAGNOSIS_GRAPH), JSON.stringify({ version, builtAt, nodes: diagnosis.nodes, edges: diagnosis.edges }, null, 2))
|
|
4639
|
+
/**
|
|
4640
|
+
* 收尾段的**产物层**:索引页(0049)。
|
|
4641
|
+
*
|
|
4642
|
+
* 放在图装配之后:索引页里那几条「相关」是**图上的边**,先写索引再装图,写出来的
|
|
4643
|
+
* 就会是上一轮的图。`indexText` 在正文层那一段已经算过一次(为了判断「这一轮到底
|
|
4644
|
+
* 有没有变化」),这里重算一遍是因为卡片清单可能被 stage 2 之后的写入改过 ——
|
|
4645
|
+
* 两处调的是同一个纯函数,所以不可能给出两种格式。
|
|
4646
|
+
*/
|
|
4647
|
+
await this.writeWikiIndex({ cards: this.cards, related: finalize.related, deadLinks: finalize.deadLinks })
|
|
2900
4648
|
// 关键词索引跟着 notes 一起落盘(0031):它的输入就是 notes,两者必须同时有效,
|
|
2901
4649
|
// 否则会出现「卡片改了、关键词路还按旧词表召回」。它不依赖编码器,所以 wiki-only
|
|
2902
4650
|
// 重建也要重建它 —— 而向量是 stage 3,仍然只在 `embeddings`/`all` 时重算。
|
|
@@ -2906,7 +4654,7 @@ export class KnowledgeRuntime {
|
|
|
2906
4654
|
// re-encode of the whole corpus.
|
|
2907
4655
|
const embeddingResult = only === 'all' ? await this.buildVectorIndex(this.units, version) : { status: 'skipped' as const, count: 0, reused: 0 }
|
|
2908
4656
|
const compileRunId = `rebuild_${hash(`${startedAt}:${this.cards.length}`)}`
|
|
2909
|
-
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version, builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: embeddingResult, fts: ftsResult, compileRunId, diagnosisHash, diagnosisGraph: { nodes: diagnosis.nodes.length, edges: diagnosis.edges.length }, lastBuild: { only, incremental: cardsReused > 0, skipped: false, cardsReused, cardsRecompiled, cardsAdded, cardsRemoved: removedCount, notesReused, notesRecompiled, ...extractionCounts, relationsExtracted, relationsDropped }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
4657
|
+
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version, builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: embeddingResult, fts: ftsResult, compileRunId, diagnosisHash, indexHash, diagnosisGraph: { nodes: diagnosis.nodes.length, edges: diagnosis.edges.length }, lastBuild: { only, incremental: cardsReused > 0, skipped: false, cardsReused, cardsRecompiled, cardsAdded, cardsRemoved: removedCount, notesReused, notesRecompiled, ...extractionCounts, ...finalizeCounts, relationsExtracted, relationsDropped }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
2910
4658
|
const report: RebuildReport = {
|
|
2911
4659
|
runId: compileRunId, compileRunId, version, only,
|
|
2912
4660
|
cards: this.cards.length, units: this.units.length, notes: this.units.length,
|
|
@@ -2920,7 +4668,7 @@ export class KnowledgeRuntime {
|
|
|
2920
4668
|
notesReused, notesRecompiled,
|
|
2921
4669
|
incremental: cardsReused > 0 || notesReused > 0,
|
|
2922
4670
|
skipped: false,
|
|
2923
|
-
...extractionCounts, relationsExtracted, relationsDropped,
|
|
4671
|
+
...extractionCounts, ...finalizeCounts, relationsExtracted, relationsDropped,
|
|
2924
4672
|
...(lineage ? { lineage } : {}), startedAt, finishedAt: new Date().toISOString(),
|
|
2925
4673
|
}
|
|
2926
4674
|
if (embeddingResult.status === 'unavailable') warnings.push(`embedding 不可用,已保留 FTS fallback:${embeddingResult.error ?? 'unknown error'}`)
|
|
@@ -2944,8 +4692,8 @@ export class KnowledgeRuntime {
|
|
|
2944
4692
|
vectorQuantization: embeddingResult.status === 'ready' ? VECTOR_QUANTIZATION : undefined,
|
|
2945
4693
|
embeddingError: embeddingResult.error,
|
|
2946
4694
|
incremental: embeddingResult.reused > 0, skipped: false,
|
|
2947
|
-
// `embeddings` 不跑 Pass 0
|
|
2948
|
-
...extractionCounts, relationsExtracted: 0, relationsDropped: 0,
|
|
4695
|
+
// `embeddings` 不跑 Pass 0,也不跑收尾段,所以这些计数是真的 0,不是省略。
|
|
4696
|
+
...extractionCounts, ...finalizeCounts, relationsExtracted: 0, relationsDropped: 0,
|
|
2949
4697
|
...(lineage ? { lineage } : {}), startedAt, finishedAt: new Date().toISOString(),
|
|
2950
4698
|
}
|
|
2951
4699
|
if (embeddingResult.status === 'unavailable') warnings.push(`embedding 不可用,已保留 FTS fallback:${embeddingResult.error ?? 'unknown error'}`)
|
|
@@ -3015,12 +4763,12 @@ export class KnowledgeRuntime {
|
|
|
3015
4763
|
}
|
|
3016
4764
|
|
|
3017
4765
|
/** 关键词路:FTS5 + BM25(0031)。索引不在时如实降级,不退回旧的计数打分。 */
|
|
3018
|
-
private async ftsRoute(query: string, eligible: WikiNote[], pool: number, threshold: number, titleOf: (sourceId: string) => string | undefined): Promise<{ ranking: RouteRanking[]; diagnostic: RecallRouteDiagnostic }> {
|
|
4766
|
+
private async ftsRoute(query: string, eligible: WikiNote[], pool: number, threshold: number, titleOf: (sourceId: string) => string | undefined): Promise<{ ranking: RouteRanking[]; rows: Array<RouteRanking & { kept: boolean }>; diagnostic: RecallRouteDiagnostic }> {
|
|
3019
4767
|
const eligibleIds = new Set(eligible.map(note => note.unitId))
|
|
3020
4768
|
const sqlite = await loadSqlite()
|
|
3021
|
-
if (!sqlite) return { ranking: [], diagnostic: { route: 'fts', status: 'degraded', reason: '当前 Node 没有 node:sqlite(需要 22.5+),关键词路已降级' } }
|
|
4769
|
+
if (!sqlite) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '当前 Node 没有 node:sqlite(需要 22.5+),关键词路已降级' } }
|
|
3022
4770
|
const index = await FtsIndex.open(this.safePath('03_compiled', FTS_FILE), sqlite.DatabaseSync)
|
|
3023
|
-
if (!index) return { ranking: [], diagnostic: { route: 'fts', status: 'degraded', reason: `关键词索引不存在(03_compiled/${FTS_FILE}),先编译一次` } }
|
|
4771
|
+
if (!index) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: `关键词索引不存在(03_compiled/${FTS_FILE}),先编译一次` } }
|
|
3024
4772
|
try {
|
|
3025
4773
|
const found = index.search(query, pool).filter(hit => eligibleIds.has(hit.unitId))
|
|
3026
4774
|
/**
|
|
@@ -3043,9 +4791,10 @@ export class KnowledgeRuntime {
|
|
|
3043
4791
|
// 单字查询(如 `表`)是刻意收窄的问法,没有「多字词覆盖率」可言:不拦它,
|
|
3044
4792
|
// 交给阈值。这里如实返回空排名之外的结果需要另走一条路,所以直接用全部命中。
|
|
3045
4793
|
const ranking = found.map(hit => ({ unitId: hit.unitId, score: hit.score }))
|
|
3046
|
-
const { kept, dropped, topScore } = applyThreshold(ranking, threshold)
|
|
4794
|
+
const { kept, dropped, topScore, rows } = applyThreshold(ranking, threshold)
|
|
3047
4795
|
return {
|
|
3048
4796
|
ranking: kept,
|
|
4797
|
+
rows,
|
|
3049
4798
|
diagnostic: {
|
|
3050
4799
|
route: 'fts', status: 'ran', candidates: ranking.length, kept: kept.length, topScore, threshold,
|
|
3051
4800
|
...(dropped > 0 ? { reason: `${dropped} 条低于关键词阈值 ${threshold} 被丢弃` } : {}),
|
|
@@ -3062,13 +4811,14 @@ export class KnowledgeRuntime {
|
|
|
3062
4811
|
})
|
|
3063
4812
|
const covered = scored.filter(hit => hit.coverage >= MIN_QUERY_COVERAGE)
|
|
3064
4813
|
const ranking = covered.map(hit => ({ unitId: hit.unitId, score: hit.score }))
|
|
3065
|
-
const { kept, dropped, topScore } = applyThreshold(ranking, threshold)
|
|
4814
|
+
const { kept, dropped, topScore, rows } = applyThreshold(ranking, threshold)
|
|
3066
4815
|
const lowCoverage = scored.length - covered.length
|
|
3067
4816
|
const reasons: string[] = []
|
|
3068
4817
|
if (lowCoverage > 0) reasons.push(`${lowCoverage} 条查询词覆盖率低于 ${MIN_QUERY_COVERAGE} 被丢弃`)
|
|
3069
4818
|
if (dropped > 0) reasons.push(`${dropped} 条低于关键词阈值 ${threshold} 被丢弃`)
|
|
3070
4819
|
return {
|
|
3071
4820
|
ranking: kept,
|
|
4821
|
+
rows,
|
|
3072
4822
|
diagnostic: {
|
|
3073
4823
|
route: 'fts', status: 'ran', candidates: ranking.length, kept: kept.length, topScore, threshold,
|
|
3074
4824
|
...(reasons.length > 0 ? { reason: reasons.join(';') } : {}),
|
|
@@ -3086,16 +4836,18 @@ export class KnowledgeRuntime {
|
|
|
3086
4836
|
* 阈值就完全失去意义(余弦 0 会被映射成 0.5,轻松过线)。原始余弦是绝对量,可以直接
|
|
3087
4837
|
* 用来判「这段内容和问题到底像不像」—— 弃答主要靠它和重排阈值。
|
|
3088
4838
|
*/
|
|
3089
|
-
private async vectorRoute(query: string, eligibleIds: Set<string>, pool: number, threshold: number): Promise<{ ranking: RouteRanking[]; diagnostic: RecallRouteDiagnostic; health: { ok: boolean; reason?: string } }> {
|
|
4839
|
+
private async vectorRoute(query: string, eligibleIds: Set<string>, pool: number, threshold: number): Promise<{ ranking: RouteRanking[]; rows: Array<RouteRanking & { kept: boolean }>; diagnostic: RecallRouteDiagnostic; health: { ok: boolean; reason?: string }; timings?: { embedMs: number; scanMs: number } }> {
|
|
3090
4840
|
const health = await this.checkEncoder()
|
|
3091
|
-
if (!health.ok) return { ranking: [], diagnostic: { route: 'vector', status: 'degraded', reason: health.reason }, health }
|
|
4841
|
+
if (!health.ok) return { ranking: [], rows: [], diagnostic: { route: 'vector', status: 'degraded', reason: health.reason }, health }
|
|
3092
4842
|
try {
|
|
3093
|
-
const scores = await this.vectorRecall(query, eligibleIds, pool)
|
|
4843
|
+
const { scores, embedMs, scanMs } = await this.vectorRecall(query, eligibleIds, pool)
|
|
3094
4844
|
const ranking = [...scores.entries()].map(([unitId, score]) => ({ unitId, score })).sort((left, right) => right.score - left.score || left.unitId.localeCompare(right.unitId))
|
|
3095
|
-
const { kept, dropped, topScore } = applyThreshold(ranking, threshold)
|
|
4845
|
+
const { kept, dropped, topScore, rows } = applyThreshold(ranking, threshold)
|
|
3096
4846
|
return {
|
|
3097
4847
|
ranking: kept,
|
|
4848
|
+
rows,
|
|
3098
4849
|
health,
|
|
4850
|
+
timings: { embedMs, scanMs },
|
|
3099
4851
|
diagnostic: {
|
|
3100
4852
|
route: 'vector', status: 'ran', candidates: ranking.length, kept: kept.length, topScore, threshold,
|
|
3101
4853
|
...(dropped > 0 ? { reason: `${dropped} 条低于向量阈值 ${threshold} 被丢弃` } : {}),
|
|
@@ -3104,7 +4856,7 @@ export class KnowledgeRuntime {
|
|
|
3104
4856
|
} catch (cause) {
|
|
3105
4857
|
const error = cause instanceof Error ? cause.message : String(cause)
|
|
3106
4858
|
this.vectorError = error
|
|
3107
|
-
return { ranking: [], diagnostic: { route: 'vector', status: 'degraded', reason: `向量召回不可用:${error}` }, health }
|
|
4859
|
+
return { ranking: [], rows: [], diagnostic: { route: 'vector', status: 'degraded', reason: `向量召回不可用:${error}` }, health }
|
|
3108
4860
|
}
|
|
3109
4861
|
}
|
|
3110
4862
|
|
|
@@ -3275,9 +5027,14 @@ export class KnowledgeRuntime {
|
|
|
3275
5027
|
includeProblems = false,
|
|
3276
5028
|
mode?: RecallMode,
|
|
3277
5029
|
lineage?: LineageContext,
|
|
3278
|
-
options: { contextTopN?: number; rerank?: boolean } = {},
|
|
5030
|
+
options: { contextTopN?: number; rerank?: boolean; contextMaxChars?: number; detail?: boolean } = {},
|
|
3279
5031
|
): Promise<Envelope<RecallResult>> {
|
|
3280
5032
|
const recallId = generatedId('recall')
|
|
5033
|
+
/**
|
|
5034
|
+
* 各阶段耗时(0050 第二版)。用 `performance.now()` 而不是 `Date.now()`:
|
|
5035
|
+
* 向量扫描这一段本身就在毫秒以下,用墙钟分辨率量不出东西来。
|
|
5036
|
+
*/
|
|
5037
|
+
const startedAt = performance.now()
|
|
3281
5038
|
const recallMode = mode ?? this.config.recallMode
|
|
3282
5039
|
const recallTopK = limit ?? this.config.recallTopK
|
|
3283
5040
|
const requestedTopN = options.contextTopN ?? this.config.contextTopN
|
|
@@ -3333,37 +5090,70 @@ export class KnowledgeRuntime {
|
|
|
3333
5090
|
const useVector = recallMode !== 'fts'
|
|
3334
5091
|
const useGraphBoost = recallMode === 'hybrid' || recallMode === 'graph'
|
|
3335
5092
|
|
|
5093
|
+
const ftsStarted = performance.now()
|
|
3336
5094
|
const ftsOutcome = useFts
|
|
3337
5095
|
? await this.ftsRoute(query, eligible, candidatePool, this.config.keywordThreshold, titleOf)
|
|
3338
|
-
: { ranking: [] as RouteRanking[], diagnostic: { route: 'fts' as const, status: 'skipped' as const, reason: '当前模式未启用关键词路' } }
|
|
5096
|
+
: { ranking: [] as RouteRanking[], rows: [] as Array<RouteRanking & { kept: boolean }>, diagnostic: { route: 'fts' as const, status: 'skipped' as const, reason: '当前模式未启用关键词路' } }
|
|
5097
|
+
const ftsMs = performance.now() - ftsStarted
|
|
5098
|
+
const vectorStarted = performance.now()
|
|
3339
5099
|
const vectorOutcome = useVector
|
|
3340
5100
|
? await this.vectorRoute(query, eligibleIds, candidatePool, this.config.vectorThreshold)
|
|
3341
|
-
: { ranking: [] as RouteRanking[], diagnostic: { route: 'vector' as const, status: 'skipped' as const, reason: '当前模式未启用向量路' }, health: { ok: false, reason: '当前模式未启用向量路' } }
|
|
5101
|
+
: { ranking: [] as RouteRanking[], rows: [] as Array<RouteRanking & { kept: boolean }>, diagnostic: { route: 'vector' as const, status: 'skipped' as const, reason: '当前模式未启用向量路' }, health: { ok: false, reason: '当前模式未启用向量路' } }
|
|
5102
|
+
const vectorMs = performance.now() - vectorStarted
|
|
5103
|
+
const graphStarted = performance.now()
|
|
3342
5104
|
const graphRanking = useGraphBoost ? await this.graphBoostRanking(query, eligible, includeProblems) : []
|
|
5105
|
+
const graphMs = performance.now() - graphStarted
|
|
3343
5106
|
|
|
3344
5107
|
if (ftsOutcome.diagnostic.status === 'degraded' && ftsOutcome.diagnostic.reason) warnings.push(ftsOutcome.diagnostic.reason)
|
|
3345
5108
|
if (vectorOutcome.diagnostic.status === 'degraded' && vectorOutcome.diagnostic.reason) warnings.push(vectorOutcome.diagnostic.reason)
|
|
3346
5109
|
|
|
3347
5110
|
// --- R2 · 名次级融合(RRF)+ 图谱 boost ---------------------------------
|
|
5111
|
+
const fuseStarted = performance.now()
|
|
3348
5112
|
const fused = rrfFuse({ fts: ftsOutcome.ranking, vector: vectorOutcome.ranking }, { k: RRF_K, weights: RRF_WEIGHTS })
|
|
3349
5113
|
const boosted = graphBoost(fused, graphRanking, { k: RRF_K, weights: RRF_WEIGHTS })
|
|
3350
5114
|
|
|
3351
5115
|
const unitById = new Map(eligible.map(unit => [unit.unitId, unit]))
|
|
5116
|
+
/**
|
|
5117
|
+
* 0044:review 评分在这里接进排序。
|
|
5118
|
+
*
|
|
5119
|
+
* 它**只改排序键,不改 `score` 字段本身** —— 命中里那个 `score` 仍然是融合分/重排分,
|
|
5120
|
+
* 是「这一路觉得它多相关」的原始答案。把评分乘进去再写回 `score`,会让读的人以为
|
|
5121
|
+
* 相关性变了;而它变的只是「同类命中里先看谁」。
|
|
5122
|
+
*
|
|
5123
|
+
* 生效点有两处,缺一处就会出现「review 了但排序没动」:
|
|
5124
|
+
* 1. 重排前的候选顺序(决定哪些条目进入重排的视线);
|
|
5125
|
+
* 2. 重排后的截断顺序(`recallTopK`)。
|
|
5126
|
+
*
|
|
5127
|
+
* 0050 之前还有第三处(上下文选择的 MMR 相关性函数)。现在上下文按**卡片**去重、
|
|
5128
|
+
* 顺序直接沿用 `hits`(`notesByWiki` 的顺序就是它),所以那第三处自然消失了 ——
|
|
5129
|
+
* 不是被删掉,而是不再需要单独算一遍。
|
|
5130
|
+
*/
|
|
5131
|
+
const rankKey = (hit: RecallHit): number => (hit.rerankScore ?? hit.score) * reviewWeight(hit.reviewScore)
|
|
3352
5132
|
let hits: RecallHit[] = boosted.hits
|
|
3353
5133
|
.filter(hit => unitById.has(hit.unitId))
|
|
3354
5134
|
.map(hit => ({ ...unitById.get(hit.unitId)!, score: hit.score, match: matchOf(hit.ranks), ranks: hit.ranks }))
|
|
5135
|
+
.sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId))
|
|
5136
|
+
|
|
5137
|
+
const fuseMs = performance.now() - fuseStarted
|
|
3355
5138
|
|
|
3356
5139
|
// --- R3 · 对**候选池**重排,再过阈值 ------------------------------------
|
|
3357
5140
|
let rerank: RerankResult
|
|
5141
|
+
const rerankStarted = performance.now()
|
|
5142
|
+
/** 逐条模型分(0050 的面板账本用):`kept` 只给活下来的,账本要全部。 */
|
|
5143
|
+
let rerankScored: Array<{ unitId: string; logit: number; rerankScore: number; kept: boolean }> = []
|
|
3358
5144
|
if (options.rerank === false || this.config.rerankProvider === 'none') {
|
|
3359
5145
|
rerank = { status: 'skipped', provider: this.config.rerankProvider }
|
|
3360
5146
|
hits = hits.slice(0, recallTopK)
|
|
3361
5147
|
} else {
|
|
3362
5148
|
const reranked = await this.rerankNotes(query, hits, this.config.rerankThreshold)
|
|
5149
|
+
rerankScored = reranked.scored ?? []
|
|
3363
5150
|
hits = reranked.hits.slice(0, recallTopK)
|
|
3364
5151
|
rerank = reranked.result
|
|
3365
5152
|
if (reranked.warning) warnings.push(reranked.warning)
|
|
3366
5153
|
}
|
|
5154
|
+
// 重排换的是相关性顺序,评分加权仍然要压在最上面 —— 否则「review 过的排前面一点」
|
|
5155
|
+
// 只在关掉重排时成立,而重排默认是开着的。
|
|
5156
|
+
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId))
|
|
3367
5157
|
|
|
3368
5158
|
// --- R4 · 证据片段 -----------------------------------------------------
|
|
3369
5159
|
await this.attachEvidence(hits, queryTerms, corpus)
|
|
@@ -3406,36 +5196,149 @@ export class KnowledgeRuntime {
|
|
|
3406
5196
|
}
|
|
3407
5197
|
|
|
3408
5198
|
// --- 组装:片段 → 整篇 → 证据 -------------------------------------------
|
|
5199
|
+
const assembleStarted = performance.now()
|
|
3409
5200
|
const cards = this.cards.length > 0 ? this.cards : await this.loadCards()
|
|
3410
5201
|
const cardByKey = new Map(cards.map(card => [`${card.entityType}:${card.entityId}`, card]))
|
|
3411
5202
|
const wikiKey = (note: Pick<RecallHit, 'entityType' | 'entityId'>) => `${note.entityType}:${note.entityId}`
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
5203
|
+
|
|
5204
|
+
/**
|
|
5205
|
+
* 按 **wiki 卡片**去重(0050 的第一半)。
|
|
5206
|
+
*
|
|
5207
|
+
* `hits` 已经按排序键降序,所以每张卡片第一次出现的那条就是它**最好的一条 note**
|
|
5208
|
+
* —— 那就是这张卡片的等级。同一张卡的其余小节挂在它下面(`notes`),
|
|
5209
|
+
* 不再各自占一个上下文名额。
|
|
5210
|
+
*/
|
|
5211
|
+
const notesByWiki = new Map<string, RecallHit[]>()
|
|
5212
|
+
for (const hit of hits) {
|
|
5213
|
+
const key = wikiKey(hit)
|
|
5214
|
+
const bucket = notesByWiki.get(key)
|
|
5215
|
+
if (bucket) bucket.push(hit)
|
|
5216
|
+
else notesByWiki.set(key, [hit])
|
|
5217
|
+
}
|
|
5218
|
+
/**
|
|
5219
|
+
* 命中的卡片,顺序 = 各自最好那条 note 的顺序。
|
|
5220
|
+
*
|
|
5221
|
+
* `contextTopN` 约束的是**这里**:进上下文的命中卡片数。1 跳扩展来的卡片另算
|
|
5222
|
+
* (它们受上下文预算与 `MAX_EXPANSION_CARDS` 约束)—— 扩展是「延伸」,
|
|
5223
|
+
* 不该挤掉「证据」的名额,也不该被 top_n 卡死。
|
|
5224
|
+
*/
|
|
5225
|
+
const hitCards = [...notesByWiki.entries()]
|
|
5226
|
+
.map(([key, notes]) => ({ key, notes, card: cardByKey.get(key) }))
|
|
5227
|
+
.slice(0, contextTopN)
|
|
5228
|
+
const wiki = hitCards.map(item => item.card).filter((card): card is KnowledgeCard => card !== undefined)
|
|
5229
|
+
const graph = this.oneHopGraph(new Set(hitCards.map(item => item.key)), includeProblems)
|
|
5230
|
+
|
|
5231
|
+
/**
|
|
5232
|
+
* **在去重之后**做 1 跳扩展(0050 的第二半)。
|
|
5233
|
+
*
|
|
5234
|
+
* 反过来(先扩展再去重)会把邻居与被命中的卡片混在一起,分不清「这是召回来的」
|
|
5235
|
+
* 还是「这是邻居」—— 而这两件事在上下文里必须能分开,前者是证据,后者是延伸。
|
|
5236
|
+
*/
|
|
5237
|
+
const expansion = this.expandWiki(hitCards.map(item => item.key), cardByKey, includeProblems)
|
|
5238
|
+
|
|
5239
|
+
/** 上下文条目:先命中的卡片(带它们的 note),再扩展来的卡片。 */
|
|
5240
|
+
const items: ContextItem[] = [
|
|
5241
|
+
...hitCards.map(item => ({
|
|
5242
|
+
wiki: item.card ?? null,
|
|
5243
|
+
notes: item.notes,
|
|
5244
|
+
note: item.notes[0]!,
|
|
5245
|
+
graph: this.oneHopGraph(new Set([item.key]), includeProblems),
|
|
5246
|
+
reason: 'hit' as const,
|
|
5247
|
+
...(item.notes[0]!.evidence ? { evidence: item.notes[0]!.evidence } : {}),
|
|
5248
|
+
})),
|
|
5249
|
+
...expansion.nodes.map(node => {
|
|
5250
|
+
const key = `${node.entityType}:${node.entityId}`
|
|
5251
|
+
const card = cardByKey.get(key) ?? null
|
|
5252
|
+
const [fromKey, relation] = (node.via[0] ?? '').split('|')
|
|
5253
|
+
const from = fromKey === undefined ? undefined : cardByKey.get(fromKey)
|
|
5254
|
+
return {
|
|
5255
|
+
wiki: card,
|
|
5256
|
+
notes: [],
|
|
5257
|
+
graph: this.oneHopGraph(new Set([key]), includeProblems),
|
|
5258
|
+
reason: 'expansion' as const,
|
|
5259
|
+
...(from === undefined ? {} : { via: { entityType: from.entityType, entityId: from.entityId, title: from.title, relation: relation ?? '' } }),
|
|
5260
|
+
}
|
|
5261
|
+
}),
|
|
5262
|
+
]
|
|
5263
|
+
|
|
5264
|
+
/**
|
|
5265
|
+
* 上下文预算(0050):装箱 + 溢出。
|
|
5266
|
+
*
|
|
5267
|
+
* 让位顺序写死在 `packContext` 里:先保证命中 note 的正文完整(它是证据),
|
|
5268
|
+
* 再给扩展卡片的标题 + 摘要 + 关系,最后才给扩展卡片的正文。
|
|
5269
|
+
*/
|
|
5270
|
+
/**
|
|
5271
|
+
* 每次调用可以覆盖预算(0050):指路行里就写着「读不到就调大预算重新召回」,
|
|
5272
|
+
* 所以那条出路必须真的存在 —— 只给一个设置项的话,Agent 改不了它。
|
|
5273
|
+
*/
|
|
5274
|
+
const maxChars = options.contextMaxChars ?? this.config.contextMaxChars
|
|
5275
|
+
const packed = await this.packContext({ items, maxChars, recallId })
|
|
5276
|
+
/** 哪些扩展卡片真的进了上下文(被预算挡住的也如实报出来)。 */
|
|
5277
|
+
expansion.included = packed.items
|
|
5278
|
+
.filter(item => item.reason === 'expansion' && item.wiki !== null)
|
|
5279
|
+
.map(item => `${item.wiki!.entityType}:${item.wiki!.entityId}`)
|
|
5280
|
+
if (packed.notice) warnings.push(packed.notice)
|
|
5281
|
+
|
|
5282
|
+
/**
|
|
5283
|
+
* 面板账本(0050):`detail: true` 才建。
|
|
5284
|
+
*
|
|
5285
|
+
* 它回答的是「这一条是怎么进来的、算了几分、有没有被丢」—— 而这三问分别对应
|
|
5286
|
+
* 关键词路 / 向量路 / 重排。工具调用不带它:几十行候选分数进模型上下文是纯噪声。
|
|
5287
|
+
*/
|
|
5288
|
+
const ledger: RecallLedger | undefined = options.detail === true
|
|
5289
|
+
? {
|
|
5290
|
+
fts: this.ledgerRows(ftsOutcome.rows, unitById),
|
|
5291
|
+
vector: this.ledgerRows(vectorOutcome.rows, unitById),
|
|
5292
|
+
rerank: (() => {
|
|
5293
|
+
const byUnit = new Map(rerankScored.map(row => [row.unitId, row]))
|
|
5294
|
+
const fusedByUnit = new Map(boosted.hits.map(hit => [hit.unitId, hit.score]))
|
|
5295
|
+
const order = rerankScored.length > 0 ? rerankScored.map(row => row.unitId) : hits.map(hit => hit.unitId)
|
|
5296
|
+
return order.flatMap((unitId, index) => {
|
|
5297
|
+
const note = unitById.get(unitId)
|
|
5298
|
+
if (note === undefined) return []
|
|
5299
|
+
const model = byUnit.get(unitId)
|
|
5300
|
+
return [{
|
|
5301
|
+
unitId,
|
|
5302
|
+
title: note.title,
|
|
5303
|
+
entityType: note.entityType,
|
|
5304
|
+
entityId: note.entityId,
|
|
5305
|
+
score: model?.rerankScore ?? fusedByUnit.get(unitId) ?? 0,
|
|
5306
|
+
rank: index + 1,
|
|
5307
|
+
kept: model?.kept ?? true,
|
|
5308
|
+
fused: fusedByUnit.get(unitId) ?? 0,
|
|
5309
|
+
...(model === undefined ? {} : { logit: model.logit, rerankScore: model.rerankScore }),
|
|
5310
|
+
// 没有模型分就是「重排没跑」或「这条没进重排」;有模型分而没 kept 就是被阈值丢了。
|
|
5311
|
+
dropped: model !== undefined && !model.kept,
|
|
5312
|
+
}]
|
|
5313
|
+
})
|
|
5314
|
+
})(),
|
|
5315
|
+
}
|
|
5316
|
+
: undefined
|
|
5317
|
+
|
|
5318
|
+
/** 每个数都是**这一段自己**的耗时(不是累计),量级差得远时一眼能看出是谁慢。 */
|
|
5319
|
+
const timings: RecallTimings = {
|
|
5320
|
+
ftsMs: round1(ftsMs),
|
|
5321
|
+
vectorMs: round1(vectorMs),
|
|
5322
|
+
...(vectorOutcome.timings === undefined ? {} : { embedMs: round1(vectorOutcome.timings.embedMs), scanMs: round1(vectorOutcome.timings.scanMs) }),
|
|
5323
|
+
graphMs: round1(graphMs),
|
|
5324
|
+
fuseMs: round1(fuseMs),
|
|
5325
|
+
...(rerank.status === 'ready' ? { rerankMs: round1(performance.now() - rerankStarted) } : {}),
|
|
5326
|
+
// 组装从「按卡片去重」算起(重排之后),所以它不含重排自己的时间。
|
|
5327
|
+
assembleMs: round1(performance.now() - assembleStarted),
|
|
5328
|
+
totalMs: round1(performance.now() - startedAt),
|
|
5329
|
+
}
|
|
3429
5330
|
|
|
3430
5331
|
const data: RecallResult = {
|
|
3431
|
-
hits, context, wiki, graph, backend, routes: ranRoutes, recallMode, rerank, recallTopK, candidatePool, contextTopN,
|
|
3432
|
-
insufficientEvidence, diagnostics, recallId,
|
|
5332
|
+
hits, context: packed.items, wiki, graph, backend, routes: ranRoutes, recallMode, rerank, recallTopK, candidatePool, contextTopN,
|
|
5333
|
+
insufficientEvidence, diagnostics, recallId, timings,
|
|
5334
|
+
expansion, budget: packed.report,
|
|
5335
|
+
...(ledger === undefined ? {} : { ledger }),
|
|
3433
5336
|
...(indexVersion === undefined ? {} : { indexVersion }),
|
|
3434
5337
|
...(compileRunId === undefined ? {} : { compileRunId }),
|
|
3435
5338
|
...(eventLineage ? { lineage: eventLineage } : {}),
|
|
3436
5339
|
}
|
|
3437
5340
|
await this.audit('knowledge.recall', {
|
|
3438
|
-
query: query.slice(0, 4000), count: hits.length, context:
|
|
5341
|
+
query: query.slice(0, 4000), count: hits.length, context: packed.items.length, backend, routes: ranRoutes,
|
|
3439
5342
|
rerank: rerank.status, rerankThreshold: this.config.rerankThreshold, candidatePool,
|
|
3440
5343
|
...(rerank.dropped === undefined ? {} : { rerankDropped: rerank.dropped }),
|
|
3441
5344
|
...(rerank.topLogit === undefined ? {} : { rerankTopLogit: Number(rerank.topLogit.toFixed(4)) }),
|
|
@@ -3457,11 +5360,16 @@ export class KnowledgeRuntime {
|
|
|
3457
5360
|
* 2. **量纲**:返回**原始余弦**,不再做 `(cos+1)/2` 的映射 —— 那个映射让 `0.15` 这类阈值
|
|
3458
5361
|
* 失去意义(余弦 0 → 0.5,轻松过线)。原始余弦是绝对量,可以直接用于弃答。
|
|
3459
5362
|
*/
|
|
3460
|
-
private async vectorRecall(query: string, eligibleIds: Set<string>, candidateLimit: number): Promise<Map<string, number
|
|
5363
|
+
private async vectorRecall(query: string, eligibleIds: Set<string>, candidateLimit: number): Promise<{ scores: Map<string, number>; embedMs: number; scanMs: number }> {
|
|
3461
5364
|
const index = await this.loadVectorIndex()
|
|
3462
5365
|
if (!index || index.manifest.itemCount === 0) throw new Error('VECTOR_INDEX_NOT_READY')
|
|
3463
5366
|
if (index.manifest.provider !== this.config.embeddingProvider || index.manifest.model !== this.embeddingModelId || index.manifest.dimension !== this.config.embeddingDimension) throw new Error('VECTOR_INDEX_CONFIG_MISMATCH')
|
|
5367
|
+
// 查询编码单独计时:模型没加载时这一段是 8 秒,加载之后是 0.3 秒 —— 这一段的
|
|
5368
|
+
// 数字就是「测试为什么慢」的答案本身,混进扫描时间里就看不出来了。
|
|
5369
|
+
const embedStarted = performance.now()
|
|
3464
5370
|
const encoded = await this.encodeEmbeddings([{ text: query }])
|
|
5371
|
+
const embedMs = performance.now() - embedStarted
|
|
5372
|
+
const scanStarted = performance.now()
|
|
3465
5373
|
const queryBinary = binaryVector(encoded.vectors[0]!, index.manifest.dimension)
|
|
3466
5374
|
const queryInt8 = int8Vector(encoded.vectors[0]!, index.manifest.dimension)
|
|
3467
5375
|
const binaryWidth = Math.ceil(index.manifest.dimension / 8)
|
|
@@ -3486,7 +5394,7 @@ export class KnowledgeRuntime {
|
|
|
3486
5394
|
for (let dimension = 0; dimension < index.manifest.dimension; dimension += 1) dot += signedByte(queryInt8[dimension]!) * signedByte(index.int8[offset + dimension]!)
|
|
3487
5395
|
scores.set(item.unitId, Math.max(-1, Math.min(1, dot / (127 * 127))))
|
|
3488
5396
|
}
|
|
3489
|
-
return scores
|
|
5397
|
+
return { scores, embedMs, scanMs: performance.now() - scanStarted }
|
|
3490
5398
|
}
|
|
3491
5399
|
|
|
3492
5400
|
/**
|
|
@@ -3556,7 +5464,7 @@ export class KnowledgeRuntime {
|
|
|
3556
5464
|
content,
|
|
3557
5465
|
// 索引命中不了时退回按 id + 分类算出来的路径:那条路径**就是**写入时用的,
|
|
3558
5466
|
// 所以它仍然指向真实文件(索引缺失只可能是并发写入还没失效缓存)。
|
|
3559
|
-
path: index.get(slug(source.sourceId)) ?? this.sourcePath(source.sourceId, source.sourceType),
|
|
5467
|
+
path: index.get(slug(source.sourceId)) ?? this.sourcePath(source.sourceId, source.sourceType, source.title),
|
|
3560
5468
|
returnedChars: content.length,
|
|
3561
5469
|
totalChars: source.content.length,
|
|
3562
5470
|
})
|