dsh-data-domain-knowledge 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +360 -0
- package/cordis.patch.yml +410 -0
- package/eval/README.md +76 -0
- package/eval/evaluate-retrieval.mjs +121 -0
- package/eval/queries.example.jsonl +2 -0
- package/lib/client.js +111415 -0
- package/lib/types/config-schema.d.ts +35 -0
- package/lib/types/config-schema.d.ts.map +1 -0
- package/lib/types/config-schema.js +34 -0
- package/lib/types/config-schema.js.map +1 -0
- package/lib/types/convert/core-loader.d.ts +32 -0
- package/lib/types/convert/core-loader.d.ts.map +1 -0
- package/lib/types/convert/core-loader.js +60 -0
- package/lib/types/convert/core-loader.js.map +1 -0
- package/lib/types/convert/host.d.ts +88 -0
- package/lib/types/convert/host.d.ts.map +1 -0
- package/lib/types/convert/host.js +479 -0
- package/lib/types/convert/host.js.map +1 -0
- package/lib/types/convert/ingest.d.ts +125 -0
- package/lib/types/convert/ingest.d.ts.map +1 -0
- package/lib/types/convert/ingest.js +228 -0
- package/lib/types/convert/ingest.js.map +1 -0
- package/lib/types/convert/settings-schema.d.ts +18 -0
- package/lib/types/convert/settings-schema.d.ts.map +1 -0
- package/lib/types/convert/settings-schema.js +26 -0
- package/lib/types/convert/settings-schema.js.map +1 -0
- package/lib/types/convert/types.d.ts +46 -0
- package/lib/types/convert/types.d.ts.map +1 -0
- package/lib/types/convert/types.js +41 -0
- package/lib/types/convert/types.js.map +1 -0
- package/lib/types/diagnosis/cards.d.ts +71 -0
- package/lib/types/diagnosis/cards.d.ts.map +1 -0
- package/lib/types/diagnosis/cards.js +482 -0
- package/lib/types/diagnosis/cards.js.map +1 -0
- package/lib/types/diagnosis/problem-classes.d.ts +51 -0
- package/lib/types/diagnosis/problem-classes.d.ts.map +1 -0
- package/lib/types/diagnosis/problem-classes.js +66 -0
- package/lib/types/diagnosis/problem-classes.js.map +1 -0
- package/lib/types/diagnosis/schema.d.ts +65 -0
- package/lib/types/diagnosis/schema.d.ts.map +1 -0
- package/lib/types/diagnosis/schema.js +208 -0
- package/lib/types/diagnosis/schema.js.map +1 -0
- package/lib/types/diagnosis/seed-source.d.ts +20 -0
- package/lib/types/diagnosis/seed-source.d.ts.map +1 -0
- package/lib/types/diagnosis/seed-source.js +96 -0
- package/lib/types/diagnosis/seed-source.js.map +1 -0
- package/lib/types/diagnosis/seed.d.ts +50 -0
- package/lib/types/diagnosis/seed.d.ts.map +1 -0
- package/lib/types/diagnosis/seed.js +242 -0
- package/lib/types/diagnosis/seed.js.map +1 -0
- package/lib/types/diagnosis/store.d.ts +81 -0
- package/lib/types/diagnosis/store.d.ts.map +1 -0
- package/lib/types/diagnosis/store.js +409 -0
- package/lib/types/diagnosis/store.js.map +1 -0
- package/lib/types/diagnosis/types.d.ts +124 -0
- package/lib/types/diagnosis/types.d.ts.map +1 -0
- package/lib/types/diagnosis/types.js +13 -0
- package/lib/types/diagnosis/types.js.map +1 -0
- package/lib/types/directory-picker.d.ts +94 -0
- package/lib/types/directory-picker.d.ts.map +1 -0
- package/lib/types/directory-picker.js +110 -0
- package/lib/types/directory-picker.js.map +1 -0
- package/lib/types/extract-model.d.ts +76 -0
- package/lib/types/extract-model.d.ts.map +1 -0
- package/lib/types/extract-model.js +193 -0
- package/lib/types/extract-model.js.map +1 -0
- package/lib/types/extract.d.ts +121 -0
- package/lib/types/extract.d.ts.map +1 -0
- package/lib/types/extract.js +549 -0
- package/lib/types/extract.js.map +1 -0
- package/lib/types/fts.d.ts +96 -0
- package/lib/types/fts.d.ts.map +1 -0
- package/lib/types/fts.js +184 -0
- package/lib/types/fts.js.map +1 -0
- package/lib/types/index.d.ts +39 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +1112 -0
- package/lib/types/index.js.map +1 -0
- package/lib/types/intent/engine.d.ts +108 -0
- package/lib/types/intent/engine.d.ts.map +1 -0
- package/lib/types/intent/engine.js +501 -0
- package/lib/types/intent/engine.js.map +1 -0
- package/lib/types/intent/host.d.ts +88 -0
- package/lib/types/intent/host.d.ts.map +1 -0
- package/lib/types/intent/host.js +389 -0
- package/lib/types/intent/host.js.map +1 -0
- package/lib/types/intent/preset-tools.d.ts +37 -0
- package/lib/types/intent/preset-tools.d.ts.map +1 -0
- package/lib/types/intent/preset-tools.js +45 -0
- package/lib/types/intent/preset-tools.js.map +1 -0
- package/lib/types/intent/settings-schema.d.ts +59 -0
- package/lib/types/intent/settings-schema.d.ts.map +1 -0
- package/lib/types/intent/settings-schema.js +57 -0
- package/lib/types/intent/settings-schema.js.map +1 -0
- package/lib/types/intent/store.d.ts +75 -0
- package/lib/types/intent/store.d.ts.map +1 -0
- package/lib/types/intent/store.js +139 -0
- package/lib/types/intent/store.js.map +1 -0
- package/lib/types/intent/tools.d.ts +63 -0
- package/lib/types/intent/tools.d.ts.map +1 -0
- package/lib/types/intent/tools.js +108 -0
- package/lib/types/intent/tools.js.map +1 -0
- package/lib/types/intent/types.d.ts +189 -0
- package/lib/types/intent/types.d.ts.map +1 -0
- package/lib/types/intent/types.js +62 -0
- package/lib/types/intent/types.js.map +1 -0
- package/lib/types/knowledge-bases.d.ts +266 -0
- package/lib/types/knowledge-bases.d.ts.map +1 -0
- package/lib/types/knowledge-bases.js +1107 -0
- package/lib/types/knowledge-bases.js.map +1 -0
- package/lib/types/layout.d.ts +262 -0
- package/lib/types/layout.d.ts.map +1 -0
- package/lib/types/layout.js +312 -0
- package/lib/types/layout.js.map +1 -0
- package/lib/types/migrate.d.ts +110 -0
- package/lib/types/migrate.d.ts.map +1 -0
- package/lib/types/migrate.js +280 -0
- package/lib/types/migrate.js.map +1 -0
- package/lib/types/recall-rank.d.ts +158 -0
- package/lib/types/recall-rank.d.ts.map +1 -0
- package/lib/types/recall-rank.js +261 -0
- package/lib/types/recall-rank.js.map +1 -0
- package/lib/types/runtime.d.ts +568 -0
- package/lib/types/runtime.d.ts.map +1 -0
- package/lib/types/runtime.js +3045 -0
- package/lib/types/runtime.js.map +1 -0
- package/lib/types/settings-schema.d.ts +117 -0
- package/lib/types/settings-schema.d.ts.map +1 -0
- package/lib/types/settings-schema.js +92 -0
- package/lib/types/settings-schema.js.map +1 -0
- package/lib/types/settings-volatile.d.ts +102 -0
- package/lib/types/settings-volatile.d.ts.map +1 -0
- package/lib/types/settings-volatile.js +114 -0
- package/lib/types/settings-volatile.js.map +1 -0
- package/lib/types/types.d.ts +799 -0
- package/lib/types/types.d.ts.map +1 -0
- package/lib/types/types.js +387 -0
- package/lib/types/types.js.map +1 -0
- package/lib/types/wiki-types.d.ts +68 -0
- package/lib/types/wiki-types.d.ts.map +1 -0
- package/lib/types/wiki-types.js +322 -0
- package/lib/types/wiki-types.js.map +1 -0
- package/package.json +101 -0
- package/preset/tita-data-agent/agent.cordis.yml +376 -0
- package/preset/tita-data-agent/preset.yml +3 -0
- package/python/requirements.txt +8 -0
- package/scripts/doctor.mjs +70 -0
- package/scripts/e2e-pipeline.mjs +355 -0
- package/scripts/install-agent-preset.mjs +92 -0
- package/scripts/setup-embedding.sh +34 -0
- package/scripts/sync-agent-preset.mjs +188 -0
- package/src/client/AgentPresetCard.tsx +247 -0
- package/src/client/DirectoryPicker.tsx +357 -0
- package/src/client/InfoHint.tsx +65 -0
- package/src/client/IntentLogicTab.tsx +185 -0
- package/src/client/IntentTab.tsx +715 -0
- package/src/client/KnowledgeBaseTab.tsx +1454 -0
- package/src/client/KnowledgeGraphView.tsx +306 -0
- package/src/client/KnowledgePanel.tsx +381 -0
- package/src/client/KnowledgeStepRail.tsx +60 -0
- package/src/client/NodeSettings.tsx +392 -0
- package/src/client/PreviewSections.tsx +116 -0
- package/src/client/RecallPanel.tsx +399 -0
- package/src/client/TypeDefinitionDrawer.tsx +215 -0
- package/src/client/WikiFlowCard.tsx +226 -0
- package/src/client/WikiTypeEditor.tsx +332 -0
- package/src/client/agent-preset.ts +121 -0
- package/src/client/arch/AgentArchitectureGraph.tsx +219 -0
- package/src/client/arch/NodeLogicPanel.tsx +133 -0
- package/src/client/arch/ProblemDetailPanel.tsx +88 -0
- package/src/client/arch/model.ts +691 -0
- package/src/client/architecture.css +191 -0
- package/src/client/architecture.tsx +530 -0
- package/src/client/attribution/AttributionGraph.tsx +104 -0
- package/src/client/attribution/AttributionTab.tsx +332 -0
- package/src/client/attribution/CaseView.tsx +102 -0
- package/src/client/attribution/GraphStage.tsx +87 -0
- package/src/client/attribution/RunView.tsx +184 -0
- package/src/client/attribution/StageChrome.tsx +403 -0
- package/src/client/attribution/ViewSwitch.tsx +107 -0
- package/src/client/attribution/graph-options.ts +381 -0
- package/src/client/attribution/model.ts +462 -0
- package/src/client/config-scope.ts +119 -0
- package/src/client/convert/ConvertPanel.tsx +594 -0
- package/src/client/convert/styles.css +138 -0
- package/src/client/convert/styles.ts +49 -0
- package/src/client/format.ts +39 -0
- package/src/client/graph/layout.ts +181 -0
- package/src/client/graph.css +113 -0
- package/src/client/index.ts +98 -0
- package/src/client/problem-domains.ts +114 -0
- package/src/client/styles.css +940 -0
- package/src/client/styles.ts +11 -0
- package/src/client/useFitScale.ts +68 -0
- package/src/config-schema.ts +45 -0
- package/src/convert/core-loader.ts +68 -0
- package/src/convert/host.ts +579 -0
- package/src/convert/ingest.ts +367 -0
- package/src/convert/settings-schema.ts +27 -0
- package/src/convert/types.ts +65 -0
- package/src/diagnosis/cards.ts +498 -0
- package/src/diagnosis/problem-classes.ts +117 -0
- package/src/diagnosis/schema.ts +260 -0
- package/src/diagnosis/seed-source.ts +107 -0
- package/src/diagnosis/seed.ts +291 -0
- package/src/diagnosis/store.ts +432 -0
- package/src/diagnosis/types.ts +152 -0
- package/src/directory-picker.ts +164 -0
- package/src/extract-model.ts +227 -0
- package/src/extract.ts +632 -0
- package/src/fts.ts +203 -0
- package/src/index.ts +1091 -0
- package/src/intent/README.md +63 -0
- package/src/intent/engine.ts +527 -0
- package/src/intent/host.ts +462 -0
- package/src/intent/preset-tools.ts +48 -0
- package/src/intent/settings-schema.ts +63 -0
- package/src/intent/store.ts +143 -0
- package/src/intent/tools.ts +139 -0
- package/src/intent/types.ts +245 -0
- package/src/knowledge-bases.ts +1160 -0
- package/src/layout.ts +350 -0
- package/src/migrate.ts +262 -0
- package/src/python/embedding_worker.py +471 -0
- package/src/recall-rank.ts +299 -0
- package/src/runtime.ts +3069 -0
- package/src/settings-schema.ts +94 -0
- package/src/settings-volatile.ts +122 -0
- package/src/types.ts +1140 -0
- package/src/wiki-types.ts +357 -0
package/src/runtime.ts
ADDED
|
@@ -0,0 +1,3069 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
|
3
|
+
import { existsSync } from 'node:fs'
|
|
4
|
+
import { appendFile, mkdir, readFile, readdir, rename, rm, rmdir, stat, unlink, writeFile } from 'node:fs/promises'
|
|
5
|
+
import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
|
6
|
+
import { fileURLToPath } from 'node:url'
|
|
7
|
+
import {
|
|
8
|
+
buildSourceTypeCatalog, defaultSourceTypes, DEFAULT_KNOWLEDGE_CONFIG, deriveSourceExt,
|
|
9
|
+
FALLBACK_EMBEDDING_MODEL, FALLBACK_RERANK_MODEL, legacyResourceHome, SOURCE_TYPE_ID_PATTERN,
|
|
10
|
+
LOCAL_EMBEDDING_MODEL_DIR, LOCAL_RERANK_MODEL_DIR,
|
|
11
|
+
type CardSchemaIssue, type ContextGraph, type ContextItem, type DocumentVersion, type Envelope,
|
|
12
|
+
type KnowledgeAssetRef, type KnowledgeCard, type KnowledgeConfig, type KnowledgeDocument,
|
|
13
|
+
type KnowledgeDocumentDetail, type KnowledgeError, type KnowledgeStatus, type KnowledgeUnit,
|
|
14
|
+
type RecallDiagnostics, type RecallEvidence, type RecallHit, type RecallMode, type RecallResult, type RecallRoute, type RecallRouteDiagnostic, type RebuildReport,
|
|
15
|
+
type RerankResult, type RuntimeKnowledgeConfig, type SchemaType, type SearchHit, type SourceRaw, type SourceRawRead, type SourceType,
|
|
16
|
+
type SourceTypeCatalog, type SourceTypeDef, type WikiCardVersion, type WikiNote,
|
|
17
|
+
type WikiSchemaMode, type WikiTypeCatalog, type WikiTypeDef,
|
|
18
|
+
} from './types.js'
|
|
19
|
+
import { defaultWikiTypes, validateCardAgainstType, WIKI_TYPE_RULES } from './wiki-types.js'
|
|
20
|
+
import { buildAliasIndex, candidateEntityId, extractCandidates, normalizeCandidates, RELATES_TO, type NormalizedCandidates } from './extract.js'
|
|
21
|
+
import { extractWithModel, type ModelCall } from './extract-model.js'
|
|
22
|
+
import { FTS_FILE, FtsIndex, loadSqlite, segment } from './fts.js'
|
|
23
|
+
import { applyRerank, applyThreshold, evidenceExcerpts, graphBoost, matchOf, noteIndexText, rrfFuse, selectContext, tokenSet, type RouteRanking } from './recall-rank.js'
|
|
24
|
+
import { legacyKnowledgeRoot } from './layout.js'
|
|
25
|
+
import { envelopeFailure, envelopeSuccess } from 'dsh-data-contracts/envelope'
|
|
26
|
+
import type { LineageContext, LineageRef } from 'dsh-data-contracts/refs'
|
|
27
|
+
|
|
28
|
+
// 05_diagnosis 是派生层(智能归因图谱):内容可由 02_wiki + 03_compiled 重建,
|
|
29
|
+
// 和 03_compiled 同性质,所以给独立编号段而不是塞进源头目录。
|
|
30
|
+
//
|
|
31
|
+
// `00_schema` 在 0030 被撤掉了:它从建出来那天起就是一个**空目录**。分类注册表
|
|
32
|
+
// 与 Wiki 类型注册表一直住在 `<agentRoot>/resource/knowledge-bases.json`(那是
|
|
33
|
+
// 配置,跟着根路径走,见 `knowledge-bases.ts`),而这个段里没有一行代码写过东西。
|
|
34
|
+
// 「磁盘上有一个文件夹」本身是一种承诺,承诺了却空着就是误导。
|
|
35
|
+
const LAYOUT = ['01_source_truth', '02_wiki', '03_compiled', '05_diagnosis'] as const
|
|
36
|
+
/**
|
|
37
|
+
* SourceRaw 段。段内**按来源分类再分一层**:`<段>/<source_type>/<docSlug>.md`。
|
|
38
|
+
*
|
|
39
|
+
* 平铺在真实使用里会先变成垃圾场:几十份来源混在一层,`ls` 读不出「我的指标口径在
|
|
40
|
+
* 哪」。而分类本来就是每份来源的必备元数据(frontmatter 的 `source_type`),把它
|
|
41
|
+
* 提到目录名上,是让「按分类找东西」在文件系统层面直接成立 —— 不依赖界面。
|
|
42
|
+
*
|
|
43
|
+
* 读路径不依赖这个层级:`sourceIndex()` 递归扫全段建 slug → path 索引,所以旧
|
|
44
|
+
* 平铺布局与迁移中间态都照常可读(见 `migrateSourceTruthLayout`)。
|
|
45
|
+
*/
|
|
46
|
+
const SOURCE_TRUTH = '01_source_truth'
|
|
47
|
+
/** Wiki 段;段内按 `entity_type` 分目录(与 SourceRaw 的分类分层同一个思路)。 */
|
|
48
|
+
const WIKI = '02_wiki'
|
|
49
|
+
const STATUS: KnowledgeStatus[] = ['unreviewed', 'draft', 'reviewed', 'active', 'deprecated', 'problem']
|
|
50
|
+
const SAFE_ID = /^[a-zA-Z0-9._:-]{1,160}$/
|
|
51
|
+
const VECTOR_MANIFEST = 'vector-manifest.json'
|
|
52
|
+
const VECTOR_BINARY = 'vectors.binary'
|
|
53
|
+
const VECTOR_INT8 = 'vectors.int8'
|
|
54
|
+
const VECTOR_QUANTIZATION = 'binary+hamming+int8' as const
|
|
55
|
+
/**
|
|
56
|
+
* 编码实现的指纹(0031)。
|
|
57
|
+
*
|
|
58
|
+
* 它与 `provider` / `model` 是**两件事**:同一个模型、同一份权重,换了「怎么调它」就足以
|
|
59
|
+
* 让已有向量全部作废 —— 0031 正是这种情况:修复前 WeMM 走 SentenceTransformers 包装路径,
|
|
60
|
+
* 对任何输入返回同一个向量;改成官方 processor 路径之后,同样的模型名会产出完全不同的向量。
|
|
61
|
+
*
|
|
62
|
+
* 没有它就会踩这个坑(实测):`rebuild all` 看到 44 条 note 的 contentHash 一个都没变,
|
|
63
|
+
* 于是把**旧的退化向量**原样复制过来,修复等于没发生 —— 而且构建还报 `reused: 44`,
|
|
64
|
+
* 读起来像「一切正常,没什么要重算的」。
|
|
65
|
+
*
|
|
66
|
+
* 所以:**改编码路径时必须一起改这个值**。
|
|
67
|
+
*/
|
|
68
|
+
const ENCODER_FINGERPRINT = 'wemm-official-processor-v1'
|
|
69
|
+
/** Stage 2 output consumed by stage 3 recall. */
|
|
70
|
+
const WIKI_NOTES = 'wiki_notes.json'
|
|
71
|
+
/** Pre-0014 artifact name; kept readable so an upgrade does not force a rebuild. */
|
|
72
|
+
const LEGACY_UNITS = 'knowledge_units.json'
|
|
73
|
+
/**
|
|
74
|
+
* SourceRaw version snapshots live under `04_memory/source-versions/<docSlug>/`.
|
|
75
|
+
* One file per historical revision, named `<compactIso>-<contentHash8>.md` and
|
|
76
|
+
* holding the *original markdown file bytes* so a rollback is byte-faithful.
|
|
77
|
+
*/
|
|
78
|
+
const SOURCE_VERSIONS = 'source-versions'
|
|
79
|
+
/**
|
|
80
|
+
* Wiki card version snapshots live under
|
|
81
|
+
* `04_memory/card-versions/<entityType>/<entityId>/`.
|
|
82
|
+
*
|
|
83
|
+
* The SourceRaw layer has had version history since 0018; the wiki layer — the
|
|
84
|
+
* thing an Agent actually recalls and the thing a compile run rewrites — had
|
|
85
|
+
* none until 0025. Two layers, one versioning model: same directory shape, same
|
|
86
|
+
* `<compactIso>-<hash8>.md` naming, same append-only rule, so "回滚本身也可回滚"
|
|
87
|
+
* holds on both sides without a second mechanism to learn.
|
|
88
|
+
*/
|
|
89
|
+
const CARD_VERSIONS = 'card-versions'
|
|
90
|
+
/**
|
|
91
|
+
* Pass 0 抽出的**关系**(0029)。
|
|
92
|
+
*
|
|
93
|
+
* 0025 起抽取是增量的:指纹没变的来源一次都不抽。但抽出来的关系原来只活在**本次
|
|
94
|
+
* 运行的内存里**(`ExtractionOutcome.relations`),没有任何持久化 —— 于是任何一次
|
|
95
|
+
* 全量重编(所有来源都被指纹跳过 → 关系为空)都会把图上**已经连上的 `relates_to`
|
|
96
|
+
* 边全部抹掉**,而报告写 `relationsExtracted=0 / dropped=0`,读起来像「本来就没有
|
|
97
|
+
* 关系」。实跑复现:`rebuild(wiki)` 得到 1 条 relates_to,紧接着 `rebuild(all)` 之后
|
|
98
|
+
* 归零且再也回不来。
|
|
99
|
+
*
|
|
100
|
+
* 所以关系按来源落一份产物:重编时**跳过的来源从这里读回它自己的关系**,新抽的
|
|
101
|
+
* 覆盖进去。它是派生数据(可由 SourceRaw 重算),所以放在 `03_compiled/`;删掉它
|
|
102
|
+
* 只会让下一次重建把关系重新抽一遍。
|
|
103
|
+
*/
|
|
104
|
+
const EXTRACTED_RELATIONS = 'extracted_relations.json'
|
|
105
|
+
/** Sections shorter than this stay in a single note; longer cards split by heading. */
|
|
106
|
+
const NOTE_SPLIT_MIN_CHARS = 800
|
|
107
|
+
/**
|
|
108
|
+
* 边类型 → 查询里可能用来指代它的词。
|
|
109
|
+
*
|
|
110
|
+
* 图召回早就支持「按关系类型命中」,但它只认英文写法:`supported_by` 要写成
|
|
111
|
+
* "supported by" 才命中,而中文提问不会这么写。Pass 0 新增的 `relates_to` 同理。
|
|
112
|
+
* 两种边都补上中文同义词,这条召回路径才不是只对英文有效。
|
|
113
|
+
*/
|
|
114
|
+
const RELATION_QUERY_WORDS: Record<string, string> = {
|
|
115
|
+
supported_by: 'supported by source 来源 出处 依据 支撑',
|
|
116
|
+
[RELATES_TO]: 'relates to related 相关 关联 关系 有关',
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* 融合与阈值的常数(0031)。
|
|
120
|
+
*
|
|
121
|
+
* `RRF_K` 与两路权重直接采用 WeKnora 的出厂值(k=60,向量 0.7 / 关键词 0.3)——
|
|
122
|
+
* 「不从零调参」比「我们自己的猜测」更可辩护,但两边的语料与模型不同,仍要按评测校准。
|
|
123
|
+
*/
|
|
124
|
+
const RRF_K = 60
|
|
125
|
+
const RRF_WEIGHTS = { fts: 0.3, vector: 0.7, graph: 0.2 } as const
|
|
126
|
+
/** 图谱只是 boost,不是路:它的权重只作用在**已经在候选池里**的条目上(见 `graphBoost`)。 */
|
|
127
|
+
const GRAPH_BOOST_WEIGHT = RRF_WEIGHTS.graph
|
|
128
|
+
/**
|
|
129
|
+
* 编码器健康门的判据:两段无关文本的余弦必须低于它。
|
|
130
|
+
*
|
|
131
|
+
* 0.99 不是一个「调出来的」阈值,而是一条分界线:真正编码不同文本的模型不会给出
|
|
132
|
+
* 0.99+ 的余弦,而 0030 那个坏掉的路径给的是 1.000000。
|
|
133
|
+
*/
|
|
134
|
+
const ENCODER_SELF_SIMILARITY_MAX = 0.99
|
|
135
|
+
/**
|
|
136
|
+
* 关键词路的**查询词覆盖率下限**(0031)。
|
|
137
|
+
*
|
|
138
|
+
* 为什么除了阈值还需要它:关键词的分是按本次最高分归一化的,所以「整批都弱」时最弱的
|
|
139
|
+
* 那条也会被归一成 1.0 并轻松过 `keywordThreshold` —— 相对阈值挡得住弱命中,挡不住
|
|
140
|
+
* 「只有弱命中」。实测:`完全无关的词汇zzz` 只蹭到一个「的」,归一化后照样过 0.3。
|
|
141
|
+
*
|
|
142
|
+
* 覆盖率是**绝对**的:命中里出现的查询词占查询全部词的比例。只蹭到一个高频单字 = 覆盖
|
|
143
|
+
* 率极低,直接淘汰;真正相关的段落会同时命中若干双字词,覆盖率自然上去。它是结构性
|
|
144
|
+
* 守卫(像「图谱不能制造候选」一样),不是需要用户调的分 —— 所以它是常数,不是设置项。
|
|
145
|
+
*/
|
|
146
|
+
const MIN_QUERY_COVERAGE = 0.25
|
|
147
|
+
/** 每条命中最多带几段证据原文。 */
|
|
148
|
+
const EVIDENCE_PER_HIT = 3
|
|
149
|
+
/** 同一张 wiki 最多几条进上下文:否则 5 条上下文可能全是同一张卡的不同小节。 */
|
|
150
|
+
const CONTEXT_PER_WIKI_CAP = 2
|
|
151
|
+
const BIT_COUNTS = Array.from({ length: 256 }, (_, value) => {
|
|
152
|
+
let count = 0
|
|
153
|
+
for (let bit = value; bit !== 0; bit &= bit - 1) count += 1
|
|
154
|
+
return count
|
|
155
|
+
})
|
|
156
|
+
|
|
157
|
+
interface VectorItem {
|
|
158
|
+
unitId: string
|
|
159
|
+
contentHash: string
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
interface VectorManifest {
|
|
163
|
+
version: number
|
|
164
|
+
createdAt: string
|
|
165
|
+
provider: KnowledgeConfig['embeddingProvider']
|
|
166
|
+
model: string
|
|
167
|
+
/**
|
|
168
|
+
* 编码实现的指纹(0031)。旧清单没有这个字段 → 视为不兼容,于是修复编码器之后
|
|
169
|
+
* 一定触发一次真正的重编码,而不是把旧向量按 contentHash 复用回来。
|
|
170
|
+
*/
|
|
171
|
+
encoder?: string
|
|
172
|
+
device: string
|
|
173
|
+
dimension: number
|
|
174
|
+
quantization: typeof VECTOR_QUANTIZATION
|
|
175
|
+
itemCount: number
|
|
176
|
+
items: VectorItem[]
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
interface LoadedVectorIndex {
|
|
180
|
+
manifest: VectorManifest
|
|
181
|
+
binary: Buffer
|
|
182
|
+
int8: Buffer
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
interface EmbeddingInput {
|
|
186
|
+
text: string
|
|
187
|
+
image?: string
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
interface EmbeddingBuildResult {
|
|
191
|
+
status: 'ready' | 'unavailable' | 'skipped'
|
|
192
|
+
count: number
|
|
193
|
+
reused: number
|
|
194
|
+
dimension?: number
|
|
195
|
+
device?: string
|
|
196
|
+
error?: string
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Pass 0(候选抽取 + 归一)一次运行的结果。
|
|
201
|
+
*
|
|
202
|
+
* 计数器在这里是**累加**出来的,不是事后算的:每个数都对应上面一次真实的
|
|
203
|
+
* `writeCard` 调用或一次指纹比较。关系先收集、等图装配时才知道能不能连上,
|
|
204
|
+
* 所以 `relationsExtracted` / `relationsDropped` 不在这里,而在 `applyExtractedRelations`。
|
|
205
|
+
*/
|
|
206
|
+
interface ExtractionOutcome {
|
|
207
|
+
extractionHashes: Record<string, string>
|
|
208
|
+
candidatesExtracted: number
|
|
209
|
+
candidatesRejected: number
|
|
210
|
+
cardsCreated: number
|
|
211
|
+
cardsMerged: number
|
|
212
|
+
sourcesExtracted: number
|
|
213
|
+
sourcesSkipped: number
|
|
214
|
+
relations: Array<{ fromKey: string; toKey: string; relation: string; sourceId: string }>
|
|
215
|
+
/**
|
|
216
|
+
* 按来源分组的关系。`relations` 是它的扁平视图(喂给 `applyExtractedRelations`),
|
|
217
|
+
* 这一份是**要落盘的形态** —— 跳过的来源下次重建靠它把关系读回来。
|
|
218
|
+
*/
|
|
219
|
+
relationsBySource: Record<string, Array<{ fromKey: string; toKey: string; relation: string; sourceId: string }>>
|
|
220
|
+
warnings: string[]
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Both helpers delegate to the shared envelope constructors: the five-key shape
|
|
225
|
+
* and the "warnings ⇒ partial" rule live in `dsh-data-contracts`,
|
|
226
|
+
* so a knowledge response and a workbook response cannot disagree about status,
|
|
227
|
+
* and neither can forget `error: null` on the happy path.
|
|
228
|
+
*/
|
|
229
|
+
export function success<T>(data: T, warnings: string[] = []): Envelope<T> { return envelopeSuccess<T, KnowledgeError>(data, warnings) }
|
|
230
|
+
export function failure<T>(data: T, code: string, message: string, retryable = true, suggestion?: string): Envelope<T> {
|
|
231
|
+
const error: KnowledgeError = { code, message, retryable, ...(suggestion ? { suggestion } : {}) }
|
|
232
|
+
return envelopeFailure<T, KnowledgeError>(data, error)
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function slug(value: string): string {
|
|
236
|
+
const normalized = value.trim().replace(/[^\p{L}\p{N}._-]+/gu, '-').replace(/^-+|-+$/g, '')
|
|
237
|
+
return normalized.slice(0, 120) || `item-${Date.now().toString(36)}`
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function hash(value: string): string { return createHash('sha256').update(value).digest('hex').slice(0, 24) }
|
|
241
|
+
|
|
242
|
+
/** The content hash used for `KnowledgeDocument.contentHash` and version ids. */
|
|
243
|
+
function contentHashOf(content: string): string { return hash(content) }
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* 把这次新抽到的正文**追加**进已有卡片的正文。
|
|
247
|
+
*
|
|
248
|
+
* 这是「wiki 编译第二条链路」的落点:实体已经在 wiki 里 → 在原卡片上追加;不在 →
|
|
249
|
+
* 新建一张。追加而不是覆盖,是因为覆盖会把别的来源贡献的内容删掉;而**丢弃**
|
|
250
|
+
* (第一版的行为)更糟 —— 来源改写了,卡片却永远停在第一次抽取的样子。
|
|
251
|
+
*
|
|
252
|
+
* 已经在正文里出现过的那一段不再追加:同一份来源重建、或同一段被两个候选命中时,
|
|
253
|
+
* 卡片不该越编译越长。判重按整段文本,所以它是幂等的。
|
|
254
|
+
*/
|
|
255
|
+
function appendExtracted(existing: string, additions: string[]): string {
|
|
256
|
+
let content = existing
|
|
257
|
+
for (const addition of additions) {
|
|
258
|
+
const body = addition.trim()
|
|
259
|
+
if (!body || content.includes(body)) continue
|
|
260
|
+
content = content ? `${content}\n\n${body}` : body
|
|
261
|
+
}
|
|
262
|
+
return content
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* `2026-01-02T03:04:05.123Z` → `20260102T030405123Z`. Millisecond precision keeps
|
|
267
|
+
* two overwrites inside the same second from colliding on the same version id.
|
|
268
|
+
*/
|
|
269
|
+
function compactIso(date: Date): string { return date.toISOString().replace(/[-:]/g, '').replace(/\.(\d{3})Z$/, '$1Z') }
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Inverse of `compactIso`. Version ids embed the timestamp, so a snapshot's age
|
|
273
|
+
* is readable from its filename alone — no stat() per row, and the ordering
|
|
274
|
+
* `cardVersionIds` already does by name is chronological as a side effect.
|
|
275
|
+
* It lives next to `compactIso` so the two formats cannot drift apart.
|
|
276
|
+
*/
|
|
277
|
+
function expandCompactIso(value: string): string {
|
|
278
|
+
const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(\d{3})Z$/.exec(value)
|
|
279
|
+
return match ? `${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}.${match[7]}Z` : ''
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function plainRecord(value: unknown): Record<string, unknown> {
|
|
283
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Declared source format of a document. Prefers a real extension from
|
|
288
|
+
* `metadata.source_path`; otherwise the upstream `source_format` label
|
|
289
|
+
* (`markdown` / `office` / …); otherwise `markdown`, the store's default shape.
|
|
290
|
+
*/
|
|
291
|
+
function sourceFormatOf(metadata: Record<string, unknown>): string {
|
|
292
|
+
const sourcePath = typeof metadata.source_path === 'string' ? metadata.source_path.trim() : ''
|
|
293
|
+
if (sourcePath) {
|
|
294
|
+
const base = sourcePath.split(/[\\/]/).pop() ?? ''
|
|
295
|
+
const dot = base.lastIndexOf('.')
|
|
296
|
+
if (dot > 0 && dot < base.length - 1) return base.slice(dot + 1).toLowerCase()
|
|
297
|
+
}
|
|
298
|
+
const format = typeof metadata.source_format === 'string' ? metadata.source_format.trim().toLowerCase() : ''
|
|
299
|
+
return format || 'markdown'
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** Failure payload for a document lookup that could not produce a detail. */
|
|
303
|
+
function emptyDocumentDetail(docId: string): KnowledgeDocumentDetail {
|
|
304
|
+
return {
|
|
305
|
+
docId, title: docId, sourceType: '', sourceTypeKnown: false, sourceFormat: '', parser: '', externalId: '', contentHash: '',
|
|
306
|
+
bytes: 0, assetCount: 0, createdAt: '', updatedAt: '', versionCount: 0,
|
|
307
|
+
relativePath: '', ext: '', content: '', frontmatter: {}, assets: [],
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function generatedId(prefix: string): string { return `${prefix}_${hash(`${Date.now()}:${Math.random()}`)}` }
|
|
312
|
+
|
|
313
|
+
function parseLineage(value: unknown): LineageContext | undefined {
|
|
314
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined
|
|
315
|
+
const record = value as Record<string, unknown>
|
|
316
|
+
const lineage: LineageContext = {}
|
|
317
|
+
for (const key of ['sessionId', 'turnId', 'intentId', 'taskId', 'subtaskId', 'parentSubtaskId', 'runId', 'traceId', 'domain'] as const) {
|
|
318
|
+
if (typeof record[key] === 'string' && record[key].trim()) lineage[key] = record[key].trim().slice(0, 160)
|
|
319
|
+
}
|
|
320
|
+
return Object.keys(lineage).length > 0 ? lineage : undefined
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function withRun(lineage: LineageContext | undefined, prefix: string): LineageContext | undefined {
|
|
324
|
+
if (lineage === undefined) return undefined
|
|
325
|
+
return { ...lineage, runId: lineage.runId ?? generatedId(prefix) }
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function parseProvenance(value: unknown): LineageRef[] | undefined {
|
|
329
|
+
if (!Array.isArray(value)) return undefined
|
|
330
|
+
const result = value.map(item => {
|
|
331
|
+
const lineage = parseLineage(item)
|
|
332
|
+
if (!lineage) return undefined
|
|
333
|
+
const record = item as Record<string, unknown>
|
|
334
|
+
return { ...lineage, ...(typeof record.role === 'string' ? { role: record.role as LineageRef['role'] } : {}), ...(typeof record.refType === 'string' ? { refType: record.refType } : {}), ...(typeof record.refId === 'string' ? { refId: record.refId } : {}) }
|
|
335
|
+
}).filter((item): item is LineageRef => item !== undefined)
|
|
336
|
+
return result.length > 0 ? result : undefined
|
|
337
|
+
}
|
|
338
|
+
function yamlValue(value: string): unknown {
|
|
339
|
+
const text = value.trim()
|
|
340
|
+
if (text === 'true') return true
|
|
341
|
+
if (text === 'false') return false
|
|
342
|
+
if (text === 'null') return null
|
|
343
|
+
if (/^-?\d+(\.\d+)?$/.test(text)) return Number(text)
|
|
344
|
+
if ((text.startsWith('[') && text.endsWith(']')) || (text.startsWith('{') && text.endsWith('}'))) {
|
|
345
|
+
try { return JSON.parse(text) } catch { /* keep string */ }
|
|
346
|
+
}
|
|
347
|
+
return text.replace(/^['"]|['"]$/g, '')
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
export function parseMarkdown(source: string): { frontmatter: Record<string, unknown>; content: string } {
|
|
351
|
+
if (!source.startsWith('---')) return { frontmatter: {}, content: source.trim() }
|
|
352
|
+
const end = source.indexOf('\n---', 3)
|
|
353
|
+
if (end < 0) return { frontmatter: {}, content: source.trim() }
|
|
354
|
+
const header = source.slice(3, end).replace(/^\n/, '')
|
|
355
|
+
const frontmatter: Record<string, unknown> = {}
|
|
356
|
+
for (const line of header.split(/\r?\n/)) {
|
|
357
|
+
const separator = line.indexOf(':')
|
|
358
|
+
if (separator <= 0) continue
|
|
359
|
+
const key = line.slice(0, separator).trim()
|
|
360
|
+
frontmatter[key] = yamlValue(line.slice(separator + 1))
|
|
361
|
+
}
|
|
362
|
+
return { frontmatter, content: source.slice(end + 4).trim() }
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
function markdown(frontmatter: Record<string, unknown>, content: string): string {
|
|
366
|
+
const lines = Object.entries(frontmatter).map(([key, value]) => `${key}: ${typeof value === 'string' ? value : JSON.stringify(value)}`)
|
|
367
|
+
return `---\n${lines.join('\n')}\n---\n\n${content.trim()}\n`
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
async function writeTextAtomic(path: string, content: string): Promise<void> {
|
|
371
|
+
// 原子写需要一个与目标同目录的临时文件,所以目标目录必须先存在。0030 起
|
|
372
|
+
// SourceRaw 落在 `<段>/<source_type>/` 下,而 `ensureLayout` 只建段目录 —— 少了
|
|
373
|
+
// 这一行,第一次写某一种分类就会 ENOENT(实测)。
|
|
374
|
+
await mkdir(dirname(path), { recursive: true })
|
|
375
|
+
const temporary = `${path}.${process.pid}.${Date.now().toString(36)}.tmp`
|
|
376
|
+
try {
|
|
377
|
+
await writeFile(temporary, content, 'utf8')
|
|
378
|
+
await rename(temporary, path)
|
|
379
|
+
} finally {
|
|
380
|
+
await unlinkIfExists(temporary)
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
async function writeBufferAtomic(path: string, content: Buffer): Promise<void> {
|
|
385
|
+
const temporary = `${path}.${process.pid}.${Date.now().toString(36)}.tmp`
|
|
386
|
+
try {
|
|
387
|
+
await writeFile(temporary, content)
|
|
388
|
+
await rename(temporary, path)
|
|
389
|
+
} finally {
|
|
390
|
+
await unlinkIfExists(temporary)
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
async function unlinkIfExists(path: string): Promise<void> {
|
|
395
|
+
try {
|
|
396
|
+
const { unlink } = await import('node:fs/promises')
|
|
397
|
+
await unlink(path)
|
|
398
|
+
} catch { /* rename already consumed the temp file */ }
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
async function filesUnder(root: string): Promise<string[]> {
|
|
402
|
+
try {
|
|
403
|
+
const entries = await readdir(root, { withFileTypes: true })
|
|
404
|
+
const result: string[] = []
|
|
405
|
+
for (const entry of entries) {
|
|
406
|
+
const path = join(root, entry.name)
|
|
407
|
+
if (entry.isDirectory()) result.push(...await filesUnder(path))
|
|
408
|
+
else if (entry.isFile() && extname(entry.name).toLowerCase() === '.md') result.push(path)
|
|
409
|
+
}
|
|
410
|
+
return result
|
|
411
|
+
} catch { return [] }
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
async function fileCount(root: string): Promise<number> {
|
|
415
|
+
try {
|
|
416
|
+
const entries = await readdir(root, { withFileTypes: true })
|
|
417
|
+
let count = 0
|
|
418
|
+
for (const entry of entries) count += entry.isDirectory() ? await fileCount(join(root, entry.name)) : entry.isFile() ? 1 : 0
|
|
419
|
+
return count
|
|
420
|
+
} catch { return 0 }
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
function tokens(text: string): string[] { return text.toLocaleLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter(token => token.length > 1) }
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* Stage 2: turn one wiki card into recall notes.
|
|
427
|
+
*
|
|
428
|
+
* Short cards stay a single note whose content is the historical
|
|
429
|
+
* `summary + content` payload, so existing recall results and qrels keep
|
|
430
|
+
* working. Long cards that carry `##`/`###` sections split on those headings
|
|
431
|
+
* and the summary rides along with the first section.
|
|
432
|
+
*/
|
|
433
|
+
export function splitWikiNotes(card: Pick<KnowledgeCard, 'summary' | 'content'>): Array<{ heading?: string; content: string }> {
|
|
434
|
+
const summary = card.summary?.trim()
|
|
435
|
+
const body = card.content.trim()
|
|
436
|
+
const combined = [summary, body].filter(Boolean).join('\n\n')
|
|
437
|
+
if (body.length <= NOTE_SPLIT_MIN_CHARS) return [{ content: combined }]
|
|
438
|
+
const sections = body.split(/\n(?=#{2,4}\s+\S)/).map(section => section.trim()).filter(Boolean)
|
|
439
|
+
// Keep a top-level preamble (`# title` and intro text) with the first
|
|
440
|
+
// section instead of emitting a contentless note for it.
|
|
441
|
+
const preamble = sections[0] !== undefined && !/^#{2,4}\s+\S/.test(sections[0]) ? sections.shift() : undefined
|
|
442
|
+
if (sections.length < 2) return [{ content: combined }]
|
|
443
|
+
return sections.map((section, index) => {
|
|
444
|
+
const heading = section.match(/^#{2,4}\s+(.+)$/m)?.[1]?.trim()
|
|
445
|
+
const content = [index === 0 ? summary : undefined, index === 0 ? preamble : undefined, section].filter(Boolean).join('\n\n')
|
|
446
|
+
return { ...(heading ? { heading } : {}), content }
|
|
447
|
+
})
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function markdownImageRefs(content: string): string[] {
|
|
451
|
+
const refs: string[] = []
|
|
452
|
+
const pattern = /!\[[^\]]*\]\(\s*(?:<([^>]+)>|([^\s)]+))/g
|
|
453
|
+
for (const match of content.matchAll(pattern)) {
|
|
454
|
+
const raw = (match[1] ?? match[2] ?? '').trim()
|
|
455
|
+
if (!raw || raw.startsWith('data:') || /^https?:\/\//i.test(raw) || raw.startsWith('#')) continue
|
|
456
|
+
try { refs.push(decodeURIComponent(raw)) } catch { /* malformed image URL; keep the markdown searchable as text */ }
|
|
457
|
+
}
|
|
458
|
+
return [...new Set(refs)]
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
function markdownImagePaths(content: string, baseDir: string): string[] {
|
|
462
|
+
return markdownImageRefs(content).map(value => isAbsolute(value) ? resolve(value) : resolve(baseDir, value))
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
async function imageFingerprints(paths: string[]): Promise<string[]> {
|
|
466
|
+
const result: string[] = []
|
|
467
|
+
for (const path of paths) {
|
|
468
|
+
try {
|
|
469
|
+
const details = await stat(path)
|
|
470
|
+
result.push(`${path}:${details.size}:${details.mtimeMs}`)
|
|
471
|
+
} catch {
|
|
472
|
+
result.push(`${path}:missing`)
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
return result
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
function binaryVector(vector: number[], dimension: number): Buffer {
|
|
479
|
+
const bytes = Buffer.alloc(Math.ceil(dimension / 8))
|
|
480
|
+
for (let index = 0; index < dimension; index += 1) if ((vector[index] ?? 0) >= 0) bytes[index >> 3] |= 1 << (index & 7)
|
|
481
|
+
return bytes
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
function int8Vector(vector: number[], dimension: number): Buffer {
|
|
485
|
+
const bytes = Buffer.alloc(dimension)
|
|
486
|
+
for (let index = 0; index < dimension; index += 1) {
|
|
487
|
+
const value = Math.max(-127, Math.min(127, Math.round((vector[index] ?? 0) * 127)))
|
|
488
|
+
bytes.writeInt8(value, index)
|
|
489
|
+
}
|
|
490
|
+
return bytes
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function signedByte(value: number): number { return value > 127 ? value - 256 : value }
|
|
494
|
+
|
|
495
|
+
function normaliseVectors(raw: unknown, dimension: number): number[][] {
|
|
496
|
+
if (!Array.isArray(raw)) throw new Error('EMBEDDING_RESPONSE_INVALID')
|
|
497
|
+
const vectors = raw.map(item => {
|
|
498
|
+
if (!Array.isArray(item) || item.length < dimension) throw new Error(`EMBEDDING_DIMENSION_INVALID: expected ${dimension}`)
|
|
499
|
+
const vector = item.slice(0, dimension).map(Number)
|
|
500
|
+
if (vector.some(value => !Number.isFinite(value))) throw new Error('EMBEDDING_VALUE_INVALID')
|
|
501
|
+
const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1
|
|
502
|
+
return vector.map(value => value / norm)
|
|
503
|
+
})
|
|
504
|
+
return vectors
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
interface PendingWorkerRequest {
|
|
508
|
+
resolve: (value: any) => void
|
|
509
|
+
reject: (reason: unknown) => void
|
|
510
|
+
timer: ReturnType<typeof setTimeout>
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/** Keep the WeMM process/model alive across rebuilds and search requests. */
|
|
514
|
+
class JsonLineWorker {
|
|
515
|
+
private child: ChildProcessWithoutNullStreams | undefined
|
|
516
|
+
private buffer = ''
|
|
517
|
+
private pending: PendingWorkerRequest[] = []
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* `label` names the subsystem in every failure code this worker can produce.
|
|
521
|
+
*
|
|
522
|
+
* The class is generic (`embedding_worker.py` serves both stage 2/3 embedding
|
|
523
|
+
* and stage 4 rerank), but its codes were hard-coded `EMBEDDING_*`. A rerank
|
|
524
|
+
* that timed out therefore reported `EMBEDDING_TIMEOUT` — pointing the operator
|
|
525
|
+
* at a subsystem that was working, while the reranker was the one that never
|
|
526
|
+
* loaded (measured 2026-09-22).
|
|
527
|
+
*/
|
|
528
|
+
constructor(private readonly command: string, private readonly args: string[], private readonly cwd: string, private readonly env: NodeJS.ProcessEnv, private readonly label = 'EMBEDDING') {}
|
|
529
|
+
|
|
530
|
+
private start(): void {
|
|
531
|
+
const child = spawn(this.command, this.args, { cwd: this.cwd, env: this.env, stdio: ['pipe', 'pipe', 'pipe'] })
|
|
532
|
+
this.child = child
|
|
533
|
+
child.stdout.setEncoding('utf8')
|
|
534
|
+
child.stderr.resume()
|
|
535
|
+
child.stdout.on('data', chunk => {
|
|
536
|
+
this.buffer += String(chunk)
|
|
537
|
+
const lines = this.buffer.split(/\r?\n/)
|
|
538
|
+
this.buffer = lines.pop() ?? ''
|
|
539
|
+
for (const line of lines) {
|
|
540
|
+
if (!line.trim()) continue
|
|
541
|
+
let result: any
|
|
542
|
+
try { result = JSON.parse(line) } catch { continue }
|
|
543
|
+
const request = this.pending.shift()
|
|
544
|
+
if (!request) continue
|
|
545
|
+
clearTimeout(request.timer)
|
|
546
|
+
if (!result?.ok) request.reject(new Error(String(result?.error ?? `${this.label}_WORKER_ERROR`)))
|
|
547
|
+
else request.resolve(result)
|
|
548
|
+
}
|
|
549
|
+
})
|
|
550
|
+
child.once('error', error => this.fail(error))
|
|
551
|
+
child.once('close', code => {
|
|
552
|
+
if (this.child !== child) return
|
|
553
|
+
this.child = undefined
|
|
554
|
+
this.fail(new Error(`${this.label}_WORKER_EXIT_${code ?? 'unknown'}`))
|
|
555
|
+
})
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
private fail(reason: unknown): void {
|
|
559
|
+
const child = this.child
|
|
560
|
+
this.child = undefined
|
|
561
|
+
if (child && !child.killed) child.kill('SIGTERM')
|
|
562
|
+
const requests = this.pending.splice(0)
|
|
563
|
+
for (const request of requests) {
|
|
564
|
+
clearTimeout(request.timer)
|
|
565
|
+
request.reject(reason)
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
request(payload: unknown, timeoutMs: number): Promise<any> {
|
|
570
|
+
if (!this.child) this.start()
|
|
571
|
+
return new Promise((resolveWorker, rejectWorker) => {
|
|
572
|
+
const request = { resolve: resolveWorker, reject: rejectWorker, timer: undefined as unknown as ReturnType<typeof setTimeout> }
|
|
573
|
+
request.timer = setTimeout(() => {
|
|
574
|
+
const index = this.pending.indexOf(request)
|
|
575
|
+
if (index >= 0) this.pending.splice(index, 1)
|
|
576
|
+
const timeout = new Error(`${this.label}_TIMEOUT`)
|
|
577
|
+
this.fail(timeout)
|
|
578
|
+
rejectWorker(timeout)
|
|
579
|
+
}, timeoutMs)
|
|
580
|
+
this.pending.push(request)
|
|
581
|
+
try {
|
|
582
|
+
this.child!.stdin.write(`${JSON.stringify(payload)}\n`)
|
|
583
|
+
} catch (error) {
|
|
584
|
+
const index = this.pending.indexOf(request)
|
|
585
|
+
if (index >= 0) this.pending.splice(index, 1)
|
|
586
|
+
clearTimeout(request.timer)
|
|
587
|
+
rejectWorker(error)
|
|
588
|
+
}
|
|
589
|
+
})
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
close(): void { this.fail(new Error(`${this.label}_WORKER_CLOSED`)) }
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* 来源分类注册表的读取入口。
|
|
597
|
+
*
|
|
598
|
+
* 做成 provider 而不是构造时快照:`KnowledgeBases` 缓存的是 runtime 而不是
|
|
599
|
+
* 注册表,用户在设置里改完分类必须**立刻**对 `add_source` 生效(0013 规范
|
|
600
|
+
* 第 5 节「新增后必须立刻对 Agent 可见」)。
|
|
601
|
+
*/
|
|
602
|
+
export type SourceTypeProvider = () => SourceTypeDef[]
|
|
603
|
+
|
|
604
|
+
/** Wiki 卡片类型注册表的读取入口(0024):类型定义 + schema 生效方式。 */
|
|
605
|
+
export interface WikiSchemaSnapshot {
|
|
606
|
+
types: WikiTypeDef[]
|
|
607
|
+
mode: WikiSchemaMode
|
|
608
|
+
}
|
|
609
|
+
export type WikiSchemaProvider = () => WikiSchemaSnapshot
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* 解析三个「家当」路径(0029)。
|
|
613
|
+
*
|
|
614
|
+
* 优先级:**配置里显式写的值**(由构造函数处理)> `<agentRoot>/resource/…`(存在时)
|
|
615
|
+
* > 旧家当目录 `~/.tita-data-agent/…`(存在时)> 兜底(`python3` / HF 模型名)。
|
|
616
|
+
*
|
|
617
|
+
* 旧目录留在链上,是为了让「还没搬家当」与「已经搬完」两种状态都能跑;搬家当这件事
|
|
618
|
+
* 本身不改变行为,只改变文件在哪。
|
|
619
|
+
*
|
|
620
|
+
* `modelId` 把本地模型的**身份**与它的位置解耦:两个已知位置都归一到
|
|
621
|
+
* `local:<模型目录名>`,所以把模型从旧目录搬进根路径**不会**让向量清单判成
|
|
622
|
+
* 「配置不匹配」—— 否则一次搬家会让整个索引作废、触发一次全量重编码。
|
|
623
|
+
*/
|
|
624
|
+
function resolveResources(resourceRoot: string | undefined): {
|
|
625
|
+
python: string
|
|
626
|
+
embeddingModel: string
|
|
627
|
+
rerankModel: string
|
|
628
|
+
modelId: (path: string) => string
|
|
629
|
+
} {
|
|
630
|
+
const home = legacyResourceHome()
|
|
631
|
+
const root = resourceRoot ? resolve(resourceRoot) : undefined
|
|
632
|
+
const firstExisting = (candidates: Array<string | undefined>, fallback: string): string => {
|
|
633
|
+
for (const candidate of candidates) if (candidate && existsSync(candidate)) return candidate
|
|
634
|
+
return fallback
|
|
635
|
+
}
|
|
636
|
+
const python = firstExisting([
|
|
637
|
+
root ? join(root, 'python', 'bin', 'python') : undefined,
|
|
638
|
+
join(home, 'embedding-venv', 'bin', 'python'),
|
|
639
|
+
], 'python3')
|
|
640
|
+
const embeddingModel = firstExisting([
|
|
641
|
+
root ? join(root, 'models', LOCAL_EMBEDDING_MODEL_DIR) : undefined,
|
|
642
|
+
join(home, 'models', LOCAL_EMBEDDING_MODEL_DIR),
|
|
643
|
+
], FALLBACK_EMBEDDING_MODEL)
|
|
644
|
+
const rerankModel = firstExisting([
|
|
645
|
+
root ? join(root, 'models', LOCAL_RERANK_MODEL_DIR) : undefined,
|
|
646
|
+
join(home, 'models', LOCAL_RERANK_MODEL_DIR),
|
|
647
|
+
], FALLBACK_RERANK_MODEL)
|
|
648
|
+
const modelId = (path: string): string => {
|
|
649
|
+
for (const base of [root, home]) {
|
|
650
|
+
if (!base || !path.startsWith(`${base}${sep}`)) continue
|
|
651
|
+
// `models/<目录名>` 与 `embedding-venv/...` 都去掉容器那一层,两个位置归一。
|
|
652
|
+
return `local:${relative(base, path).replace(/^models[\\/]/, '')}`
|
|
653
|
+
}
|
|
654
|
+
return path
|
|
655
|
+
}
|
|
656
|
+
return { python, embeddingModel, rerankModel, modelId }
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
export interface KnowledgeRuntimeOptions {
|
|
660
|
+
/** 注册表来源;省略时用出厂内置分类。 */
|
|
661
|
+
sourceTypes?: SourceTypeProvider
|
|
662
|
+
/** Wiki 卡片类型来源;省略时用出厂 11 个类型 + `report` 模式。 */
|
|
663
|
+
wikiSchema?: WikiSchemaProvider
|
|
664
|
+
/**
|
|
665
|
+
* 0029:家当目录(`<agentRoot>/resource`)。
|
|
666
|
+
*
|
|
667
|
+
* 配置里那三个路径字段(`python` / `embeddingModel` / `rerankModel`)为空时,
|
|
668
|
+
* 它们从这里解析。省略时退回旧的 `TITA_DATA_AGENT_HOME` —— 独立运行时与单元
|
|
669
|
+
* 测试不必先造一个根路径。
|
|
670
|
+
*/
|
|
671
|
+
resourceRoot?: string
|
|
672
|
+
/**
|
|
673
|
+
* 0032:这个知识库的记忆段(`<agentRoot>/memory/<知识库 id>`)。
|
|
674
|
+
*
|
|
675
|
+
* 版本快照、反馈与访问审计写到这里,**不再写进知识树**。省略时退回
|
|
676
|
+
* `<rootDir>/04_memory` —— 独立运行时与单元测试不必先造一个根路径,迁移期间
|
|
677
|
+
* 「还没搬」的知识库也照常能读写。
|
|
678
|
+
*/
|
|
679
|
+
memoryRoot?: string
|
|
680
|
+
/**
|
|
681
|
+
* #3b:模型调用接缝。
|
|
682
|
+
*
|
|
683
|
+
* 运行时自己**不认识** `ctx.llm` —— 那是插件层的事。把调用以函数形式注进来,
|
|
684
|
+
* 运行时才能在 `rebuild` 里用模型做实体抽取与归并(`extract-model.ts`),
|
|
685
|
+
* 而单元测试可以塞一个假模型进去,不必真的跑一次推理。
|
|
686
|
+
*
|
|
687
|
+
* 省略 = 退回纯规则抽取(Pass 0)。那条链路本来就不依赖模型,这条接缝的存在
|
|
688
|
+
* 不能让「没有模型就跑不动」成为新的前提。
|
|
689
|
+
*/
|
|
690
|
+
modelCall?: ModelCall
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
export class KnowledgeRuntime {
|
|
694
|
+
/** 当前生效的配置。volatile 改动会就地合并进来(见 `updateConfig`),所以它不是只读的。 */
|
|
695
|
+
config: RuntimeKnowledgeConfig
|
|
696
|
+
readonly rootDir: string
|
|
697
|
+
/**
|
|
698
|
+
* 来源分类注册表的来源(0013)。省略时用出厂内置分类——独立 runtime 与单元
|
|
699
|
+
* 测试是这个形态;`KnowledgeBases` 会注入「按库读取」的 provider,于是每个
|
|
700
|
+
* 知识库有自己的注册表,自定义分类立刻对 Agent 可见,不需要重启。
|
|
701
|
+
*/
|
|
702
|
+
private readonly sourceTypeProvider?: SourceTypeProvider
|
|
703
|
+
/** Wiki 卡片类型注册表的来源(0024);省略时用出厂 11 个类型 + `report` 模式。 */
|
|
704
|
+
private readonly wikiSchemaProvider?: WikiSchemaProvider
|
|
705
|
+
/**
|
|
706
|
+
* Id of the knowledge base this runtime serves. `KnowledgeBases` sets it when
|
|
707
|
+
* it creates/caches a runtime; asset URLs are built from it. Standalone
|
|
708
|
+
* runtimes keep the default so `/knowledge/assets/default/<rel>` still works.
|
|
709
|
+
*/
|
|
710
|
+
baseId = 'default'
|
|
711
|
+
private cards: KnowledgeCard[] = []
|
|
712
|
+
private units: KnowledgeUnit[] = []
|
|
713
|
+
private graph: { nodes: Array<Record<string, unknown>>; edges: Array<Record<string, unknown>> } = { nodes: [], edges: [] }
|
|
714
|
+
/**
|
|
715
|
+
* `docSlug → 绝对路径` 的 SourceRaw 索引(0030)。
|
|
716
|
+
*
|
|
717
|
+
* 有了它,文档 id 不需要携带来源分类也能定位文件 —— `add_source` 允许调用方自
|
|
718
|
+
* 己给 `source_id`,那串 id 里没有类型信息;而目录层级里有。索引递归覆盖整段,
|
|
719
|
+
* 所以旧平铺布局、新分类布局、以及两者并存的迁移中间态都能解析。
|
|
720
|
+
* 写入与删除后必须失效(见 `addSource` / `deleteDocument`)。
|
|
721
|
+
*/
|
|
722
|
+
private sourceIndex: Map<string, string> | undefined
|
|
723
|
+
/** 段内布局整理是否已经跑过(每个 runtime 一次)。 */
|
|
724
|
+
private segmentsOrganized = false
|
|
725
|
+
private vectorIndex: LoadedVectorIndex | null = null
|
|
726
|
+
private vectorError: string | undefined
|
|
727
|
+
private embeddingWorker: JsonLineWorker | null = null
|
|
728
|
+
/** Separate process so a rerank model failure cannot take embeddings down. */
|
|
729
|
+
private rerankWorker: JsonLineWorker | null = null
|
|
730
|
+
/** Cache the first rerank failure; do not respawn a doomed model per recall. */
|
|
731
|
+
private rerankUnavailable: string | undefined
|
|
732
|
+
/** 解析后的 Python 解释器路径(配置为空时从家当目录解析)。 */
|
|
733
|
+
private pythonPath: string
|
|
734
|
+
/** 解析后的嵌入模型:本地目录或 HF 模型名。 */
|
|
735
|
+
private embeddingModelPath: string
|
|
736
|
+
/** 解析后的重排模型。 */
|
|
737
|
+
private rerankModelPath: string
|
|
738
|
+
/**
|
|
739
|
+
* 向量清单里记的**模型身份**。
|
|
740
|
+
*
|
|
741
|
+
* 本地模型记 `resource:<相对路径>`,不记绝对路径 —— 否则把根路径搬一次,
|
|
742
|
+
* 清单里的模型名就对不上,整个向量索引被判成「配置不匹配」而作废,用户会莫名
|
|
743
|
+
* 收到一次全量重编码。清单该记「是哪一个模型」,不是「它在哪」。
|
|
744
|
+
*/
|
|
745
|
+
private readonly embeddingModelId: string
|
|
746
|
+
/**
|
|
747
|
+
* 这个知识库的记忆段(0032)。
|
|
748
|
+
*
|
|
749
|
+
* 它可以**在知识树之外**(正常情况就是 `<agentRoot>/memory/<库 id>`),所以
|
|
750
|
+
* 不能再走 `safePath` —— 那个守卫的契约是「不许跑出 rootDir」。这里另起一个
|
|
751
|
+
* `memoryPath`,守卫换成「不许跑出 memoryDir」。
|
|
752
|
+
*/
|
|
753
|
+
private readonly memoryDir: string
|
|
754
|
+
/** #3b 的模型调用接缝;undefined = 退回纯规则抽取。 */
|
|
755
|
+
private readonly modelCall: ModelCall | undefined
|
|
756
|
+
|
|
757
|
+
constructor(rootDir: string, config: Partial<RuntimeKnowledgeConfig> = {}, options: KnowledgeRuntimeOptions = {}) {
|
|
758
|
+
this.rootDir = resolve(rootDir)
|
|
759
|
+
// `config` 里没有路径字段:这个运行时的位置就是 `this.rootDir`,配置只带
|
|
760
|
+
// 「怎么算」的参数(embedding / rerank / topK…)。这样就不存在「配置说 A、
|
|
761
|
+
// 运行时实际在 B」这种两处说法。
|
|
762
|
+
this.config = { ...DEFAULT_KNOWLEDGE_CONFIG, ...config }
|
|
763
|
+
this.sourceTypeProvider = options.sourceTypes
|
|
764
|
+
this.wikiSchemaProvider = options.wikiSchema
|
|
765
|
+
this.memoryDir = options.memoryRoot ? resolve(options.memoryRoot) : join(this.rootDir, '04_memory')
|
|
766
|
+
this.modelCall = options.modelCall
|
|
767
|
+
const resources = resolveResources(options.resourceRoot)
|
|
768
|
+
this.pythonPath = config.python?.trim() || resources.python
|
|
769
|
+
this.embeddingModelPath = config.embeddingModel?.trim() || resources.embeddingModel
|
|
770
|
+
this.rerankModelPath = config.rerankModel?.trim() || resources.rerankModel
|
|
771
|
+
this.embeddingModelId = resources.modelId(this.embeddingModelPath)
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/**
|
|
775
|
+
* 就地应用一份新的运行时配置(0.1.7 的 volatile 改动走这条路)。
|
|
776
|
+
*
|
|
777
|
+
* 宿主改 volatile 字段时**不会重载插件**(`Entry._commitVolatile()` 只调
|
|
778
|
+
* `updateVolatile()` 就地改引用,再发一个 `loader/volatile-update`),所以「设置存下了
|
|
779
|
+
* 但没有生效」必须由这里兜住 —— 只标 volatile 不应用,就是假设置。
|
|
780
|
+
*
|
|
781
|
+
* 大部分字段是**每次调用现读** `this.config` 的(`recall` 读 `recallTopK`、
|
|
782
|
+
* `sourceTruth` 读 `allowSourceTruth`……),合并进去就生效。但有几类是**建好就缓存**
|
|
783
|
+
* 的,配置换了却还用旧的,那正是「看起来改了其实没改」:
|
|
784
|
+
*
|
|
785
|
+
* - 两个模型进程:解释器或模型路径变了,旧进程跑的还是旧模型;
|
|
786
|
+
* - 向量索引:维度或模型变了,旧向量与新查询不在同一个空间里,比出来的分没有意义。
|
|
787
|
+
*
|
|
788
|
+
* @param patch - 要合并进来的配置片段。
|
|
789
|
+
*/
|
|
790
|
+
updateConfig(patch: Partial<RuntimeKnowledgeConfig>): void {
|
|
791
|
+
const previous = this.config
|
|
792
|
+
this.config = { ...previous, ...patch }
|
|
793
|
+
|
|
794
|
+
const modelChanged =
|
|
795
|
+
previous.embeddingProvider !== this.config.embeddingProvider ||
|
|
796
|
+
previous.embeddingModel !== this.config.embeddingModel ||
|
|
797
|
+
previous.embeddingDimension !== this.config.embeddingDimension ||
|
|
798
|
+
previous.embeddingDevice !== this.config.embeddingDevice ||
|
|
799
|
+
previous.python !== this.config.python
|
|
800
|
+
const rerankChanged =
|
|
801
|
+
previous.rerankProvider !== this.config.rerankProvider ||
|
|
802
|
+
previous.rerankModel !== this.config.rerankModel ||
|
|
803
|
+
previous.python !== this.config.python
|
|
804
|
+
|
|
805
|
+
if (modelChanged) {
|
|
806
|
+
this.embeddingWorker?.close()
|
|
807
|
+
this.embeddingWorker = null
|
|
808
|
+
// 旧索引建在旧模型/旧维度下,留着它只会给出没有意义的相似度。
|
|
809
|
+
this.vectorIndex = null
|
|
810
|
+
this.vectorError = undefined
|
|
811
|
+
}
|
|
812
|
+
if (rerankChanged) {
|
|
813
|
+
this.rerankWorker?.close()
|
|
814
|
+
this.rerankWorker = null
|
|
815
|
+
this.rerankUnavailable = undefined
|
|
816
|
+
}
|
|
817
|
+
if (patch.python !== undefined) this.pythonPath = patch.python.trim() || this.pythonPath
|
|
818
|
+
if (patch.embeddingModel !== undefined) this.embeddingModelPath = patch.embeddingModel.trim() || this.embeddingModelPath
|
|
819
|
+
if (patch.rerankModel !== undefined) this.rerankModelPath = patch.rerankModel.trim() || this.rerankModelPath
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// ------------------------------------------------- Wiki 卡片类型注册表(0024)
|
|
823
|
+
|
|
824
|
+
/** 当前生效的卡片 schema:类型定义 + 生效方式。 */
|
|
825
|
+
wikiSchema(): WikiSchemaSnapshot {
|
|
826
|
+
return this.wikiSchemaProvider?.() ?? { types: defaultWikiTypes(), mode: 'report' }
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Agent 与设置面板共用的类型目录:注册表 + 生效方式 + 内置只读判定规则 +
|
|
831
|
+
* 现有卡片的三层校验结果(`report` 模式下这是"待补"清单)。
|
|
832
|
+
*/
|
|
833
|
+
async wikiTypeCatalog(): Promise<Envelope<WikiTypeCatalog>> {
|
|
834
|
+
const schema = this.wikiSchema()
|
|
835
|
+
const issues = await this.lintCards()
|
|
836
|
+
return success({
|
|
837
|
+
types: schema.types,
|
|
838
|
+
mode: schema.mode,
|
|
839
|
+
rules: WIKI_TYPE_RULES,
|
|
840
|
+
issues: issues.slice(0, 50),
|
|
841
|
+
issueCount: issues.length,
|
|
842
|
+
})
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/** 对已落库的卡片跑三层校验。不写盘,只报告。 */
|
|
846
|
+
async lintCards(): Promise<Array<CardSchemaIssue & { entityId: string; entityType: string }>> {
|
|
847
|
+
const schema = this.wikiSchema()
|
|
848
|
+
const byId = new Map(schema.types.map(def => [def.id, def]))
|
|
849
|
+
const result: Array<CardSchemaIssue & { entityId: string; entityType: string }> = []
|
|
850
|
+
for (const card of await this.loadCards()) {
|
|
851
|
+
const type = byId.get(card.entityType)
|
|
852
|
+
if (!type) continue
|
|
853
|
+
for (const issue of validateCardAgainstType(type, card)) {
|
|
854
|
+
result.push({ ...issue, entityId: card.entityId, entityType: card.entityType })
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
return result
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
/**
|
|
861
|
+
* `write_knowledge_card` 的类型闸门 + 三层校验。
|
|
862
|
+
*
|
|
863
|
+
* 返回 `{ type, issues }` 或一个失败信封。是否因 issues 拒绝写入由 `mode` 决定:
|
|
864
|
+
* `strict` 直接拒绝,`report` 把问题写进卡片并回报——存量卡片因此不会被升级挡在门外。
|
|
865
|
+
*/
|
|
866
|
+
private cardSchemaGate(input: { entityType: string; title: string; content: string; sourceRefs: string[]; metadata: Record<string, unknown>; owner?: string; summary?: string }):
|
|
867
|
+
{ type: WikiTypeDef; issues: CardSchemaIssue[] } | { error: KnowledgeError } {
|
|
868
|
+
const schema = this.wikiSchema()
|
|
869
|
+
const type = schema.types.find(def => def.id === input.entityType)
|
|
870
|
+
if (!type) {
|
|
871
|
+
return {
|
|
872
|
+
error: {
|
|
873
|
+
code: 'SCHEMA_TYPE_INVALID',
|
|
874
|
+
message: `未知卡片类型: ${input.entityType}`,
|
|
875
|
+
retryable: false,
|
|
876
|
+
suggestion: `先在「知识库设置 → Wiki 类型」里登记它,或调 list_wiki_types 查看可用类型。可用类型: ${schema.types.map(def => def.id).join(' / ')}`,
|
|
877
|
+
},
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
if (!type.enabled) {
|
|
881
|
+
return {
|
|
882
|
+
error: {
|
|
883
|
+
code: 'WIKI_TYPE_DISABLED',
|
|
884
|
+
message: `卡片类型已停用: ${input.entityType}`,
|
|
885
|
+
retryable: false,
|
|
886
|
+
suggestion: '在「知识库设置 → Wiki 类型」里重新启用它,或换一个类型。',
|
|
887
|
+
},
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
return { type, issues: validateCardAgainstType(type, input) }
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
// ------------------------------------------------- 来源分类注册表(0013)
|
|
894
|
+
|
|
895
|
+
/** 当前生效的来源分类注册表(含计算字段 `sourceCount`)。 */
|
|
896
|
+
sourceTypes(): SourceTypeDef[] {
|
|
897
|
+
return this.sourceTypeProvider?.() ?? defaultSourceTypes()
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
/** Agent 与设置面板共用的分类目录:注册表 + 内置只读判定提示词。 */
|
|
901
|
+
sourceTypeCatalog(): Envelope<SourceTypeCatalog> {
|
|
902
|
+
return success(buildSourceTypeCatalog(this.sourceTypes()))
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* `add_source` 的分类闸门。
|
|
907
|
+
*
|
|
908
|
+
* 三态区分得很清楚,因为它们的修法完全不同:
|
|
909
|
+
* - 未知分类 → `SOURCE_TYPE_UNKNOWN`(去注册表里加,或改用一个已存在的 id);
|
|
910
|
+
* - 已停用 → `SOURCE_TYPE_DISABLED`(去设置里重新启用,或换一个分类);
|
|
911
|
+
* - 兜底分类被停用 → 不可能,`document` 由注册表守卫保证始终启用。
|
|
912
|
+
*/
|
|
913
|
+
private sourceTypeGate(sourceType: string): KnowledgeError | null {
|
|
914
|
+
const defs = this.sourceTypes()
|
|
915
|
+
const def = defs.find(item => item.id === sourceType)
|
|
916
|
+
if (!def) {
|
|
917
|
+
return {
|
|
918
|
+
code: 'SOURCE_TYPE_UNKNOWN',
|
|
919
|
+
message: `未知来源分类: ${sourceType}`,
|
|
920
|
+
retryable: false,
|
|
921
|
+
suggestion: `先调 list_source_types 查看该知识库的分类注册表。可用分类: ${defs.map(item => item.id).join(' / ')}`,
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
if (!def.enabled) {
|
|
925
|
+
return {
|
|
926
|
+
code: 'SOURCE_TYPE_DISABLED',
|
|
927
|
+
message: `来源分类已停用: ${sourceType}`,
|
|
928
|
+
retryable: false,
|
|
929
|
+
suggestion: '在「知识库设置 → 来源类型」里重新启用它,或改用另一个分类。',
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
return null
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
directory(name: typeof LAYOUT[number]): string { return join(this.rootDir, name) }
|
|
936
|
+
async ensureLayout(): Promise<void> {
|
|
937
|
+
await Promise.all(LAYOUT.map(name => mkdir(this.directory(name), { recursive: true })))
|
|
938
|
+
// 记忆段在知识树**外面**(0032),所以要单独建。它与 LAYOUT 一起建的理由相同:
|
|
939
|
+
// 「目录结构」本身是承诺,用户第一次打开面板就该在磁盘上看到它。
|
|
940
|
+
await mkdir(this.memoryDir, { recursive: true })
|
|
941
|
+
// 一次性的段内布局整理(0030)。放在 `ensureLayout` 里而不是构造时:构造发生在
|
|
942
|
+
// 设置解析的回调链上,那里做 FS 扫描会把启动拖住;而 `ensureLayout` 已经是所有
|
|
943
|
+
// 写入与统计的入口。实例级标志保证每个 runtime 只扫一次。
|
|
944
|
+
if (!this.segmentsOrganized) {
|
|
945
|
+
this.segmentsOrganized = true
|
|
946
|
+
await this.migrateSourceTruthLayout()
|
|
947
|
+
await this.dropLegacySchemaSegment()
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
/**
|
|
952
|
+
* 0030:把平铺的 `<段>/*.md` 搬进 `<段>/<source_type>/*.md`。
|
|
953
|
+
*
|
|
954
|
+
* 三条纪律与 `migrate.ts` 一致:**只动直接躺在段目录下的 `.md`**(已经在子目录里的
|
|
955
|
+
* 不碰,所以可重入)、**读不出合法 `source_type` 的就留在原地**(宁可少搬,不可搬错
|
|
956
|
+
* ——分类只是元数据,搬错目录会让人按分类找不到东西)、**失败不回滚也不报错**(
|
|
957
|
+
* `loadSources` 两种布局都能读,搬不动不影响正确性)。
|
|
958
|
+
*/
|
|
959
|
+
private async migrateSourceTruthLayout(): Promise<void> {
|
|
960
|
+
const root = this.directory(SOURCE_TRUTH)
|
|
961
|
+
let entries
|
|
962
|
+
try { entries = await readdir(root, { withFileTypes: true }) } catch { return }
|
|
963
|
+
for (const entry of entries) {
|
|
964
|
+
if (!entry.isFile() || extname(entry.name).toLowerCase() !== '.md') continue
|
|
965
|
+
const from = join(root, entry.name)
|
|
966
|
+
let sourceType = ''
|
|
967
|
+
try {
|
|
968
|
+
sourceType = String(parseMarkdown(await readFile(from, 'utf8')).frontmatter.source_type ?? '')
|
|
969
|
+
} catch { continue }
|
|
970
|
+
if (!SOURCE_TYPE_ID_PATTERN.test(sourceType)) continue
|
|
971
|
+
const target = join(root, sourceType, entry.name)
|
|
972
|
+
try {
|
|
973
|
+
await mkdir(dirname(target), { recursive: true })
|
|
974
|
+
await rename(from, target)
|
|
975
|
+
} catch { /* 搬不动就留在原地:读取不依赖目录层级 */ }
|
|
976
|
+
}
|
|
977
|
+
// 目录变了,索引必须重建。
|
|
978
|
+
this.sourceIndex = undefined
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
/**
|
|
982
|
+
* 0030:删掉已经被撤掉的 `00_schema` 段。
|
|
983
|
+
*
|
|
984
|
+
* **只在它确实是空的时候删。** 这个段从来没被写过东西,但如果用户的目录里恰好
|
|
985
|
+
* 有人放了文件,那就不属于「没用的空文件夹」了 —— 那种情况下留着并让人自己判断,
|
|
986
|
+
* 比替他们删掉安全。
|
|
987
|
+
*/
|
|
988
|
+
private async dropLegacySchemaSegment(): Promise<void> {
|
|
989
|
+
const path = this.safePath('00_schema')
|
|
990
|
+
try {
|
|
991
|
+
const entries = await readdir(path)
|
|
992
|
+
if (entries.length > 0) return
|
|
993
|
+
// `rmdir`,不是 `rm`:它在目录非空时会失败,这正好是我们要的保证 —— 万一在
|
|
994
|
+
// readdir 与删除之间有人放了东西进来,失败比删掉别人的文件安全。
|
|
995
|
+
// (`rm` 不带 `recursive` 对目录直接抛 EISDIR,什么也不会发生,看着像删了。)
|
|
996
|
+
await rmdir(path)
|
|
997
|
+
} catch { /* 不存在、非空、或删不掉都不是错误 */ }
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
private async loadVectorIndex(): Promise<LoadedVectorIndex | null> {
|
|
1001
|
+
if (this.vectorIndex) return this.vectorIndex
|
|
1002
|
+
try {
|
|
1003
|
+
const manifest = JSON.parse(await readFile(this.safePath('03_compiled', VECTOR_MANIFEST), 'utf8')) as VectorManifest
|
|
1004
|
+
if (manifest.quantization !== VECTOR_QUANTIZATION || !Number.isInteger(manifest.dimension) || !Array.isArray(manifest.items)) return null
|
|
1005
|
+
// 编码实现换了 → 盘上这份索引不是「同一个东西」,当作不存在(于是 recall 会如实降级,
|
|
1006
|
+
// 而不是拿一份用别的编码器算出来的向量去排序)。
|
|
1007
|
+
if (manifest.encoder !== ENCODER_FINGERPRINT) return null
|
|
1008
|
+
const binary = await readFile(this.safePath('03_compiled', VECTOR_BINARY))
|
|
1009
|
+
const int8 = await readFile(this.safePath('03_compiled', VECTOR_INT8))
|
|
1010
|
+
const binaryWidth = Math.ceil(manifest.dimension / 8)
|
|
1011
|
+
if (manifest.itemCount !== manifest.items.length || binary.length !== manifest.itemCount * binaryWidth || int8.length !== manifest.itemCount * manifest.dimension) return null
|
|
1012
|
+
this.vectorIndex = { manifest, binary, int8 }
|
|
1013
|
+
return this.vectorIndex
|
|
1014
|
+
} catch {
|
|
1015
|
+
return null
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
private vectorWorkerPath(): string {
|
|
1020
|
+
return process.env.DSH_EMBEDDING_WORKER || resolve(dirname(fileURLToPath(import.meta.url)), '../../src/python/embedding_worker.py')
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
private async encodeWithOllama(inputs: EmbeddingInput[]): Promise<{ vectors: number[][]; device: string }> {
|
|
1024
|
+
const base = this.config.embeddingBaseUrl.replace(/\/$/, '')
|
|
1025
|
+
const controller = new AbortController()
|
|
1026
|
+
const timer = setTimeout(() => controller.abort(), this.config.embeddingTimeoutMs)
|
|
1027
|
+
try {
|
|
1028
|
+
const body = JSON.stringify({ model: this.embeddingModelPath, input: inputs.map(item => item.text) })
|
|
1029
|
+
const response = await fetch(`${base}/api/embed`, { method: 'POST', headers: { 'content-type': 'application/json' }, body, signal: controller.signal })
|
|
1030
|
+
if (response.status === 404) {
|
|
1031
|
+
// Older Ollama releases expose only the single-prompt endpoint. Do
|
|
1032
|
+
// not silently return one vector for a batch; that used to corrupt
|
|
1033
|
+
// row-to-vector alignment during a rebuild.
|
|
1034
|
+
const legacy: unknown[] = []
|
|
1035
|
+
for (const input of inputs) {
|
|
1036
|
+
const itemResponse = await fetch(`${base}/api/embeddings`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ model: this.embeddingModelPath, prompt: input.text }), signal: controller.signal })
|
|
1037
|
+
if (!itemResponse.ok) throw new Error(`OLLAMA_HTTP_${itemResponse.status}`)
|
|
1038
|
+
const itemPayload = await itemResponse.json() as { embedding?: unknown }
|
|
1039
|
+
if (!Array.isArray(itemPayload.embedding)) throw new Error('OLLAMA_EMBEDDING_RESPONSE_INVALID')
|
|
1040
|
+
legacy.push(itemPayload.embedding)
|
|
1041
|
+
}
|
|
1042
|
+
return { vectors: normaliseVectors(legacy, this.config.embeddingDimension), device: 'ollama' }
|
|
1043
|
+
}
|
|
1044
|
+
if (!response.ok) throw new Error(`OLLAMA_HTTP_${response.status}`)
|
|
1045
|
+
const payload = await response.json() as { embeddings?: unknown; embedding?: unknown }
|
|
1046
|
+
const raw = Array.isArray(payload.embeddings) ? payload.embeddings : payload.embedding ? [payload.embedding] : []
|
|
1047
|
+
return { vectors: normaliseVectors(raw, this.config.embeddingDimension), device: 'ollama' }
|
|
1048
|
+
} finally {
|
|
1049
|
+
clearTimeout(timer)
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
private async encodeEmbeddings(inputs: EmbeddingInput[]): Promise<{ vectors: number[][]; device: string }> {
|
|
1054
|
+
if (this.config.embeddingProvider === 'none') throw new Error('EMBEDDING_DISABLED')
|
|
1055
|
+
// Keep each JSON response bounded for large stores. The WeMM process is
|
|
1056
|
+
// persistent, so chunking does not reload the 1.8GB model between chunks.
|
|
1057
|
+
const chunkSize = Math.max(1, this.config.embeddingBatchSize * 64)
|
|
1058
|
+
const vectors: number[][] = []
|
|
1059
|
+
let device = this.config.embeddingProvider === 'ollama' ? 'ollama' : this.config.embeddingDevice
|
|
1060
|
+
for (let start = 0; start < inputs.length; start += chunkSize) {
|
|
1061
|
+
const chunk = inputs.slice(start, start + chunkSize)
|
|
1062
|
+
if (this.config.embeddingProvider === 'ollama') {
|
|
1063
|
+
const result = await this.encodeWithOllama(chunk)
|
|
1064
|
+
vectors.push(...result.vectors)
|
|
1065
|
+
device = result.device
|
|
1066
|
+
continue
|
|
1067
|
+
}
|
|
1068
|
+
if (!this.embeddingWorker) this.embeddingWorker = this.spawnWorker()
|
|
1069
|
+
const result = await this.embeddingWorker.request({
|
|
1070
|
+
model: this.embeddingModelPath,
|
|
1071
|
+
device: this.config.embeddingDevice,
|
|
1072
|
+
dimension: this.config.embeddingDimension,
|
|
1073
|
+
batchSize: this.config.embeddingBatchSize,
|
|
1074
|
+
inputs: chunk,
|
|
1075
|
+
}, this.config.embeddingTimeoutMs)
|
|
1076
|
+
vectors.push(...normaliseVectors(result.vectors, this.config.embeddingDimension))
|
|
1077
|
+
device = String(result.device ?? this.config.embeddingDevice)
|
|
1078
|
+
}
|
|
1079
|
+
return { vectors, device }
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
close(): void {
|
|
1083
|
+
this.embeddingWorker?.close()
|
|
1084
|
+
this.embeddingWorker = null
|
|
1085
|
+
this.rerankWorker?.close()
|
|
1086
|
+
this.rerankWorker = null
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
/** Both the encoder and the reranker speak the same JSONL worker protocol. */
|
|
1090
|
+
private spawnWorker(label = 'EMBEDDING'): JsonLineWorker {
|
|
1091
|
+
return new JsonLineWorker(this.pythonPath, [this.vectorWorkerPath()], this.rootDir, process.env, label)
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
private async vectorInputs(units: KnowledgeUnit[]): Promise<Array<{ unit: KnowledgeUnit; input: EmbeddingInput; contentHash: string }>> {
|
|
1095
|
+
const result: Array<{ unit: KnowledgeUnit; input: EmbeddingInput; contentHash: string }> = []
|
|
1096
|
+
for (const unit of units) {
|
|
1097
|
+
const imagePaths = unit.imagePaths ?? []
|
|
1098
|
+
const fingerprints = await imageFingerprints(imagePaths)
|
|
1099
|
+
const text = `${unit.title}\n\n${unit.content}`.slice(0, 120_000)
|
|
1100
|
+
const input: EmbeddingInput = { text, ...(imagePaths[0] ? { image: imagePaths[0] } : {}) }
|
|
1101
|
+
result.push({ unit, input, contentHash: hash(JSON.stringify({ unitId: unit.unitId, text, sourceRefs: unit.sourceRefs, images: fingerprints })) })
|
|
1102
|
+
}
|
|
1103
|
+
return result
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
/** 关键词索引的最近一次失败原因;`undefined` = 上次构建成功。 */
|
|
1107
|
+
private ftsError: string | undefined
|
|
1108
|
+
|
|
1109
|
+
/**
|
|
1110
|
+
* 关键词索引(0031):FTS5 + BM25,见 `fts.ts`。
|
|
1111
|
+
*
|
|
1112
|
+
* 与向量索引的区别是它**不依赖模型**,所以 `only: 'wiki'` 也重建它 —— 卡片改了、
|
|
1113
|
+
* 关键词索引不跟着改,就会出现「按旧词表召回」。`node:sqlite` 不可用时如实返回
|
|
1114
|
+
* `unavailable`(诊断里会写明),而不是退回 0030 那套对中文无效的计数打分。
|
|
1115
|
+
*/
|
|
1116
|
+
private async buildFtsIndex(units: KnowledgeUnit[], sources: SourceRaw[] = []): Promise<'ready' | 'unavailable'> {
|
|
1117
|
+
const byId = new Map(sources.map(source => [source.sourceId, source.title]))
|
|
1118
|
+
const sqlite = await loadSqlite()
|
|
1119
|
+
if (!sqlite) {
|
|
1120
|
+
this.ftsError = '当前 Node 没有 node:sqlite(需要 22.5+)'
|
|
1121
|
+
return 'unavailable'
|
|
1122
|
+
}
|
|
1123
|
+
// 索引的不只是正文,还有这条 note 的**溯源身份**:它引用的 sourceId 与那些来源的标题。
|
|
1124
|
+
//
|
|
1125
|
+
// 这不是锦上添花,而是「图谱降为 boost」之后的必要补偿:boost 只能给已有候选加权,
|
|
1126
|
+
// 不能制造候选,所以「只提到某份来源、没提到任何卡片正文」的查询会一条都召不回
|
|
1127
|
+
// (0031 实测:查 `ledger` 命中 source 节点,但 0 命中)。把溯源身份放进关键词索引,
|
|
1128
|
+
// 「这份台账派生出了哪些结论」就回到关键词路回答 —— 那本来就是词面问题,不是图问题。
|
|
1129
|
+
const titleOf = (sourceId: string): string | undefined => byId.get(sourceId)
|
|
1130
|
+
try {
|
|
1131
|
+
await FtsIndex.build(
|
|
1132
|
+
this.safePath('03_compiled', FTS_FILE),
|
|
1133
|
+
units.map(unit => ({
|
|
1134
|
+
unitId: unit.unitId,
|
|
1135
|
+
title: unit.title,
|
|
1136
|
+
content: noteIndexText({ title: unit.title, content: unit.content, sourceRefs: unit.sourceRefs }, titleOf),
|
|
1137
|
+
})),
|
|
1138
|
+
sqlite.DatabaseSync,
|
|
1139
|
+
)
|
|
1140
|
+
this.ftsError = undefined
|
|
1141
|
+
return 'ready'
|
|
1142
|
+
} catch (cause) {
|
|
1143
|
+
this.ftsError = cause instanceof Error ? cause.message : String(cause)
|
|
1144
|
+
return 'unavailable'
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
private async buildVectorIndex(units: KnowledgeUnit[], version: number): Promise<EmbeddingBuildResult> {
|
|
1149
|
+
if (this.config.embeddingProvider === 'none') return { status: 'skipped', count: 0, reused: 0 }
|
|
1150
|
+
if (this.config.embeddingDimension % 8 !== 0) return { status: 'unavailable', count: 0, reused: 0, error: 'embeddingDimension 必须是 8 的倍数,才能进行 binary quantization' }
|
|
1151
|
+
const inputs = await this.vectorInputs(units)
|
|
1152
|
+
if (inputs.length === 0) return { status: 'ready', count: 0, reused: 0, dimension: this.config.embeddingDimension, device: 'none' }
|
|
1153
|
+
|
|
1154
|
+
const previous = await this.loadVectorIndex()
|
|
1155
|
+
const compatiblePrevious = previous && previous.manifest.dimension === this.config.embeddingDimension && previous.manifest.provider === this.config.embeddingProvider && previous.manifest.model === this.embeddingModelId && previous.manifest.encoder === ENCODER_FINGERPRINT ? previous : null
|
|
1156
|
+
const previousById = new Map((compatiblePrevious?.manifest.items ?? []).map((item, index) => [item.unitId, { item, index }]))
|
|
1157
|
+
const binaryWidth = Math.ceil(this.config.embeddingDimension / 8)
|
|
1158
|
+
const binary = Buffer.alloc(inputs.length * binaryWidth)
|
|
1159
|
+
const int8 = Buffer.alloc(inputs.length * this.config.embeddingDimension)
|
|
1160
|
+
const pending: typeof inputs = []
|
|
1161
|
+
let reused = 0
|
|
1162
|
+
for (let index = 0; index < inputs.length; index += 1) {
|
|
1163
|
+
const current = inputs[index]!
|
|
1164
|
+
const old = previousById.get(current.unit.unitId)
|
|
1165
|
+
if (old && old.item.contentHash === current.contentHash && compatiblePrevious && old.index < compatiblePrevious.manifest.itemCount) {
|
|
1166
|
+
compatiblePrevious.binary.copy(binary, index * binaryWidth, old.index * binaryWidth, (old.index + 1) * binaryWidth)
|
|
1167
|
+
compatiblePrevious.int8.copy(int8, index * this.config.embeddingDimension, old.index * this.config.embeddingDimension, (old.index + 1) * this.config.embeddingDimension)
|
|
1168
|
+
reused += 1
|
|
1169
|
+
} else pending.push(current)
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
let device = compatiblePrevious?.manifest.device ?? 'cache'
|
|
1173
|
+
try {
|
|
1174
|
+
if (pending.length > 0) {
|
|
1175
|
+
const encoded = await this.encodeEmbeddings(pending.map(item => item.input))
|
|
1176
|
+
if (encoded.vectors.length !== pending.length) throw new Error(`EMBEDDING_COUNT_INVALID: expected ${pending.length}, got ${encoded.vectors.length}`)
|
|
1177
|
+
device = encoded.device
|
|
1178
|
+
const pendingRows = new Map(pending.map((item, index) => [item.unit.unitId, index]))
|
|
1179
|
+
for (let index = 0; index < inputs.length; index += 1) {
|
|
1180
|
+
const row = pendingRows.get(inputs[index]!.unit.unitId)
|
|
1181
|
+
if (row === undefined) continue
|
|
1182
|
+
const vector = encoded.vectors[row]!
|
|
1183
|
+
binaryVector(vector, this.config.embeddingDimension).copy(binary, index * binaryWidth)
|
|
1184
|
+
int8Vector(vector, this.config.embeddingDimension).copy(int8, index * this.config.embeddingDimension)
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
/**
|
|
1188
|
+
* 索引多样性断言(0031 的 P0)。
|
|
1189
|
+
*
|
|
1190
|
+
* 编码器坏掉时,44 条 note 会产出**同一份向量**(0030 实测 44 行里 39 行逐字节
|
|
1191
|
+
* 相同),而构建照样报 `ready`、界面照样显示「三路都在跑」。这种索引比没有索引
|
|
1192
|
+
* 更坏:它让排序退化成 `unitId` 字典序,却看起来一切正常。
|
|
1193
|
+
*
|
|
1194
|
+
* 所以写盘之前先数一遍不同的向量。宁可报 unavailable 并把原因写进编码器健康状态,
|
|
1195
|
+
* 也不留下一份骗人的索引。
|
|
1196
|
+
*/
|
|
1197
|
+
const dimension = this.config.embeddingDimension
|
|
1198
|
+
const distinct = new Set<string>()
|
|
1199
|
+
for (let row = 0; row < inputs.length; row += 1) distinct.add(int8.subarray(row * dimension, (row + 1) * dimension).toString('base64'))
|
|
1200
|
+
if (inputs.length >= 4 && distinct.size < Math.max(2, Math.ceil(inputs.length / 2))) {
|
|
1201
|
+
this.encoderHealth = { ok: false, reason: `编码器退化:${inputs.length} 条 note 只产出 ${distinct.size} 个不同向量` }
|
|
1202
|
+
throw new Error(`EMBEDDING_INDEX_DEGENERATE: ${inputs.length} 条只产出 ${distinct.size} 个不同向量`)
|
|
1203
|
+
}
|
|
1204
|
+
const manifest: VectorManifest = { version, createdAt: new Date().toISOString(), provider: this.config.embeddingProvider, model: this.embeddingModelId, encoder: ENCODER_FINGERPRINT, device, dimension: this.config.embeddingDimension, quantization: VECTOR_QUANTIZATION, itemCount: inputs.length, items: inputs.map(item => ({ unitId: item.unit.unitId, contentHash: item.contentHash })) }
|
|
1205
|
+
await writeBufferAtomic(this.safePath('03_compiled', VECTOR_BINARY), binary)
|
|
1206
|
+
await writeBufferAtomic(this.safePath('03_compiled', VECTOR_INT8), int8)
|
|
1207
|
+
await writeTextAtomic(this.safePath('03_compiled', VECTOR_MANIFEST), JSON.stringify(manifest, null, 2))
|
|
1208
|
+
this.vectorIndex = { manifest, binary, int8 }
|
|
1209
|
+
this.vectorError = undefined
|
|
1210
|
+
return { status: 'ready', count: inputs.length, reused, dimension: this.config.embeddingDimension, device }
|
|
1211
|
+
} catch (cause) {
|
|
1212
|
+
const error = cause instanceof Error ? cause.message : String(cause)
|
|
1213
|
+
this.vectorError = error
|
|
1214
|
+
return { status: 'unavailable', count: compatiblePrevious?.manifest.itemCount ?? 0, reused, dimension: this.config.embeddingDimension, error }
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
private async loadCompiledState(): Promise<void> {
|
|
1219
|
+
// Stage 2 writes wiki_notes.json; a store compiled before 0014 only has
|
|
1220
|
+
// knowledge_units.json, which stays readable until the next rebuild.
|
|
1221
|
+
for (const artifact of [WIKI_NOTES, LEGACY_UNITS]) {
|
|
1222
|
+
try {
|
|
1223
|
+
const notes = JSON.parse(await readFile(this.safePath('03_compiled', artifact), 'utf8'))
|
|
1224
|
+
if (Array.isArray(notes) && notes.length > 0) { this.units = notes as KnowledgeUnit[]; break }
|
|
1225
|
+
} catch { /* the first build has not happened yet */ }
|
|
1226
|
+
}
|
|
1227
|
+
try {
|
|
1228
|
+
const graph = JSON.parse(await readFile(this.safePath('03_compiled', 'knowledge_graph.json'), 'utf8'))
|
|
1229
|
+
if (graph && Array.isArray(graph.nodes) && Array.isArray(graph.edges)) this.graph = graph as typeof this.graph
|
|
1230
|
+
} catch { /* the first build has not happened yet */ }
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
private safePath(...parts: string[]): string {
|
|
1234
|
+
const path = resolve(this.rootDir, ...parts)
|
|
1235
|
+
const root = this.rootDir.endsWith(sep) ? this.rootDir : `${this.rootDir}${sep}`
|
|
1236
|
+
if (path !== this.rootDir && !path.startsWith(root)) throw new Error('PATH_OUTSIDE_KNOWLEDGE_ROOT')
|
|
1237
|
+
return path
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/**
|
|
1241
|
+
* 记忆段里的一个路径(0032)。
|
|
1242
|
+
*
|
|
1243
|
+
* 与 `safePath` 同一套守卫,边界换成 `memoryDir` —— 记忆段现在住在知识树
|
|
1244
|
+
* **外面**(`<agentRoot>/memory/<库 id>`),拿 `safePath` 拼会直接抛
|
|
1245
|
+
* `PATH_OUTSIDE_KNOWLEDGE_ROOT`。两个守卫各自成立,不合并成一个「允许两个根」的
|
|
1246
|
+
* 宽松版本:那样一旦哪个根算错,路径就会悄悄写到别人的目录里去。
|
|
1247
|
+
*/
|
|
1248
|
+
private memoryPath(...parts: string[]): string {
|
|
1249
|
+
const path = resolve(this.memoryDir, ...parts)
|
|
1250
|
+
const root = this.memoryDir.endsWith(sep) ? this.memoryDir : `${this.memoryDir}${sep}`
|
|
1251
|
+
if (path !== this.memoryDir && !path.startsWith(root)) throw new Error('PATH_OUTSIDE_MEMORY_ROOT')
|
|
1252
|
+
return path
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
/** 记忆段的真实位置(面板与诊断要显示它,而不是让人以为还在知识树里)。 */
|
|
1256
|
+
get memoryRoot(): string { return this.memoryDir }
|
|
1257
|
+
|
|
1258
|
+
async stats(): Promise<Record<string, unknown>> {
|
|
1259
|
+
await this.ensureLayout()
|
|
1260
|
+
const counts = await Promise.all(LAYOUT.map(item => fileCount(this.directory(item))))
|
|
1261
|
+
if (this.cards.length === 0) this.cards = await this.loadCards()
|
|
1262
|
+
if (this.units.length === 0 && this.graph.nodes.length === 0) await this.loadCompiledState()
|
|
1263
|
+
let buildManifest: {
|
|
1264
|
+
version?: number
|
|
1265
|
+
builtAt?: string
|
|
1266
|
+
cardHashes?: Record<string, string>
|
|
1267
|
+
embedding?: { status?: string; reused?: number; count?: number }
|
|
1268
|
+
compileRunId?: string
|
|
1269
|
+
lastBuild?: {
|
|
1270
|
+
only?: string; incremental?: boolean; skipped?: boolean
|
|
1271
|
+
cardsReused?: number; cardsRecompiled?: number; cardsAdded?: number; cardsRemoved?: number
|
|
1272
|
+
notesReused?: number; notesRecompiled?: number
|
|
1273
|
+
// Pass 0 的计数:面板据此显示「这一轮抽了几个候选、建了几张卡」。
|
|
1274
|
+
candidatesExtracted?: number; candidatesRejected?: number
|
|
1275
|
+
cardsCreated?: number; cardsMerged?: number
|
|
1276
|
+
relationsExtracted?: number; relationsDropped?: number
|
|
1277
|
+
sourcesExtracted?: number; sourcesSkipped?: number
|
|
1278
|
+
}
|
|
1279
|
+
} = {}
|
|
1280
|
+
try {
|
|
1281
|
+
buildManifest = JSON.parse(await readFile(this.safePath('03_compiled', 'build-manifest.json'), 'utf8')) as typeof buildManifest
|
|
1282
|
+
} catch { /* the first build has not happened yet */ }
|
|
1283
|
+
const vector = await this.loadVectorIndex()
|
|
1284
|
+
// 卡片版本快照数:界面要能说清「wiki 也有历史」,而不只是 SourceRaw 有。
|
|
1285
|
+
const cardVersionCount = await fileCount(this.memoryPath(CARD_VERSIONS))
|
|
1286
|
+
const layout = Object.fromEntries(LAYOUT.map((item, index) => [item, counts[index] ?? 0]))
|
|
1287
|
+
return {
|
|
1288
|
+
rootDir: this.rootDir,
|
|
1289
|
+
layout,
|
|
1290
|
+
/** 0032:记忆段的真实位置与落盘文件数。它不在 `layout` 里了 —— 那一段
|
|
1291
|
+
* 已经不是知识树的一段,混进去会让「知识树有哪几段」这个问题的答案漂移。 */
|
|
1292
|
+
memoryRoot: this.memoryDir,
|
|
1293
|
+
memoryFiles: await fileCount(this.memoryDir),
|
|
1294
|
+
/**
|
|
1295
|
+
* 按**段名**取,不按位置取。这里原先是 `counts[1]`,而它之所以曾经是对的,
|
|
1296
|
+
* 只是因为 `LAYOUT[1]` 恰好是 `01_source_truth` —— 0030 撤掉 `00_schema`
|
|
1297
|
+
* 之后同一条表达式就静默变成了「wiki 卡片数」,界面上的来源数直接归零。
|
|
1298
|
+
* 布局的索引是实现的偶然,段名才是契约。
|
|
1299
|
+
*/
|
|
1300
|
+
sourceCount: layout[SOURCE_TRUTH] ?? 0,
|
|
1301
|
+
cards: this.cards.length,
|
|
1302
|
+
units: this.units.length,
|
|
1303
|
+
notes: this.units.length,
|
|
1304
|
+
graphNodes: this.graph.nodes.length,
|
|
1305
|
+
graphEdges: this.graph.edges.length,
|
|
1306
|
+
buildVersion: Number(buildManifest.version ?? 0),
|
|
1307
|
+
builtAt: buildManifest.builtAt,
|
|
1308
|
+
trackedCards: Object.keys(buildManifest.cardHashes ?? {}).length,
|
|
1309
|
+
cardVersions: cardVersionCount,
|
|
1310
|
+
lastBuild: buildManifest.lastBuild ?? null,
|
|
1311
|
+
lastCompileRunId: buildManifest.compileRunId,
|
|
1312
|
+
recallTopK: this.config.recallTopK,
|
|
1313
|
+
recallMode: this.config.recallMode,
|
|
1314
|
+
contextTopN: this.config.contextTopN,
|
|
1315
|
+
rerankProvider: this.config.rerankProvider,
|
|
1316
|
+
rerankModel: this.rerankModelPath,
|
|
1317
|
+
embeddingProvider: this.config.embeddingProvider,
|
|
1318
|
+
embeddingModel: this.embeddingModelPath,
|
|
1319
|
+
embeddingDevice: this.config.embeddingDevice,
|
|
1320
|
+
embeddingDimension: vector?.manifest.dimension ?? this.config.embeddingDimension,
|
|
1321
|
+
vectorCount: vector?.manifest.itemCount ?? 0,
|
|
1322
|
+
vectorQuantization: vector?.manifest.quantization ?? VECTOR_QUANTIZATION,
|
|
1323
|
+
searchBackend: vector && vector.manifest.itemCount > 0 ? 'hybrid' : 'fts',
|
|
1324
|
+
embeddingStatus: vector ? 'ready' : this.config.embeddingProvider === 'none' ? 'disabled' : 'unavailable',
|
|
1325
|
+
embeddingError: this.vectorError,
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
async addSource(input: { sourceType: SourceType; title?: string; content: string; externalId?: string; metadata?: Record<string, unknown>; sourceId?: string; lineage?: LineageContext }): Promise<Envelope<{ sourceId: string; path: string; lineage?: LineageContext }>> {
|
|
1330
|
+
if (!this.config.allowSourceTruth) return failure({ sourceId: '', path: '' }, 'SOURCE_TRUTH_DISABLED', '当前配置禁止写入 SourceTruth', false)
|
|
1331
|
+
// 分类闸门走注册表(0013),不再是硬编码枚举:自定义分类建完即可用。
|
|
1332
|
+
const gate = this.sourceTypeGate(input.sourceType)
|
|
1333
|
+
if (gate) return failure({ sourceId: '', path: '' }, gate.code, gate.message, gate.retryable, gate.suggestion)
|
|
1334
|
+
if (typeof input.content !== 'string' || input.content.trim().length === 0) return failure({ sourceId: '', path: '' }, 'CONTENT_REQUIRED', 'content 不能为空', true)
|
|
1335
|
+
/**
|
|
1336
|
+
* 这里**没有**长度上限(0029 撤掉了 `maxSourceBytes`)。
|
|
1337
|
+
*
|
|
1338
|
+
* 它曾经是 4,000,000 字符的硬上限,而那个设置在 0029 已经从设置面板撤走 —— 于是
|
|
1339
|
+
* 它变成一个**用户改不了、也不知道存在**的闸门:长文档被拒,报的还是「去调大设置里
|
|
1340
|
+
* 那个你已经找不到的字段」。
|
|
1341
|
+
*
|
|
1342
|
+
* 真正的边界在别处,而且各自都有名有姓:转换内核的 `maxInputBytes` 管入口大小,
|
|
1343
|
+
* 嵌入按 `embeddingBatchSize` 分批、并把交给模型的文本切到模型上下文长度。在存储
|
|
1344
|
+
* 这层再留一个谁也不认识的字符数,只会让「这份文档为什么进不来」变成查不到答案的
|
|
1345
|
+
* 问题。留下的只有「不能是空串」那一条 —— 那是语义,不是配额。
|
|
1346
|
+
*/
|
|
1347
|
+
await this.ensureLayout()
|
|
1348
|
+
const sourceId = input.sourceId?.trim() || `${input.sourceType}:${hash(input.externalId ?? input.content)}`
|
|
1349
|
+
if (!SAFE_ID.test(sourceId)) return failure({ sourceId: '', path: '' }, 'SOURCE_ID_INVALID', 'source_id 含有非法字符', false)
|
|
1350
|
+
const path = this.sourcePath(sourceId, input.sourceType)
|
|
1351
|
+
/**
|
|
1352
|
+
* 这一份文档**可能已经以另一个分类存在**。0030 起分类是目录名,所以
|
|
1353
|
+
* 「改一个来源的分类」不再是原地覆盖,而是换目录 —— 必须先解析它现在落在哪,
|
|
1354
|
+
* 否则会有两个后果:版本快照比对的是一份不存在的新文件(历史被静默丢掉),
|
|
1355
|
+
* 而旧目录里那份留下来变成孤儿(统计成两个来源、分类永远删不掉)。
|
|
1356
|
+
*/
|
|
1357
|
+
const previousPath = await this.sourcePathOf(sourceId)
|
|
1358
|
+
let createdAt = new Date().toISOString()
|
|
1359
|
+
// Version history: before overwriting an existing document whose *body*
|
|
1360
|
+
// changed, snapshot the previous file bytes under
|
|
1361
|
+
// `04_memory/source-versions/<docSlug>/`. The comparison is on the body
|
|
1362
|
+
// only, so re-writing identical content (a common idempotent re-ingest)
|
|
1363
|
+
// does not manufacture a version. Because `revertDocument` calls
|
|
1364
|
+
// `addSource`, a rollback snapshots the pre-rollback head too: history is
|
|
1365
|
+
// append-only and a rollback is itself reversible.
|
|
1366
|
+
let previousRaw: string | undefined
|
|
1367
|
+
if (previousPath !== undefined) { try { previousRaw = await readFile(previousPath, 'utf8') } catch { /* vanished */ } }
|
|
1368
|
+
if (previousRaw !== undefined) {
|
|
1369
|
+
const previous = parseMarkdown(previousRaw)
|
|
1370
|
+
createdAt = String(previous.frontmatter.created_at ?? createdAt)
|
|
1371
|
+
if (previous.content.trim() !== input.content.trim()) {
|
|
1372
|
+
await this.snapshotSource(sourceId, previousRaw, previous.content)
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
const record: SourceRaw = { sourceId, sourceType: input.sourceType, title: input.title?.trim() || sourceId, content: input.content, ...(input.externalId ? { externalId: input.externalId } : {}), metadata: input.metadata ?? {}, ...(input.lineage ? { lineage: input.lineage } : {}), createdAt, updatedAt: new Date().toISOString() }
|
|
1376
|
+
await writeTextAtomic(path, markdown({ source_id: sourceId, source_type: input.sourceType, title: record.title, created_at: record.createdAt, updated_at: record.updatedAt, external_id: input.externalId ?? '', metadata: record.metadata, ...(record.lineage ? { lineage: record.lineage } : {}) }, record.content))
|
|
1377
|
+
// 换了分类的那一次:新文件已经落好,旧目录里那份才删 —— 顺序反过来的话,写入
|
|
1378
|
+
// 失败就同时丢了旧的那一份。
|
|
1379
|
+
if (previousPath !== undefined && previousPath !== path) {
|
|
1380
|
+
try { await unlink(previousPath) } catch { /* already moved or removed */ }
|
|
1381
|
+
}
|
|
1382
|
+
this.sourceIndex?.set(slug(sourceId), path)
|
|
1383
|
+
// 证据片段取自 SourceRaw 正文,写入后那份缓存必须失效(否则上下文会引用旧原文)。
|
|
1384
|
+
this.sourceCorpusCache = undefined
|
|
1385
|
+
return success({ sourceId, path, ...(record.lineage ? { lineage: record.lineage } : {}) })
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
/** Directory holding one document's version snapshots. */
|
|
1389
|
+
private versionDir(sourceId: string): string { return this.memoryPath(SOURCE_VERSIONS, slug(sourceId)) }
|
|
1390
|
+
|
|
1391
|
+
/** Write the previous file bytes as one immutable version snapshot. */
|
|
1392
|
+
private async snapshotSource(sourceId: string, raw: string, body: string): Promise<string> {
|
|
1393
|
+
const dir = this.versionDir(sourceId)
|
|
1394
|
+
await mkdir(dir, { recursive: true })
|
|
1395
|
+
const versionId = `${compactIso(new Date())}-${contentHashOf(body).slice(0, 8)}`
|
|
1396
|
+
const path = join(dir, `${versionId}.md`)
|
|
1397
|
+
if (!existsSync(path)) await writeTextAtomic(path, raw)
|
|
1398
|
+
return versionId
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
/** Snapshot file basenames (without `.md`) for one document. */
|
|
1402
|
+
private async versionIds(sourceId: string): Promise<string[]> {
|
|
1403
|
+
try {
|
|
1404
|
+
const entries = await readdir(this.versionDir(sourceId), { withFileTypes: true })
|
|
1405
|
+
return entries.filter(entry => entry.isFile() && entry.name.endsWith('.md')).map(entry => basename(entry.name, '.md')).sort().reverse()
|
|
1406
|
+
} catch { return [] }
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
async loadSources(): Promise<SourceRaw[]> {
|
|
1410
|
+
const result: SourceRaw[] = []
|
|
1411
|
+
for (const path of await filesUnder(this.directory(SOURCE_TRUTH))) {
|
|
1412
|
+
const parsed = parseMarkdown(await readFile(path, 'utf8'))
|
|
1413
|
+
const sourceId = String(parsed.frontmatter.source_id ?? basename(path, '.md'))
|
|
1414
|
+
const sourceType = String(parsed.frontmatter.source_type ?? 'document') as SourceType
|
|
1415
|
+
// 0013:未知分类**不再静默丢弃**。分类只是元数据,内容仍然可召回;
|
|
1416
|
+
// 丢掉文档才是真正的数据损失。未知分类在列表里以「未知分类」呈现。
|
|
1417
|
+
result.push({ sourceId, sourceType, title: String(parsed.frontmatter.title ?? sourceId), content: parsed.content, externalId: typeof parsed.frontmatter.external_id === 'string' && parsed.frontmatter.external_id ? parsed.frontmatter.external_id : undefined, metadata: typeof parsed.frontmatter.metadata === 'object' && parsed.frontmatter.metadata ? parsed.frontmatter.metadata as Record<string, unknown> : {}, ...(parseLineage(parsed.frontmatter.lineage) ? { lineage: parseLineage(parsed.frontmatter.lineage) } : {}), createdAt: String(parsed.frontmatter.created_at ?? ''), updatedAt: String(parsed.frontmatter.updated_at ?? '') })
|
|
1418
|
+
}
|
|
1419
|
+
return result
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
// ------------------------------------------------- 文档(SourceRaw)列表 / 详情
|
|
1423
|
+
|
|
1424
|
+
/**
|
|
1425
|
+
* Absolute path a SourceRaw document is written to.
|
|
1426
|
+
*
|
|
1427
|
+
* 0030:段内按来源分类分目录。`sourceType` 在调用点已经过了注册表闸门
|
|
1428
|
+
* (`addSource` 的 `sourceTypeGate`),所以它一定匹配 `SOURCE_TYPE_ID_PATTERN`,
|
|
1429
|
+
* 不会引入路径穿越。
|
|
1430
|
+
*/
|
|
1431
|
+
private sourcePath(sourceId: string, sourceType: string): string {
|
|
1432
|
+
return this.safePath(SOURCE_TRUTH, sourceType, `${slug(sourceId)}.md`)
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
/** 递归扫 SourceRaw 段,建 `docSlug → path` 索引;两种布局都覆盖。 */
|
|
1436
|
+
private async sourceIndexMap(): Promise<Map<string, string>> {
|
|
1437
|
+
if (this.sourceIndex) return this.sourceIndex
|
|
1438
|
+
const index = new Map<string, string>()
|
|
1439
|
+
for (const path of await filesUnder(this.directory(SOURCE_TRUTH))) index.set(basename(path, '.md'), path)
|
|
1440
|
+
this.sourceIndex = index
|
|
1441
|
+
return index
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
/** 已知文档 id 的落盘位置;不存在时 `undefined`(调用方各自决定怎么报错)。 */
|
|
1445
|
+
private async sourcePathOf(docId: string): Promise<string | undefined> {
|
|
1446
|
+
return (await this.sourceIndexMap()).get(slug(docId))
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
/**
|
|
1450
|
+
* Assets a document really has on disk.
|
|
1451
|
+
*
|
|
1452
|
+
* Preferred source of truth is `metadata.assets` (knowledge-root-relative
|
|
1453
|
+
* paths written by `convert/ingest.ts`). Documents written by hand fall back
|
|
1454
|
+
* to parsing local markdown image links. A link is resolved **knowledge-root
|
|
1455
|
+
* first, then relative to the document's own directory**: the convert ingest
|
|
1456
|
+
* seam re-bases assets to `_assets/<bucket>/<name>` (root-relative), while
|
|
1457
|
+
* hand-written docs typically use a sibling path. Only existing files are
|
|
1458
|
+
* returned, so `assetCount` is never an over-count.
|
|
1459
|
+
*/
|
|
1460
|
+
private async assetsOf(path: string, content: string, metadata: Record<string, unknown>): Promise<KnowledgeAssetRef[]> {
|
|
1461
|
+
const refs = new Set<string>()
|
|
1462
|
+
if (Array.isArray(metadata.assets)) for (const item of metadata.assets) if (typeof item === 'string' && item.trim()) refs.add(item.trim())
|
|
1463
|
+
for (const ref of markdownImageRefs(content)) refs.add(ref)
|
|
1464
|
+
const root = this.rootDir
|
|
1465
|
+
const insideRoot = (value: string): boolean => value === root || value.startsWith(root.endsWith(sep) ? root : `${root}${sep}`)
|
|
1466
|
+
const assets: KnowledgeAssetRef[] = []
|
|
1467
|
+
const seen = new Set<string>()
|
|
1468
|
+
for (const ref of refs) {
|
|
1469
|
+
const candidates = isAbsolute(ref) ? [resolve(ref)] : [resolve(root, ref), resolve(dirname(path), ref)]
|
|
1470
|
+
for (const absolute of candidates) {
|
|
1471
|
+
if (!insideRoot(absolute)) continue
|
|
1472
|
+
let details
|
|
1473
|
+
try { details = await stat(absolute) } catch { continue }
|
|
1474
|
+
if (!details.isFile()) continue
|
|
1475
|
+
const rel = relative(root, absolute).split(sep).join('/')
|
|
1476
|
+
if (seen.has(rel)) break
|
|
1477
|
+
seen.add(rel)
|
|
1478
|
+
const encoded = rel.split('/').map(segment => encodeURIComponent(segment)).join('/')
|
|
1479
|
+
assets.push({ rel, bytes: details.size, url: `/knowledge/assets/${encodeURIComponent(this.baseId)}/${encoded}` })
|
|
1480
|
+
break
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
return assets.sort((left, right) => left.rel.localeCompare(right.rel))
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
/** Parse one on-disk SourceRaw file into the detail projection. */
|
|
1487
|
+
private async readDocumentFile(path: string): Promise<KnowledgeDocumentDetail | undefined> {
|
|
1488
|
+
let raw: string
|
|
1489
|
+
try { raw = await readFile(path, 'utf8') } catch { return undefined }
|
|
1490
|
+
const parsed = parseMarkdown(raw)
|
|
1491
|
+
const sourceId = String(parsed.frontmatter.source_id ?? basename(path, '.md'))
|
|
1492
|
+
const sourceType = String(parsed.frontmatter.source_type ?? 'document')
|
|
1493
|
+
// 未知分类照样投影出来(0013);前端按 sourceTypeKnown 渲染「未知分类」徽标。
|
|
1494
|
+
const sourceTypeKnown = this.sourceTypes().some(def => def.id === sourceType)
|
|
1495
|
+
const metadata = plainRecord(parsed.frontmatter.metadata)
|
|
1496
|
+
const ext = deriveSourceExt(metadata)
|
|
1497
|
+
const assets = await this.assetsOf(path, parsed.content, metadata)
|
|
1498
|
+
const lineage = parseLineage(parsed.frontmatter.lineage)
|
|
1499
|
+
return {
|
|
1500
|
+
docId: sourceId,
|
|
1501
|
+
title: String(parsed.frontmatter.title ?? sourceId),
|
|
1502
|
+
sourceType,
|
|
1503
|
+
sourceTypeKnown,
|
|
1504
|
+
sourceFormat: sourceFormatOf(metadata),
|
|
1505
|
+
/**
|
|
1506
|
+
* `parser` 说的是**谁解析了这份文档**,不是「它本该被谁解析」。
|
|
1507
|
+
*
|
|
1508
|
+
* 0029 之前这里回退到 `engineForExt(ext)` —— 按扩展名查一张表,猜一个引擎名。
|
|
1509
|
+
* 那是在**编造一个没有发生过的事实**:直接写进来的 SourceRaw(`add_source`、
|
|
1510
|
+
* 浏览器直读文本)根本没有解析器,而猜出来的 `anydoc` 会让人以为它经过了转换
|
|
1511
|
+
* 内核。现在如实写 `direct`:没有解析器。
|
|
1512
|
+
*
|
|
1513
|
+
* 经转换内核进来的文档由 `convert/ingest.ts` 写 `metadata.parser`
|
|
1514
|
+
* (如 `anydoc@1/pdf`),那才是真话。
|
|
1515
|
+
*/
|
|
1516
|
+
parser: typeof metadata.parser === 'string' && metadata.parser.trim() ? metadata.parser.trim() : 'direct',
|
|
1517
|
+
externalId: typeof parsed.frontmatter.external_id === 'string' ? parsed.frontmatter.external_id : '',
|
|
1518
|
+
contentHash: contentHashOf(parsed.content),
|
|
1519
|
+
bytes: Buffer.byteLength(raw, 'utf8'),
|
|
1520
|
+
assetCount: assets.length,
|
|
1521
|
+
createdAt: String(parsed.frontmatter.created_at ?? ''),
|
|
1522
|
+
updatedAt: String(parsed.frontmatter.updated_at ?? ''),
|
|
1523
|
+
versionCount: (await this.versionIds(sourceId)).length + 1,
|
|
1524
|
+
relativePath: relative(this.rootDir, path).split(sep).join('/'),
|
|
1525
|
+
ext,
|
|
1526
|
+
content: parsed.content,
|
|
1527
|
+
frontmatter: parsed.frontmatter,
|
|
1528
|
+
assets,
|
|
1529
|
+
...(lineage ? { lineage: lineage as Record<string, string> } : {}),
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
|
|
1533
|
+
/** 文档列表(新→旧),正文/资产明细留在详情接口里。 */
|
|
1534
|
+
async listDocuments(): Promise<KnowledgeDocument[]> {
|
|
1535
|
+
const documents: KnowledgeDocument[] = []
|
|
1536
|
+
for (const path of await filesUnder(this.directory(SOURCE_TRUTH))) {
|
|
1537
|
+
const detail = await this.readDocumentFile(path)
|
|
1538
|
+
if (!detail) continue
|
|
1539
|
+
documents.push({
|
|
1540
|
+
docId: detail.docId, title: detail.title, sourceType: detail.sourceType, sourceTypeKnown: detail.sourceTypeKnown,
|
|
1541
|
+
sourceFormat: detail.sourceFormat,
|
|
1542
|
+
parser: detail.parser, externalId: detail.externalId, contentHash: detail.contentHash, bytes: detail.bytes,
|
|
1543
|
+
assetCount: detail.assetCount, createdAt: detail.createdAt, updatedAt: detail.updatedAt,
|
|
1544
|
+
versionCount: detail.versionCount, relativePath: detail.relativePath, ext: detail.ext,
|
|
1545
|
+
})
|
|
1546
|
+
}
|
|
1547
|
+
return documents.sort((left, right) => right.updatedAt.localeCompare(left.updatedAt) || left.docId.localeCompare(right.docId))
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
/** 文档详情:正文、frontmatter、资产与 lineage。 */
|
|
1551
|
+
async readDocument(docId: string): Promise<KnowledgeDocumentDetail | undefined> {
|
|
1552
|
+
const path = await this.sourcePathOf(docId)
|
|
1553
|
+
return path ? this.readDocumentFile(path) : undefined
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
/**
|
|
1557
|
+
* 版本历史,新→旧。
|
|
1558
|
+
*
|
|
1559
|
+
* 第一条永远是当前 head(`isCurrent: true`,`versionId` 形如
|
|
1560
|
+
* `current-<hash8>`),其后是 `04_memory/source-versions/<docSlug>/` 里的
|
|
1561
|
+
* 历史快照。快照保存的是**被覆盖前的原文件字节**,因此回滚是字节级的。
|
|
1562
|
+
*/
|
|
1563
|
+
async listVersions(docId: string): Promise<DocumentVersion[]> {
|
|
1564
|
+
const path = await this.sourcePathOf(docId)
|
|
1565
|
+
if (!path) return []
|
|
1566
|
+
let raw: string
|
|
1567
|
+
try { raw = await readFile(path, 'utf8') } catch { return [] }
|
|
1568
|
+
const head = parseMarkdown(raw)
|
|
1569
|
+
const headHash = contentHashOf(head.content)
|
|
1570
|
+
const versions: DocumentVersion[] = [{
|
|
1571
|
+
versionId: `current-${headHash.slice(0, 8)}`,
|
|
1572
|
+
createdAt: String(head.frontmatter.updated_at ?? ''),
|
|
1573
|
+
contentHash: headHash,
|
|
1574
|
+
bytes: Buffer.byteLength(raw, 'utf8'),
|
|
1575
|
+
sourceFormat: sourceFormatOf(plainRecord(head.frontmatter.metadata)),
|
|
1576
|
+
label: '当前版本',
|
|
1577
|
+
isCurrent: true,
|
|
1578
|
+
}]
|
|
1579
|
+
for (const versionId of await this.versionIds(docId)) {
|
|
1580
|
+
try {
|
|
1581
|
+
const snapshotRaw = await readFile(join(this.versionDir(docId), `${versionId}.md`), 'utf8')
|
|
1582
|
+
const parsed = parseMarkdown(snapshotRaw)
|
|
1583
|
+
const createdAt = String(parsed.frontmatter.updated_at ?? '')
|
|
1584
|
+
versions.push({
|
|
1585
|
+
versionId,
|
|
1586
|
+
createdAt,
|
|
1587
|
+
contentHash: contentHashOf(parsed.content),
|
|
1588
|
+
bytes: Buffer.byteLength(snapshotRaw, 'utf8'),
|
|
1589
|
+
sourceFormat: sourceFormatOf(plainRecord(parsed.frontmatter.metadata)),
|
|
1590
|
+
label: `历史版本 · ${createdAt || versionId}`,
|
|
1591
|
+
isCurrent: false,
|
|
1592
|
+
})
|
|
1593
|
+
} catch { /* snapshot vanished between readdir and read */ }
|
|
1594
|
+
}
|
|
1595
|
+
return versions
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
/**
|
|
1599
|
+
* 回滚到某个历史版本。
|
|
1600
|
+
*
|
|
1601
|
+
* 语义:用快照内容重新调用 `addSource`。因为 `addSource` 在覆盖前会把当前
|
|
1602
|
+
* head 再快照一次,所以**回滚本身也可回滚**,历史只增不减。
|
|
1603
|
+
* `current-<hash8>` 指向当前 head,属于 no-op,直接返回当前详情。
|
|
1604
|
+
*/
|
|
1605
|
+
async revertDocument(docId: string, versionId: string): Promise<Envelope<KnowledgeDocumentDetail>> {
|
|
1606
|
+
const path = await this.sourcePathOf(docId)
|
|
1607
|
+
if (path === undefined) return failure(emptyDocumentDetail(docId), 'DOCUMENT_NOT_FOUND', `文档不存在: ${docId}`, true)
|
|
1608
|
+
const head = await this.readDocumentFile(path)
|
|
1609
|
+
if (!head) return failure(emptyDocumentDetail(docId), 'DOCUMENT_NOT_FOUND', `文档不存在: ${docId}`, true)
|
|
1610
|
+
const target = typeof versionId === 'string' ? versionId.trim() : ''
|
|
1611
|
+
if (!target) return failure(head, 'VERSION_ID_REQUIRED', 'versionId 不能为空', false, '用 list_versions / GET /knowledge/documents/:docId/versions 先取版本列表。')
|
|
1612
|
+
if (target.startsWith('current-')) return success(head)
|
|
1613
|
+
const snapshotPath = join(this.versionDir(docId), `${basename(target)}.md`)
|
|
1614
|
+
if (!existsSync(snapshotPath)) return failure(head, 'VERSION_NOT_FOUND', `版本不存在: ${target}`, true)
|
|
1615
|
+
const parsed = parseMarkdown(await readFile(snapshotPath, 'utf8'))
|
|
1616
|
+
const restored = await this.addSource({
|
|
1617
|
+
sourceType: (String(parsed.frontmatter.source_type ?? 'document')) as SourceType,
|
|
1618
|
+
title: typeof parsed.frontmatter.title === 'string' ? parsed.frontmatter.title : undefined,
|
|
1619
|
+
content: parsed.content,
|
|
1620
|
+
externalId: typeof parsed.frontmatter.external_id === 'string' && parsed.frontmatter.external_id ? parsed.frontmatter.external_id : undefined,
|
|
1621
|
+
metadata: plainRecord(parsed.frontmatter.metadata),
|
|
1622
|
+
sourceId: docId,
|
|
1623
|
+
lineage: parseLineage(parsed.frontmatter.lineage),
|
|
1624
|
+
})
|
|
1625
|
+
if (!restored.ok) return failure(head, restored.error?.code ?? 'REVERT_FAILED', restored.error?.message ?? '回滚写入失败', restored.error?.retryable ?? false, restored.error?.suggestion)
|
|
1626
|
+
const document = await this.readDocumentFile(path)
|
|
1627
|
+
if (!document) return failure(head, 'DOCUMENT_NOT_FOUND', `回滚后文档不可读: ${docId}`, true)
|
|
1628
|
+
await this.audit('source_truth.revert', { docId, versionId: target })
|
|
1629
|
+
return success(document)
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
/** 删除文档本体与其版本历史(重新写入同一 sourceId 会从头开始记版本)。 */
|
|
1633
|
+
async deleteDocument(docId: string): Promise<Envelope<{ deleted: true }>> {
|
|
1634
|
+
const path = await this.sourcePathOf(docId)
|
|
1635
|
+
if (path === undefined) return failure({ deleted: true }, 'DOCUMENT_NOT_FOUND', `文档不存在: ${docId}`, true)
|
|
1636
|
+
await unlink(path)
|
|
1637
|
+
this.sourceIndex?.delete(slug(docId))
|
|
1638
|
+
this.sourceCorpusCache = undefined
|
|
1639
|
+
await rm(this.versionDir(docId), { recursive: true, force: true })
|
|
1640
|
+
await this.audit('source_truth.delete', { docId })
|
|
1641
|
+
return success({ deleted: true })
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
async writeCard(input: { entityType: SchemaType; entityId: string; title: string; content: string; sourceRefs: string[]; status?: KnowledgeStatus; summary?: string; /**
|
|
1645
|
+
* 负责人。它属于 0021 §6.4 的卡片信封(智能归因的五类卡片里 `check.owner` 是必填 ——
|
|
1646
|
+
* 「这一步该找谁」和成本、判定阈值一样是执行信息);其余 entity_type 不使用它。
|
|
1647
|
+
*/
|
|
1648
|
+
owner?: string; metadata?: Record<string, unknown>; lineage?: LineageContext; provenance?: LineageRef[] }): Promise<Envelope<{ entityId: string; path: string; lineage?: LineageContext }>> {
|
|
1649
|
+
if (!SAFE_ID.test(input.entityId)) return failure({ entityId: input.entityId, path: '' }, 'ENTITY_ID_INVALID', 'entity_id 必须是稳定且安全的标识符', false)
|
|
1650
|
+
if (!input.title?.trim() || !input.content?.trim()) return failure({ entityId: input.entityId, path: '' }, 'CARD_CONTENT_REQUIRED', 'title 和 content 不能为空', true)
|
|
1651
|
+
if (!Array.isArray(input.sourceRefs) || input.sourceRefs.length === 0) return failure({ entityId: input.entityId, path: '' }, 'SOURCE_REFS_REQUIRED', '知识卡片必须引用 SourceTruth', false)
|
|
1652
|
+
const sourceIds = new Set((await this.loadSources()).map(source => source.sourceId))
|
|
1653
|
+
const missingRefs = input.sourceRefs.filter(sourceRef => !sourceIds.has(sourceRef))
|
|
1654
|
+
if (missingRefs.length > 0) return failure({ entityId: input.entityId, path: '' }, 'SOURCE_REF_NOT_FOUND', `source_refs 引用不存在: ${missingRefs.join(', ')}`, false, '先调用 add_source 写入对应的 SourceTruth')
|
|
1655
|
+
const status = input.status ?? 'unreviewed'
|
|
1656
|
+
if (!STATUS.includes(status)) return failure({ entityId: input.entityId, path: '' }, 'STATUS_INVALID', 'status 不合法', false)
|
|
1657
|
+
// 0024:类型闸门 + 三层校验(format / reference / semantic)。
|
|
1658
|
+
const gate = this.cardSchemaGate({
|
|
1659
|
+
entityType: input.entityType, title: input.title.trim(), content: input.content,
|
|
1660
|
+
sourceRefs: input.sourceRefs, metadata: input.metadata ?? {}, ...(input.owner ? { owner: input.owner } : {}), ...(input.summary ? { summary: input.summary } : {}),
|
|
1661
|
+
})
|
|
1662
|
+
if ('error' in gate) return failure({ entityId: input.entityId, path: '' }, gate.error.code, gate.error.message, gate.error.retryable, gate.error.suggestion)
|
|
1663
|
+
const { issues } = gate
|
|
1664
|
+
const warnings = issues.map(issue => `${issue.level}: ${issue.message}`)
|
|
1665
|
+
if (issues.length > 0 && this.wikiSchema().mode === 'strict') {
|
|
1666
|
+
return failure(
|
|
1667
|
+
{ entityId: input.entityId, path: '' },
|
|
1668
|
+
'CARD_SCHEMA_INVALID',
|
|
1669
|
+
`卡片不符合「${gate.type.label}」的 schema:${issues.map(issue => issue.message).join(';')}`,
|
|
1670
|
+
false,
|
|
1671
|
+
'补齐必填小节 / 字段后重试;或在「知识库设置 → Wiki 类型」里把生效方式改为 report(只标记不拦截)。',
|
|
1672
|
+
)
|
|
1673
|
+
}
|
|
1674
|
+
await this.ensureLayout()
|
|
1675
|
+
const path = this.safePath(WIKI, input.entityType, `${slug(input.entityId)}.md`)
|
|
1676
|
+
await mkdir(dirname(path), { recursive: true })
|
|
1677
|
+
const updatedAt = new Date().toISOString()
|
|
1678
|
+
// 卡片和 SourceRaw 一样先快照旧版本再覆盖。判据是**正文**:只改 status 或
|
|
1679
|
+
// provenance(setStatus 就是这么写的)不会凭空造出一个"内容版本",否则
|
|
1680
|
+
// 「审核一次 = 多一个版本」,版本列表会被状态流转灌满。
|
|
1681
|
+
let createdAt = updatedAt
|
|
1682
|
+
let previousRaw: string | undefined
|
|
1683
|
+
try { previousRaw = await readFile(path, 'utf8') } catch { /* 首次写入这张卡片 */ }
|
|
1684
|
+
if (previousRaw !== undefined) {
|
|
1685
|
+
const previous = parseMarkdown(previousRaw)
|
|
1686
|
+
createdAt = String(previous.frontmatter.created_at ?? createdAt)
|
|
1687
|
+
if (previous.content.trim() !== input.content.trim()) {
|
|
1688
|
+
await this.snapshotCard(input.entityType, input.entityId, previousRaw, previous.content)
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
const provenance = [...(input.provenance ?? []), ...(input.lineage ? [{ ...input.lineage, role: 'created' as const, refType: 'knowledge_card', refId: input.entityId }] : [])]
|
|
1692
|
+
// report 模式:把问题留在卡片上,界面据此显示"待补",而不是静默通过。
|
|
1693
|
+
const metadata = { ...(input.metadata ?? {}), ...(issues.length > 0 ? { schema_issues: issues } : {}) }
|
|
1694
|
+
await writeTextAtomic(path, markdown({ entity_id: input.entityId, entity_type: input.entityType, title: input.title.trim(), status, summary: input.summary ?? '', ...(input.owner ? { owner: input.owner } : {}), source_refs: input.sourceRefs, created_at: createdAt, updated_at: updatedAt, metadata, ...(provenance.length > 0 ? { provenance } : {}) }, input.content))
|
|
1695
|
+
return success({ entityId: input.entityId, path, ...(input.lineage ? { lineage: input.lineage } : {}) }, warnings)
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
/** Directory holding one wiki card's version snapshots. */
|
|
1699
|
+
private cardVersionDir(entityType: SchemaType, entityId: string): string {
|
|
1700
|
+
return this.memoryPath(CARD_VERSIONS, slug(entityType), slug(entityId))
|
|
1701
|
+
}
|
|
1702
|
+
|
|
1703
|
+
/** Write the previous card file bytes as one immutable version snapshot. */
|
|
1704
|
+
private async snapshotCard(entityType: SchemaType, entityId: string, raw: string, body: string): Promise<string> {
|
|
1705
|
+
const dir = this.cardVersionDir(entityType, entityId)
|
|
1706
|
+
await mkdir(dir, { recursive: true })
|
|
1707
|
+
const versionId = `${compactIso(new Date())}-${contentHashOf(body).slice(0, 8)}`
|
|
1708
|
+
const path = join(dir, `${versionId}.md`)
|
|
1709
|
+
if (!existsSync(path)) await writeTextAtomic(path, raw)
|
|
1710
|
+
return versionId
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
/** Snapshot basenames (without `.md`) for one card, newest first. */
|
|
1714
|
+
private async cardVersionIds(entityType: SchemaType, entityId: string): Promise<string[]> {
|
|
1715
|
+
try {
|
|
1716
|
+
const entries = await readdir(this.cardVersionDir(entityType, entityId), { withFileTypes: true })
|
|
1717
|
+
return entries.filter(entry => entry.isFile() && entry.name.endsWith('.md')).map(entry => basename(entry.name, '.md')).sort().reverse()
|
|
1718
|
+
} catch { return [] }
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
/** Resolve one card by id, optionally constrained to a type (ids are unique per type). */
|
|
1722
|
+
async readCard(entityId: string, entityType?: SchemaType): Promise<KnowledgeCard | undefined> {
|
|
1723
|
+
const cards = await this.loadCards()
|
|
1724
|
+
return cards.find(card => card.entityId === entityId && (entityType === undefined || card.entityType === entityType))
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
/**
|
|
1728
|
+
* Version list for one card: the newest snapshot reported as current *plus* the
|
|
1729
|
+
* live head. The head is not itself a snapshot — it is the file on disk — but a
|
|
1730
|
+
* version list that omits what you are looking at is unreadable.
|
|
1731
|
+
*/
|
|
1732
|
+
async listCardVersions(entityId: string, entityType?: SchemaType): Promise<WikiCardVersion[]> {
|
|
1733
|
+
const card = await this.readCard(entityId, entityType)
|
|
1734
|
+
if (!card) return []
|
|
1735
|
+
const ids = await this.cardVersionIds(card.entityType, entityId)
|
|
1736
|
+
const versions: WikiCardVersion[] = []
|
|
1737
|
+
for (const versionId of ids) {
|
|
1738
|
+
try {
|
|
1739
|
+
const raw = await readFile(join(this.cardVersionDir(card.entityType, entityId), `${versionId}.md`), 'utf8')
|
|
1740
|
+
const parsed = parseMarkdown(raw)
|
|
1741
|
+
versions.push({
|
|
1742
|
+
versionId,
|
|
1743
|
+
// `<compactIso>` is `YYYYMMDDThhmmssSSSZ`; expand it back to a real ISO
|
|
1744
|
+
// string so the panel formats it like every other timestamp.
|
|
1745
|
+
createdAt: expandCompactIso(versionId.slice(0, versionId.lastIndexOf('-'))) || card.updatedAt,
|
|
1746
|
+
contentHash: contentHashOf(parsed.content),
|
|
1747
|
+
bytes: Buffer.byteLength(raw, 'utf8'),
|
|
1748
|
+
title: String(parsed.frontmatter.title ?? entityId),
|
|
1749
|
+
status: String(parsed.frontmatter.status ?? 'draft'),
|
|
1750
|
+
label: `V${ids.length - versions.length + 1} · ${String(parsed.frontmatter.title ?? entityId)}`,
|
|
1751
|
+
isCurrent: false,
|
|
1752
|
+
})
|
|
1753
|
+
} catch { /* 快照在 readdir 与 read 之间消失了 */ }
|
|
1754
|
+
}
|
|
1755
|
+
versions.unshift({
|
|
1756
|
+
versionId: 'head',
|
|
1757
|
+
createdAt: card.updatedAt || new Date().toISOString(),
|
|
1758
|
+
contentHash: contentHashOf(card.content),
|
|
1759
|
+
bytes: Buffer.byteLength(card.content, 'utf8'),
|
|
1760
|
+
title: card.title,
|
|
1761
|
+
status: card.status,
|
|
1762
|
+
label: `当前 · ${card.title}`,
|
|
1763
|
+
isCurrent: true,
|
|
1764
|
+
})
|
|
1765
|
+
return versions
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
/** 回滚一张卡片到历史快照;写入走 writeCard,所以**回滚前的当前版本也会被快照**。 */
|
|
1769
|
+
async revertCard(entityId: string, versionId: string, entityType?: SchemaType): Promise<Envelope<KnowledgeCard>> {
|
|
1770
|
+
const card = await this.readCard(entityId, entityType)
|
|
1771
|
+
if (!card) return failure({} as KnowledgeCard, 'CARD_NOT_FOUND', `知识卡片不存在: ${entityId}`, true)
|
|
1772
|
+
const target = versionId.replace(/\.md$/, '')
|
|
1773
|
+
// 「当前版本」先判:它没有快照文件,先查文件会得到「版本不存在」,那是在
|
|
1774
|
+
// 用错误的原因回答一个合理的请求。
|
|
1775
|
+
if (String(target) === 'head') return failure(card, 'VERSION_IS_HEAD', '当前版本不能作为回滚目标', false, '从版本列表里选一个历史快照。')
|
|
1776
|
+
const snapshotPath = join(this.cardVersionDir(card.entityType, entityId), `${target}.md`)
|
|
1777
|
+
if (!existsSync(snapshotPath)) return failure(card, 'VERSION_NOT_FOUND', `版本不存在: ${target}`, true)
|
|
1778
|
+
const parsed = parseMarkdown(await readFile(snapshotPath, 'utf8'))
|
|
1779
|
+
const refs = Array.isArray(parsed.frontmatter.source_refs) ? parsed.frontmatter.source_refs.map(String) : card.sourceRefs
|
|
1780
|
+
const status = STATUS.includes(String(parsed.frontmatter.status) as KnowledgeStatus) ? String(parsed.frontmatter.status) as KnowledgeStatus : card.status
|
|
1781
|
+
const restored = await this.writeCard({
|
|
1782
|
+
entityType: card.entityType,
|
|
1783
|
+
entityId,
|
|
1784
|
+
title: String(parsed.frontmatter.title ?? card.title),
|
|
1785
|
+
content: parsed.content,
|
|
1786
|
+
sourceRefs: refs,
|
|
1787
|
+
status,
|
|
1788
|
+
summary: typeof parsed.frontmatter.summary === 'string' ? parsed.frontmatter.summary : card.summary,
|
|
1789
|
+
metadata: typeof parsed.frontmatter.metadata === 'object' && parsed.frontmatter.metadata ? parsed.frontmatter.metadata as Record<string, unknown> : card.metadata,
|
|
1790
|
+
})
|
|
1791
|
+
if (!restored.ok) return failure(card, restored.error?.code ?? 'REVERT_FAILED', restored.error?.message ?? '回滚写入失败', restored.error?.retryable ?? false, restored.error?.suggestion)
|
|
1792
|
+
const next = await this.readCard(entityId, card.entityType)
|
|
1793
|
+
if (!next) return failure(card, 'CARD_NOT_FOUND', `回滚后卡片不可读: ${entityId}`, true)
|
|
1794
|
+
await this.audit('wiki.revert', { entityId, entityType: card.entityType, versionId: target })
|
|
1795
|
+
return success(next)
|
|
1796
|
+
}
|
|
1797
|
+
|
|
1798
|
+
async loadCards(): Promise<KnowledgeCard[]> {
|
|
1799
|
+
const result: KnowledgeCard[] = []
|
|
1800
|
+
for (const path of await filesUnder(this.directory(WIKI))) {
|
|
1801
|
+
const parsed = parseMarkdown(await readFile(path, 'utf8'))
|
|
1802
|
+
const entityId = String(parsed.frontmatter.entity_id ?? basename(path, '.md'))
|
|
1803
|
+
const entityType = String(parsed.frontmatter.entity_type ?? relative(this.directory(WIKI), dirname(path)).split(sep)[0]) as SchemaType
|
|
1804
|
+
// 0024:未知类型**不再静默丢弃**。类型只是元数据,丢掉卡片才是真正的数据损失;
|
|
1805
|
+
// 未知类型在界面上显示成「未登记类型」,由人决定是补登记还是改卡片。
|
|
1806
|
+
const refs = Array.isArray(parsed.frontmatter.source_refs) ? parsed.frontmatter.source_refs.map(String) : []
|
|
1807
|
+
result.push({ entityId, entityType, title: String(parsed.frontmatter.title ?? entityId), status: STATUS.includes(String(parsed.frontmatter.status) as KnowledgeStatus) ? String(parsed.frontmatter.status) as KnowledgeStatus : 'draft', summary: typeof parsed.frontmatter.summary === 'string' ? parsed.frontmatter.summary : undefined, content: parsed.content, sourceRefs: refs, ...(parseProvenance(parsed.frontmatter.provenance) ? { provenance: parseProvenance(parsed.frontmatter.provenance) } : {}), metadata: typeof parsed.frontmatter.metadata === 'object' && parsed.frontmatter.metadata ? parsed.frontmatter.metadata as Record<string, unknown> : {}, updatedAt: String(parsed.frontmatter.updated_at ?? ''), imagePaths: markdownImagePaths(parsed.content, dirname(path)) })
|
|
1808
|
+
}
|
|
1809
|
+
return result
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
async setStatus(entityId: string, status: KnowledgeStatus, problemNote?: string, lineage?: LineageContext): Promise<Envelope<{ entityId: string; status: KnowledgeStatus; lineage?: LineageContext }>> {
|
|
1813
|
+
if (!SAFE_ID.test(entityId) || !STATUS.includes(status)) return failure({ entityId, status }, 'STATUS_INPUT_INVALID', 'entity_id 或 status 不合法', false)
|
|
1814
|
+
const cards = await this.loadCards()
|
|
1815
|
+
const card = cards.find(item => item.entityId === entityId)
|
|
1816
|
+
if (!card) return failure({ entityId, status }, 'CARD_NOT_FOUND', '知识卡片不存在', true)
|
|
1817
|
+
if (status === 'problem' && !problemNote?.trim()) return failure({ entityId, status }, 'PROBLEM_NOTE_REQUIRED', 'status=problem 时必须提供 problem_note', false)
|
|
1818
|
+
if (card.status === 'problem' && status === 'reviewed') return failure({ entityId, status }, 'STATUS_TRANSITION_INVALID', 'problem 不能直接恢复为 reviewed,请先改为 unreviewed', false)
|
|
1819
|
+
const eventLineage = withRun(lineage, 'knowledge-status')
|
|
1820
|
+
const provenance = [...(card.provenance ?? []), ...(eventLineage ? [{ ...eventLineage, role: 'executed' as const, refType: 'knowledge_status', refId: entityId }] : [])]
|
|
1821
|
+
await this.writeCard({ ...card, status, sourceRefs: card.sourceRefs, provenance, metadata: { ...card.metadata, ...(problemNote?.trim() ? { problem_note: problemNote.trim() } : {}) } })
|
|
1822
|
+
return success({ entityId, status, ...(eventLineage ? { lineage: eventLineage } : {}) })
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
/**
|
|
1826
|
+
* Stage 2: compile wiki cards into wikiNotes.
|
|
1827
|
+
*
|
|
1828
|
+
* A note is the unit stage 3 recalls and stage 4 reranks. Every note keeps a
|
|
1829
|
+
* `wikiId` pointer so recall can always resolve the authoritative wiki card,
|
|
1830
|
+
* and the first note of a wiki keeps the historical `<type>:<id>` unitId.
|
|
1831
|
+
*
|
|
1832
|
+
* Split per card (not per corpus) so the incremental rebuild can recompile one
|
|
1833
|
+
* card's notes while keeping every other card's `unitId`s — and therefore their
|
|
1834
|
+
* vectors — exactly as they were.
|
|
1835
|
+
*/
|
|
1836
|
+
private compileNotes(cards: KnowledgeCard[], sourceTypes: Map<string, SourceType>): WikiNote[] {
|
|
1837
|
+
return cards.flatMap(card => this.compileCardNotes(card, sourceTypes))
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
private compileCardNotes(card: KnowledgeCard, sourceTypes: Map<string, SourceType>): WikiNote[] {
|
|
1841
|
+
if (card.status === 'problem') return []
|
|
1842
|
+
const notes: WikiNote[] = []
|
|
1843
|
+
const sourceTypesFor = card.sourceRefs.flatMap(ref => { const sourceType = sourceTypes.get(ref); return sourceType ? [sourceType] : [] })
|
|
1844
|
+
for (const [index, chunk] of splitWikiNotes(card).entries()) {
|
|
1845
|
+
const baseId = `${card.entityType}:${card.entityId}`
|
|
1846
|
+
const unitId = index === 0 ? baseId : `${baseId}#${index}`
|
|
1847
|
+
notes.push({
|
|
1848
|
+
unitId,
|
|
1849
|
+
noteId: unitId,
|
|
1850
|
+
entityId: card.entityId,
|
|
1851
|
+
wikiId: card.entityId,
|
|
1852
|
+
entityType: card.entityType,
|
|
1853
|
+
title: chunk.heading ? `${card.title} · ${chunk.heading}` : card.title,
|
|
1854
|
+
...(chunk.heading ? { heading: chunk.heading } : {}),
|
|
1855
|
+
content: chunk.content,
|
|
1856
|
+
sourceRefs: card.sourceRefs,
|
|
1857
|
+
sourceTypes: sourceTypesFor,
|
|
1858
|
+
...(card.provenance ? { provenance: card.provenance } : {}),
|
|
1859
|
+
status: card.status,
|
|
1860
|
+
...(card.imagePaths?.length ? { imagePaths: card.imagePaths } : {}),
|
|
1861
|
+
})
|
|
1862
|
+
}
|
|
1863
|
+
return notes
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
/** 1-hop neighbourhood of a set of wiki cards inside the compiled graph. */
|
|
1867
|
+
private oneHopGraph(keys: Set<string>, includeProblem: boolean): ContextGraph {
|
|
1868
|
+
const visible = (node: Record<string, unknown>) => includeProblem || node.status !== 'problem'
|
|
1869
|
+
const edges = this.graph.edges.filter(edge => keys.has(String(edge.from)) || keys.has(String(edge.to))).slice(0, 50)
|
|
1870
|
+
const ids = new Set(keys)
|
|
1871
|
+
for (const edge of edges) { ids.add(String(edge.from)); ids.add(String(edge.to)) }
|
|
1872
|
+
const nodes = this.graph.nodes
|
|
1873
|
+
.filter(visible)
|
|
1874
|
+
.filter(node => ids.has(`${node.type}:${node.id}`))
|
|
1875
|
+
.slice(0, 50)
|
|
1876
|
+
return { nodes, edges }
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
/**
|
|
1880
|
+
* Stage 4: BERT cross-encoder rerank over the recalled wikiNotes.
|
|
1881
|
+
*
|
|
1882
|
+
* Mirrors the embedding path: a missing model or worker degrades to the
|
|
1883
|
+
* fused score order with an explicit warning instead of faking a score.
|
|
1884
|
+
*
|
|
1885
|
+
* 0031 的两处改动:
|
|
1886
|
+
* 1. **作用在候选池上**,不再是已经截断到 `recallTopK` 的那 10 条 —— 否则重排只能
|
|
1887
|
+
* 换顺序,救不回被融合排到窗口外的那一条(0014 点名的「分层只做了一半」)。
|
|
1888
|
+
* 2. **带阈值**:低于 `threshold` 的一律丢弃,并报出丢了几条。「弃答」主要靠它 ——
|
|
1889
|
+
* cross-encoder 的分是**跨查询可比**的,而关键词/向量的阈值做不到这一点。
|
|
1890
|
+
*
|
|
1891
|
+
* `threshold` 与 WeKnora 同刻度(**原始 logit**,默认 0.2);worker 用
|
|
1892
|
+
* `predict_raw` 保证这一点(`predict` 默认会套 sigmoid),`rerankScore` 是
|
|
1893
|
+
* sigmoid 之后的展示值,两者单调等价,排序不受影响。刻度换算集中在
|
|
1894
|
+
* `applyRerank`(纯函数,有测试),这里不再自己乘除。
|
|
1895
|
+
*/
|
|
1896
|
+
private async rerankNotes(query: string, hits: RecallHit[], threshold: number): Promise<{ hits: RecallHit[]; result: RerankResult; warning?: string }> {
|
|
1897
|
+
const provider = this.config.rerankProvider
|
|
1898
|
+
if (provider === 'none') return { hits, result: { status: 'skipped', provider, threshold } }
|
|
1899
|
+
if (hits.length < 2) return { hits, result: { status: 'skipped', provider, model: this.rerankModelPath, scored: hits.length, threshold } }
|
|
1900
|
+
if (this.rerankUnavailable) return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error: this.rerankUnavailable, threshold }, warning: `重排不可用,已按融合分数排序:${this.rerankUnavailable}` }
|
|
1901
|
+
try {
|
|
1902
|
+
if (!this.rerankWorker) this.rerankWorker = this.spawnWorker('RERANK')
|
|
1903
|
+
const response = await this.rerankWorker.request({
|
|
1904
|
+
task: 'rerank',
|
|
1905
|
+
model: this.rerankModelPath,
|
|
1906
|
+
device: this.config.embeddingDevice,
|
|
1907
|
+
query,
|
|
1908
|
+
documents: hits.map(hit => `${hit.title}\n\n${hit.content}`.slice(0, 8000)),
|
|
1909
|
+
}, this.config.rerankTimeoutMs)
|
|
1910
|
+
const raw: number[] = Array.isArray(response?.scores) ? (response.scores as unknown[]).map(Number) : []
|
|
1911
|
+
const outcome = applyRerank(raw, hits, threshold)
|
|
1912
|
+
const vocabSize = Number(response?.vocabSize)
|
|
1913
|
+
return {
|
|
1914
|
+
hits: outcome.kept,
|
|
1915
|
+
result: {
|
|
1916
|
+
status: 'ready', provider, model: this.rerankModelPath,
|
|
1917
|
+
scored: outcome.kept.length, threshold, dropped: outcome.dropped,
|
|
1918
|
+
topLogit: outcome.topLogit,
|
|
1919
|
+
...(Number.isFinite(vocabSize) && vocabSize > 0 ? { vocabSize } : {}),
|
|
1920
|
+
},
|
|
1921
|
+
}
|
|
1922
|
+
} catch (cause) {
|
|
1923
|
+
const error = cause instanceof Error ? cause.message : String(cause)
|
|
1924
|
+
// Drop the worker (a timeout already killed it) and remember the failure
|
|
1925
|
+
// so a later recall does not respawn a doomed model every time.
|
|
1926
|
+
this.rerankWorker?.close()
|
|
1927
|
+
this.rerankWorker = null
|
|
1928
|
+
this.rerankUnavailable = error
|
|
1929
|
+
return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error, threshold }, warning: `重排不可用,已按融合分数排序:${error}` }
|
|
1930
|
+
}
|
|
1931
|
+
}
|
|
1932
|
+
|
|
1933
|
+
/**
|
|
1934
|
+
* Fingerprint of everything a wikiNote is compiled *from*.
|
|
1935
|
+
*
|
|
1936
|
+
* Deliberately excludes the fields that are cheap to refresh on a reused note
|
|
1937
|
+
* (`provenance`, `imagePaths`, `metadata`) and includes the ones that change
|
|
1938
|
+
* what the note says (`title`, `status`, `content`, `sourceRefs`). A card whose
|
|
1939
|
+
* fingerprint is unchanged produces byte-identical notes, so reusing them is
|
|
1940
|
+
* not a shortcut — it is the same answer without the churn.
|
|
1941
|
+
*/
|
|
1942
|
+
private cardHashOf(card: KnowledgeCard): string {
|
|
1943
|
+
return contentHashOf(JSON.stringify({ title: card.title, status: card.status, content: card.content, sourceRefs: card.sourceRefs }))
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
/** Previous `wiki_notes.json` (or the pre-0014 mirror), without touching `this.units`. */
|
|
1947
|
+
private async readNotesArtifact(): Promise<WikiNote[]> {
|
|
1948
|
+
for (const artifact of [WIKI_NOTES, LEGACY_UNITS]) {
|
|
1949
|
+
try {
|
|
1950
|
+
const notes = JSON.parse(await readFile(this.safePath('03_compiled', artifact), 'utf8'))
|
|
1951
|
+
if (Array.isArray(notes) && notes.length > 0) return notes as WikiNote[]
|
|
1952
|
+
} catch { /* the first build has not happened yet */ }
|
|
1953
|
+
}
|
|
1954
|
+
return []
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
// ------------------------------------------------- Pass 0 · 候选抽取 + 归一
|
|
1958
|
+
|
|
1959
|
+
/**
|
|
1960
|
+
* 来源的抽取指纹。
|
|
1961
|
+
*
|
|
1962
|
+
* 三项缺一不可:
|
|
1963
|
+
* - **正文**:文档本身变了,抽出来的东西自然要重算;
|
|
1964
|
+
* - **声明式候选**:它们只写在 `metadata` 里(`SourceRaw` 只投影这一块),
|
|
1965
|
+
* 正文不变而声明变了是常见操作,不带上它就会漏抽;
|
|
1966
|
+
* - **启用的类型注册表**:用户新登记一个 wiki 类型后,同一份文档的判型结论会变。
|
|
1967
|
+
* 不把注册表算进指纹,「来源没变就永远不重抽」会把新类型永远锁在门外。
|
|
1968
|
+
*/
|
|
1969
|
+
private extractionFingerprint(source: SourceRaw, registryHash: string): string {
|
|
1970
|
+
const declared = ['candidates', 'entities', 'concepts', 'relations']
|
|
1971
|
+
.map(key => plainRecord(source.metadata)[key] ?? null)
|
|
1972
|
+
return contentHashOf(JSON.stringify({
|
|
1973
|
+
sourceId: source.sourceId,
|
|
1974
|
+
content: contentHashOf(source.content),
|
|
1975
|
+
updatedAt: source.updatedAt,
|
|
1976
|
+
declared: contentHashOf(JSON.stringify(declared)),
|
|
1977
|
+
registry: registryHash,
|
|
1978
|
+
}))
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
/** 一次 Pass 0 的真实结果:计数器 + 待装配的关系 + 来源指纹。 */
|
|
1982
|
+
private emptyExtractionOutcome(extractionHashes: Record<string, string>): ExtractionOutcome {
|
|
1983
|
+
return {
|
|
1984
|
+
extractionHashes,
|
|
1985
|
+
candidatesExtracted: 0, candidatesRejected: 0, cardsCreated: 0, cardsMerged: 0,
|
|
1986
|
+
sourcesExtracted: 0, sourcesSkipped: 0, relations: [], relationsBySource: {}, warnings: [],
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
|
|
1990
|
+
/**
|
|
1991
|
+
* Pass 0:对指纹变了的来源做候选抽取 + 归一,并把结果落成 wiki 卡片。
|
|
1992
|
+
*
|
|
1993
|
+
* 三条硬约束:
|
|
1994
|
+
* 1. **增量**:指纹没变的来源一次都不抽(这是 0025 增量承诺的一部分);
|
|
1995
|
+
* 2. **不重写没变的卡片**:合并只追加 `source_refs`,正文不动,所以不会凭空造
|
|
1996
|
+
* 版本快照(`snapshotCard` 只认正文变化);
|
|
1997
|
+
* 3. **fail-soft**:一个候选被闸门拒绝、甚至一份来源整个抛错,都只记一笔,
|
|
1998
|
+
* 绝不带走整次重建 —— 重建失败比少一张卡片严重得多。
|
|
1999
|
+
*/
|
|
2000
|
+
private async runExtractionStage(sources: SourceRaw[], previousHashes: Record<string, string>): Promise<ExtractionOutcome> {
|
|
2001
|
+
const schema = this.wikiSchema()
|
|
2002
|
+
const enabledTypes = schema.types.filter(def => def.enabled)
|
|
2003
|
+
const registryHash = contentHashOf(JSON.stringify(enabledTypes.map(def => ({ id: def.id, description: def.description, examples: def.examples }))))
|
|
2004
|
+
const outcome = this.emptyExtractionOutcome({})
|
|
2005
|
+
/**
|
|
2006
|
+
* 上一轮落下的关系产物。**跳过的来源从这里读回自己的关系** —— 没有这一步,
|
|
2007
|
+
* 一次全量重编就会把图上所有 `relates_to` 边抹掉(见 `EXTRACTED_RELATIONS`)。
|
|
2008
|
+
*/
|
|
2009
|
+
const persisted = await this.readExtractedRelations()
|
|
2010
|
+
// 计数器与产物不能由 readdir 的顺序决定,所以先按 sourceId 排序。
|
|
2011
|
+
const ordered = [...sources].sort((left, right) => left.sourceId.localeCompare(right.sourceId))
|
|
2012
|
+
let cards = await this.loadCards()
|
|
2013
|
+
for (const source of ordered) {
|
|
2014
|
+
const fingerprint = this.extractionFingerprint(source, registryHash)
|
|
2015
|
+
outcome.extractionHashes[source.sourceId] = fingerprint
|
|
2016
|
+
if (previousHashes[source.sourceId] === fingerprint) {
|
|
2017
|
+
outcome.sourcesSkipped += 1
|
|
2018
|
+
const kept = persisted[source.sourceId] ?? []
|
|
2019
|
+
if (kept.length > 0) {
|
|
2020
|
+
outcome.relationsBySource[source.sourceId] = kept
|
|
2021
|
+
outcome.relations.push(...kept)
|
|
2022
|
+
}
|
|
2023
|
+
continue
|
|
2024
|
+
}
|
|
2025
|
+
outcome.sourcesExtracted += 1
|
|
2026
|
+
try {
|
|
2027
|
+
cards = await this.extractFromSource(source, cards, enabledTypes, outcome)
|
|
2028
|
+
} catch (cause) {
|
|
2029
|
+
outcome.warnings.push(`来源 ${source.sourceId} 的候选抽取失败,已跳过该来源:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
// 只留这一轮真实存在的来源:删掉的文档不该在图上留下幽灵关系。
|
|
2033
|
+
const alive = new Set(ordered.map(source => source.sourceId))
|
|
2034
|
+
const next: Record<string, ExtractionOutcome['relations']> = {}
|
|
2035
|
+
for (const [sourceId, relations] of Object.entries(outcome.relationsBySource)) {
|
|
2036
|
+
if (alive.has(sourceId)) next[sourceId] = relations
|
|
2037
|
+
}
|
|
2038
|
+
outcome.relationsBySource = next
|
|
2039
|
+
await this.writeExtractedRelations(next)
|
|
2040
|
+
return outcome
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
/** 读回关系产物。文件不在或读不动都当「没有」,不抛。 */
|
|
2044
|
+
private async readExtractedRelations(): Promise<Record<string, ExtractionOutcome['relations']>> {
|
|
2045
|
+
try {
|
|
2046
|
+
const raw = await readFile(this.safePath('03_compiled', EXTRACTED_RELATIONS), 'utf8')
|
|
2047
|
+
const parsed = JSON.parse(raw) as { bySource?: unknown }
|
|
2048
|
+
const bySource = plainRecord(parsed.bySource)
|
|
2049
|
+
const result: Record<string, ExtractionOutcome['relations']> = {}
|
|
2050
|
+
for (const [sourceId, value] of Object.entries(bySource)) {
|
|
2051
|
+
if (!Array.isArray(value)) continue
|
|
2052
|
+
const relations = value.flatMap(entry => {
|
|
2053
|
+
const item = plainRecord(entry)
|
|
2054
|
+
const fromKey = typeof item.fromKey === 'string' ? item.fromKey : ''
|
|
2055
|
+
const toKey = typeof item.toKey === 'string' ? item.toKey : ''
|
|
2056
|
+
const relation = typeof item.relation === 'string' ? item.relation : ''
|
|
2057
|
+
if (!fromKey || !toKey || !relation) return []
|
|
2058
|
+
return [{ fromKey, toKey, relation, sourceId }]
|
|
2059
|
+
})
|
|
2060
|
+
if (relations.length > 0) result[sourceId] = relations
|
|
2061
|
+
}
|
|
2062
|
+
return result
|
|
2063
|
+
} catch { return {} }
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
/** 落盘关系产物。失败只记一笔:它不该带走整次重建。 */
|
|
2067
|
+
private async writeExtractedRelations(bySource: Record<string, ExtractionOutcome['relations']>): Promise<void> {
|
|
2068
|
+
try {
|
|
2069
|
+
await writeTextAtomic(this.safePath('03_compiled', EXTRACTED_RELATIONS), `${JSON.stringify({ version: 1, bySource }, null, 2)}\n`)
|
|
2070
|
+
} catch { /* 写不进去就当这一轮没有产物,下一次重建会重新抽 */ }
|
|
2071
|
+
}
|
|
2072
|
+
|
|
2073
|
+
/** 一份来源的抽取结果:新建候选卡片、把证据来源并进已有卡片、收集关系候选。 */
|
|
2074
|
+
private async extractFromSource(source: SourceRaw, cards: KnowledgeCard[], enabledTypes: WikiTypeDef[], outcome: ExtractionOutcome): Promise<KnowledgeCard[]> {
|
|
2075
|
+
/**
|
|
2076
|
+
* #3b:**模型先抽实体,规则补关系。**
|
|
2077
|
+
*
|
|
2078
|
+
* 规则抽取把「小节标题」当实体身份,实测下来的后果是同一件东西横跨两份文档就
|
|
2079
|
+
* 变成两张卡片(`新建 2 / 并入 0`)—— 追加分支永远不触发。所以实体这一层交给
|
|
2080
|
+
* 模型(它能判断「毛利率变动及其驱动因素」与「毛利率(口径补充)」是同一个东西)。
|
|
2081
|
+
*
|
|
2082
|
+
* 但仍然**照跑一遍规则抽取**,理由有两个:关系候选只有规则会抽(模型这一层不产
|
|
2083
|
+
* 关系),而且模型不可用时它就是那条降级路径。两者不是二选一。
|
|
2084
|
+
*/
|
|
2085
|
+
const ruleCandidates = extractCandidates(source, enabledTypes)
|
|
2086
|
+
outcome.candidatesExtracted += ruleCandidates.length
|
|
2087
|
+
const ruleNormalized = normalizeCandidates(ruleCandidates, buildAliasIndex(cards))
|
|
2088
|
+
|
|
2089
|
+
let normalized: NormalizedCandidates = ruleNormalized
|
|
2090
|
+
if (this.modelCall && this.config.extractWithModel) {
|
|
2091
|
+
const modelled = await extractWithModel({ source, wikiTypes: enabledTypes, existing: cards, call: this.modelCall })
|
|
2092
|
+
for (const warning of modelled.warnings) outcome.warnings.push(warning)
|
|
2093
|
+
// 模型一条实体都没给(或者不可用)时,完整退回规则结果,而不是丢掉整轮抽取。
|
|
2094
|
+
if (modelled.fresh.length > 0 || modelled.merges.length > 0) {
|
|
2095
|
+
normalized = { fresh: modelled.fresh, merges: modelled.merges, relations: ruleNormalized.relations }
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
const candidates = normalized.fresh
|
|
2099
|
+
if (candidates.length === 0 && normalized.merges.length === 0) return cards
|
|
2100
|
+
const next = [...cards]
|
|
2101
|
+
|
|
2102
|
+
// 先建后并:同一轮里新建的卡片也可能被别名指回,先建才能让后面的合并找得到对象。
|
|
2103
|
+
for (const candidate of normalized.fresh) {
|
|
2104
|
+
/**
|
|
2105
|
+
* 候选**必须带正文**才落卡。
|
|
2106
|
+
*
|
|
2107
|
+
* 0029 实跑发现:这里原来在候选没有正文时拿 `type.template` 顶上,于是
|
|
2108
|
+
* `[[链接]]` 指到、但还没人写过的实体(那时链接候选确实没有正文)会落成一张
|
|
2109
|
+
* **正文是模板**的卡片,而模板里的「(必填:写什么)」会被编进 wikiNotes、
|
|
2110
|
+
* 被召回、被塞进模型的上下文包 —— 管线写出了一张它自己的校验器判为不合格的
|
|
2111
|
+
* 卡片。链接候选现在会带上它所在小节的正文(`extract.ts`),所以这条回退基本
|
|
2112
|
+
* 不会被走到;真走到了就**不落卡**,并如实计数:没有正文 = 没有可说的。
|
|
2113
|
+
*/
|
|
2114
|
+
const body = candidate.body?.trim() ?? ''
|
|
2115
|
+
if (!body) {
|
|
2116
|
+
outcome.candidatesRejected += 1
|
|
2117
|
+
outcome.warnings.push(`候选「${candidate.name}」没有任何正文(只在别处被提到,所在小节也没取到),不落卡`)
|
|
2118
|
+
continue
|
|
2119
|
+
}
|
|
2120
|
+
// 重复的写法都进 aliases:判重认的是名字,不是卡片 id(中文名会退化成哈希 id)。
|
|
2121
|
+
const aliases = [...new Set([candidate.name, ...(candidate.aliases ?? [])].map(value => value.trim()).filter(Boolean))]
|
|
2122
|
+
const metadata: Record<string, unknown> = {
|
|
2123
|
+
extraction: 'pass0',
|
|
2124
|
+
candidate_kind: candidate.kind,
|
|
2125
|
+
confidence: candidate.confidence,
|
|
2126
|
+
evidence: {
|
|
2127
|
+
sourceId: candidate.evidence.sourceId,
|
|
2128
|
+
...(candidate.evidence.heading ? { heading: candidate.evidence.heading } : {}),
|
|
2129
|
+
},
|
|
2130
|
+
...(aliases.length > 0 ? { aliases } : {}),
|
|
2131
|
+
}
|
|
2132
|
+
const written = await this.writeCard({
|
|
2133
|
+
entityType: candidate.type,
|
|
2134
|
+
entityId: candidateEntityId(candidate.name),
|
|
2135
|
+
title: candidate.name,
|
|
2136
|
+
content: body,
|
|
2137
|
+
sourceRefs: [candidate.evidence.sourceId],
|
|
2138
|
+
status: 'unreviewed',
|
|
2139
|
+
metadata,
|
|
2140
|
+
})
|
|
2141
|
+
if (!written.ok) {
|
|
2142
|
+
outcome.candidatesRejected += 1
|
|
2143
|
+
outcome.warnings.push(`候选「${candidate.name}」未写入(${written.error?.code ?? 'UNKNOWN'}):${written.error?.message ?? ''}`)
|
|
2144
|
+
continue
|
|
2145
|
+
}
|
|
2146
|
+
outcome.cardsCreated += 1
|
|
2147
|
+
next.push({
|
|
2148
|
+
entityId: written.data.entityId,
|
|
2149
|
+
entityType: candidate.type,
|
|
2150
|
+
title: candidate.name,
|
|
2151
|
+
status: 'unreviewed',
|
|
2152
|
+
content: body,
|
|
2153
|
+
sourceRefs: [candidate.evidence.sourceId],
|
|
2154
|
+
metadata,
|
|
2155
|
+
updatedAt: '',
|
|
2156
|
+
})
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
// 归一:命中的候选把证据来源并进已有卡片。同一张卡片在这一轮里的多次命中合并且只写一次。
|
|
2160
|
+
const targets = new Map<string, { card: KnowledgeCard; keys: string[]; sourceId: string; bodies: string[] }>()
|
|
2161
|
+
for (const merge of normalized.merges) {
|
|
2162
|
+
const card = next.find(item => item.entityId === merge.entityId && item.entityType === merge.entityType)
|
|
2163
|
+
if (!card) continue
|
|
2164
|
+
// 「只在没有的时候追加」:来源已在 source_refs 里就什么都不做 —— 空写也会
|
|
2165
|
+
// 刷新 updated_at,并让「并入了多少张卡」这个计数变成谎话。
|
|
2166
|
+
if (card.sourceRefs.includes(merge.sourceId)) continue
|
|
2167
|
+
const id = `${merge.entityType}:${merge.entityId}`
|
|
2168
|
+
const entry = targets.get(id)
|
|
2169
|
+
if (entry) {
|
|
2170
|
+
if (!entry.keys.includes(merge.key)) entry.keys.push(merge.key)
|
|
2171
|
+
entry.bodies.push(merge.body)
|
|
2172
|
+
} else targets.set(id, { card, keys: [merge.key], sourceId: merge.sourceId, bodies: [merge.body] })
|
|
2173
|
+
}
|
|
2174
|
+
for (const { card, keys, sourceId, bodies } of targets.values()) {
|
|
2175
|
+
const history = Array.isArray(card.metadata.extraction_merges) ? card.metadata.extraction_merges : []
|
|
2176
|
+
// 只留最近 20 条:这是「这张卡被哪些来源判重命中过」的可见痕迹,不是审计日志。
|
|
2177
|
+
const extraction_merges = [...history, { sourceId, keys, at: new Date().toISOString() }].slice(-20)
|
|
2178
|
+
// **追加**而不是覆盖,也不是丢弃:实体已存在时,这次新抽到的正文并进原卡片,
|
|
2179
|
+
// 于是知识是累积的,来源改写了也不会停在旧版本上。
|
|
2180
|
+
const content = appendExtracted(card.content, bodies)
|
|
2181
|
+
const written = await this.writeCard({
|
|
2182
|
+
...card,
|
|
2183
|
+
content,
|
|
2184
|
+
sourceRefs: [...card.sourceRefs, sourceId],
|
|
2185
|
+
metadata: { ...card.metadata, extraction_merges },
|
|
2186
|
+
})
|
|
2187
|
+
if (!written.ok) {
|
|
2188
|
+
outcome.candidatesRejected += 1
|
|
2189
|
+
outcome.warnings.push(`候选并入卡片「${card.entityId}」失败(${written.error?.code ?? 'UNKNOWN'}):${written.error?.message ?? ''}`)
|
|
2190
|
+
continue
|
|
2191
|
+
}
|
|
2192
|
+
outcome.cardsMerged += 1
|
|
2193
|
+
// 让后面的来源看到已经追加过的正文与 source_refs,避免重复写入。
|
|
2194
|
+
const index = next.findIndex(item => item.entityId === card.entityId && item.entityType === card.entityType)
|
|
2195
|
+
if (index >= 0) next[index] = { ...card, content, sourceRefs: [...card.sourceRefs, sourceId], metadata: { ...card.metadata, extraction_merges } }
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
outcome.relations.push(...normalized.relations)
|
|
2199
|
+
// 按来源存一份:跳过的来源下次重建要靠它把关系读回来(见 EXTRACTED_RELATIONS)。
|
|
2200
|
+
outcome.relationsBySource[source.sourceId] = [
|
|
2201
|
+
...(outcome.relationsBySource[source.sourceId] ?? []),
|
|
2202
|
+
...normalized.relations,
|
|
2203
|
+
]
|
|
2204
|
+
return next
|
|
2205
|
+
}
|
|
2206
|
+
|
|
2207
|
+
/**
|
|
2208
|
+
* 把关系候选装配成 wiki↔wiki 的边。
|
|
2209
|
+
*
|
|
2210
|
+
* 只有两端都解析到**已存在的 wiki 节点**时才加边,其余丢弃并计数:凭空造一个
|
|
2211
|
+
* 节点会给图上留一个点不开的孤立点,比没有这条边更难解释。自环同理丢弃。
|
|
2212
|
+
* `supported_by`(卡片 → 来源)完全不受影响,它是另一套边。
|
|
2213
|
+
*/
|
|
2214
|
+
private applyExtractedRelations(
|
|
2215
|
+
nodeMap: Map<string, Record<string, unknown>>,
|
|
2216
|
+
edges: Array<Record<string, unknown>>,
|
|
2217
|
+
relations: Array<{ fromKey: string; toKey: string; relation: string }>,
|
|
2218
|
+
): { extracted: number; dropped: number } {
|
|
2219
|
+
const seen = new Set(edges.map(edge => `${String(edge.from)}|${String(edge.to)}|${String(edge.type)}`))
|
|
2220
|
+
let extracted = 0
|
|
2221
|
+
let dropped = 0
|
|
2222
|
+
for (const relation of relations) {
|
|
2223
|
+
const from = relation.fromKey
|
|
2224
|
+
const to = relation.toKey
|
|
2225
|
+
const type = relation.relation || 'relates_to'
|
|
2226
|
+
if (!from || !to || from === to || !nodeMap.has(from) || !nodeMap.has(to)) { dropped += 1; continue }
|
|
2227
|
+
const key = `${from}|${to}|${type}`
|
|
2228
|
+
// 多份来源声明同一条关系是很常见的;它已经连上了,不重复加边,但也不算丢弃。
|
|
2229
|
+
if (!seen.has(key)) { seen.add(key); edges.push({ from, to, type }) }
|
|
2230
|
+
extracted += 1
|
|
2231
|
+
}
|
|
2232
|
+
return { extracted, dropped }
|
|
2233
|
+
}
|
|
2234
|
+
|
|
2235
|
+
async rebuild(only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
2236
|
+
await this.ensureLayout()
|
|
2237
|
+
const startedAt = new Date().toISOString()
|
|
2238
|
+
this.cards = await this.loadCards()
|
|
2239
|
+
const sources = await this.loadSources()
|
|
2240
|
+
const sourceTypes = new Map(sources.map(source => [source.sourceId, source.sourceType]))
|
|
2241
|
+
|
|
2242
|
+
// ------------------------------------------------ previous build state
|
|
2243
|
+
let previousManifest: { version?: number; builtAt?: string; cardHashes?: Record<string, string>; extractionHashes?: Record<string, string>; embedding?: unknown } = {}
|
|
2244
|
+
try {
|
|
2245
|
+
previousManifest = JSON.parse(await readFile(this.safePath('03_compiled', 'build-manifest.json'), 'utf8')) as typeof previousManifest
|
|
2246
|
+
} catch { /* first build */ }
|
|
2247
|
+
const previousVersion = Number(previousManifest.version ?? 0)
|
|
2248
|
+
const previousHashes = previousManifest.cardHashes ?? {}
|
|
2249
|
+
const previousExtractionHashes = previousManifest.extractionHashes ?? {}
|
|
2250
|
+
|
|
2251
|
+
// ------------------------------------------------ Pass 0 · 候选抽取 + 归一
|
|
2252
|
+
/**
|
|
2253
|
+
* Pass 0 只在会走 stage 2 的重建里跑:`embeddings` 是纯向量重编码,不碰 wiki 层。
|
|
2254
|
+
* 关掉时走一条显式的空路径而不是省略——报告里的八个计数必须始终是「真的 0」,
|
|
2255
|
+
* 不能靠缺字段表达,否则读的人分不清「关掉了」和「这一步没有实现」。
|
|
2256
|
+
*/
|
|
2257
|
+
const extractEnabled = only !== 'embeddings' && (options.extract ?? this.config.extractCandidates)
|
|
2258
|
+
const extraction = extractEnabled
|
|
2259
|
+
? await this.runExtractionStage(sources, previousExtractionHashes)
|
|
2260
|
+
: this.emptyExtractionOutcome(previousExtractionHashes)
|
|
2261
|
+
// 抽取刚落过盘:stage 2 必须看到新的卡片清单,否则这一轮建的卡片要等下一次
|
|
2262
|
+
// 重建才进 wikiNotes,用户点了编译却召不回刚抽出来的东西。
|
|
2263
|
+
if (extraction.cardsCreated > 0 || extraction.cardsMerged > 0) this.cards = await this.loadCards()
|
|
2264
|
+
const extractionCounts = {
|
|
2265
|
+
candidatesExtracted: extraction.candidatesExtracted,
|
|
2266
|
+
candidatesRejected: extraction.candidatesRejected,
|
|
2267
|
+
cardsCreated: extraction.cardsCreated,
|
|
2268
|
+
cardsMerged: extraction.cardsMerged,
|
|
2269
|
+
sourcesExtracted: extraction.sourcesExtracted,
|
|
2270
|
+
sourcesSkipped: extraction.sourcesSkipped,
|
|
2271
|
+
}
|
|
2272
|
+
const warnings: string[] = [...extraction.warnings]
|
|
2273
|
+
let relationsExtracted = 0
|
|
2274
|
+
let relationsDropped = 0
|
|
2275
|
+
|
|
2276
|
+
// ------------------------------------------------ stage 2 · compile notes
|
|
2277
|
+
let cardsReused = 0
|
|
2278
|
+
let cardsRecompiled = 0
|
|
2279
|
+
let cardsAdded = 0
|
|
2280
|
+
let notesReused = 0
|
|
2281
|
+
let notesRecompiled = 0
|
|
2282
|
+
const cardHashes: Record<string, string> = {}
|
|
2283
|
+
/**
|
|
2284
|
+
* 复用与「跳过」只发生在**增量编译(`wiki`)**上。
|
|
2285
|
+
*
|
|
2286
|
+
* `all` 是用户点「全量重编」时明确要求的「从头来一遍」:它照样重切所有 wikiNotes、
|
|
2287
|
+
* 照样升版本(向量层的 `contentHash` 复用仍然生效,所以不会真的重编码一遍)。
|
|
2288
|
+
* 把 `all` 也做成「没变就跳过」会让这个按钮变成第二个增量编译,用户就失去了
|
|
2289
|
+
* 那个「我怀疑产物不对,强制重来一次」的手段。
|
|
2290
|
+
*/
|
|
2291
|
+
const incremental = only === 'wiki'
|
|
2292
|
+
if (only === 'embeddings') {
|
|
2293
|
+
if (this.units.length === 0) await this.loadCompiledState()
|
|
2294
|
+
} else {
|
|
2295
|
+
const previousNotes = incremental ? await this.readNotesArtifact() : []
|
|
2296
|
+
const previousByWiki = new Map<string, WikiNote[]>()
|
|
2297
|
+
for (const note of previousNotes) {
|
|
2298
|
+
const key = `${note.entityType}:${note.entityId}`
|
|
2299
|
+
const bucket = previousByWiki.get(key)
|
|
2300
|
+
if (bucket) bucket.push(note)
|
|
2301
|
+
else previousByWiki.set(key, [note])
|
|
2302
|
+
}
|
|
2303
|
+
const notes: WikiNote[] = []
|
|
2304
|
+
for (const card of this.cards) {
|
|
2305
|
+
const key = `${card.entityType}:${card.entityId}`
|
|
2306
|
+
const fingerprint = this.cardHashOf(card)
|
|
2307
|
+
cardHashes[key] = fingerprint
|
|
2308
|
+
if (!previousHashes[key]) cardsAdded += 1
|
|
2309
|
+
const kept = incremental && previousHashes[key] === fingerprint ? previousByWiki.get(key) : undefined
|
|
2310
|
+
if (kept && kept.length > 0) {
|
|
2311
|
+
// Reuse the notes verbatim, but refresh the two derived fields that can
|
|
2312
|
+
// change *without* the card changing: a SourceRaw's business type and
|
|
2313
|
+
// the card's provenance chain. Keeping a stale `sourceTypes` would make
|
|
2314
|
+
// a source_type-filtered recall silently miss this card.
|
|
2315
|
+
const sourceTypesFor = card.sourceRefs.flatMap(ref => { const sourceType = sourceTypes.get(ref); return sourceType ? [sourceType] : [] })
|
|
2316
|
+
notes.push(...kept.map(note => ({
|
|
2317
|
+
...note,
|
|
2318
|
+
sourceTypes: sourceTypesFor,
|
|
2319
|
+
...(card.provenance ? { provenance: card.provenance } : {}),
|
|
2320
|
+
})))
|
|
2321
|
+
cardsReused += 1
|
|
2322
|
+
notesReused += kept.length
|
|
2323
|
+
} else {
|
|
2324
|
+
const fresh = this.compileCardNotes(card, sourceTypes)
|
|
2325
|
+
notes.push(...fresh)
|
|
2326
|
+
cardsRecompiled += 1
|
|
2327
|
+
notesRecompiled += fresh.length
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
this.units = notes
|
|
2331
|
+
const currentKeys = new Set(Object.keys(cardHashes))
|
|
2332
|
+
const cardsRemoved = Object.keys(previousHashes).filter(key => !currentKeys.has(key)).length
|
|
2333
|
+
|
|
2334
|
+
// A wiki rebuild over an unchanged corpus is a no-op, not a new version:
|
|
2335
|
+
// bumping the version there would invalidate the vector cache and every
|
|
2336
|
+
// qrels pointing at it for nothing. `all` never takes this path (see
|
|
2337
|
+
// `incremental` above).
|
|
2338
|
+
const corpusChanged = !incremental || previousVersion === 0 || cardsRecompiled > 0 || cardsRemoved > 0
|
|
2339
|
+
if (!corpusChanged) {
|
|
2340
|
+
const compileRunId = `rebuild_${hash(`${startedAt}:${this.cards.length}`)}`
|
|
2341
|
+
const report: RebuildReport = {
|
|
2342
|
+
runId: compileRunId, compileRunId, version: previousVersion, only,
|
|
2343
|
+
cards: this.cards.length, units: this.units.length, notes: this.units.length,
|
|
2344
|
+
graphNodes: this.graph.nodes.length, graphEdges: this.graph.edges.length,
|
|
2345
|
+
embedding: 'skipped',
|
|
2346
|
+
cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0,
|
|
2347
|
+
notesReused, notesRecompiled: 0,
|
|
2348
|
+
incremental: true, skipped: true,
|
|
2349
|
+
...extractionCounts,
|
|
2350
|
+
// 这一轮没有重装图,所以没有一条边是「这次连上的」或「这次丢掉的」。
|
|
2351
|
+
// 报 0 而不是把图里已有的边算进来:那会把历史成果记到这次空操作头上。
|
|
2352
|
+
relationsExtracted: 0, relationsDropped: 0,
|
|
2353
|
+
...(lineage ? { lineage } : {}), startedAt, finishedAt: new Date().toISOString(),
|
|
2354
|
+
}
|
|
2355
|
+
// 跳过一次也要落盘 lastBuild:否则界面显示的是「上一次真正重建」的增量数字,
|
|
2356
|
+
// 用户刚点了编译却看到上次的统计,会以为按钮没生效。version 不变,所以
|
|
2357
|
+
// 这个写入既不改索引也不让向量缓存失效。extractionHashes 必须跟着更新:
|
|
2358
|
+
// 来源变了但没抽出卡片时,不落新指纹会让下一次又把它重抽一遍。
|
|
2359
|
+
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version: previousVersion, builtAt: previousManifest.builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: previousManifest.embedding, compileRunId, lastBuild: { only, incremental: true, skipped: true, cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0, notesReused, notesRecompiled: 0, ...extractionCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
2360
|
+
return success(report, ['知识库没有变化:已复用全部 wikiNotes,未生成新的索引版本。', ...extraction.warnings])
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
const nodeMap = new Map<string, Record<string, unknown>>()
|
|
2364
|
+
const edges: Array<Record<string, unknown>> = []
|
|
2365
|
+
for (const card of this.cards) {
|
|
2366
|
+
nodeMap.set(`${card.entityType}:${card.entityId}`, { id: card.entityId, type: card.entityType, title: card.title, status: card.status })
|
|
2367
|
+
for (const sourceRef of card.sourceRefs) {
|
|
2368
|
+
nodeMap.set(`source:${sourceRef}`, { id: sourceRef, type: 'source', title: sourceRef })
|
|
2369
|
+
edges.push({ from: `${card.entityType}:${card.entityId}`, to: `source:${sourceRef}`, type: 'supported_by' })
|
|
2370
|
+
}
|
|
2371
|
+
}
|
|
2372
|
+
// Pass 0 抽出的关系在这里才成为边:两端都要落在真实存在的 wiki 节点上。
|
|
2373
|
+
const relationStats = this.applyExtractedRelations(nodeMap, edges, extraction.relations)
|
|
2374
|
+
relationsExtracted = relationStats.extracted
|
|
2375
|
+
relationsDropped = relationStats.dropped
|
|
2376
|
+
this.graph = { nodes: [...nodeMap.values()], edges }
|
|
2377
|
+
const removedCount = cardsRemoved
|
|
2378
|
+
const version = previousVersion + 1
|
|
2379
|
+
const builtAt = new Date().toISOString()
|
|
2380
|
+
await writeTextAtomic(this.safePath('03_compiled', WIKI_NOTES), JSON.stringify(this.units, null, 2))
|
|
2381
|
+
// Compatibility mirror for tooling that predates wiki_notes.json.
|
|
2382
|
+
await writeTextAtomic(this.safePath('03_compiled', LEGACY_UNITS), JSON.stringify(this.units, null, 2))
|
|
2383
|
+
await writeTextAtomic(this.safePath('03_compiled', 'knowledge_graph.json'), JSON.stringify(this.graph, null, 2))
|
|
2384
|
+
// 关键词索引跟着 notes 一起落盘(0031):它的输入就是 notes,两者必须同时有效,
|
|
2385
|
+
// 否则会出现「卡片改了、关键词路还按旧词表召回」。它不依赖编码器,所以 wiki-only
|
|
2386
|
+
// 重建也要重建它 —— 而向量是 stage 3,仍然只在 `embeddings`/`all` 时重算。
|
|
2387
|
+
const ftsResult = await this.buildFtsIndex(this.units, sources)
|
|
2388
|
+
// `only: 'wiki'` stops at the notes + graph: vectors are stage 3 and get
|
|
2389
|
+
// rebuilt on demand (`embeddings`/`all`), so a wiki edit does not pay for a
|
|
2390
|
+
// re-encode of the whole corpus.
|
|
2391
|
+
const embeddingResult = only === 'all' ? await this.buildVectorIndex(this.units, version) : { status: 'skipped' as const, count: 0, reused: 0 }
|
|
2392
|
+
const compileRunId = `rebuild_${hash(`${startedAt}:${this.cards.length}`)}`
|
|
2393
|
+
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version, builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: embeddingResult, fts: ftsResult, compileRunId, lastBuild: { only, incremental: cardsReused > 0, skipped: false, cardsReused, cardsRecompiled, cardsAdded, cardsRemoved: removedCount, notesReused, notesRecompiled, ...extractionCounts, relationsExtracted, relationsDropped }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
2394
|
+
const report: RebuildReport = {
|
|
2395
|
+
runId: compileRunId, compileRunId, version, only,
|
|
2396
|
+
cards: this.cards.length, units: this.units.length, notes: this.units.length,
|
|
2397
|
+
graphNodes: this.graph.nodes.length, graphEdges: this.graph.edges.length,
|
|
2398
|
+
embedding: embeddingResult.status, embeddingItems: embeddingResult.count, embeddingReused: embeddingResult.reused,
|
|
2399
|
+
embeddingDimension: embeddingResult.dimension,
|
|
2400
|
+
vectorQuantization: embeddingResult.status === 'ready' ? VECTOR_QUANTIZATION : undefined,
|
|
2401
|
+
embeddingError: embeddingResult.error,
|
|
2402
|
+
fts: ftsResult,
|
|
2403
|
+
cardsReused, cardsRecompiled, cardsAdded, cardsRemoved: removedCount,
|
|
2404
|
+
notesReused, notesRecompiled,
|
|
2405
|
+
incremental: cardsReused > 0 || notesReused > 0,
|
|
2406
|
+
skipped: false,
|
|
2407
|
+
...extractionCounts, relationsExtracted, relationsDropped,
|
|
2408
|
+
...(lineage ? { lineage } : {}), startedAt, finishedAt: new Date().toISOString(),
|
|
2409
|
+
}
|
|
2410
|
+
if (embeddingResult.status === 'unavailable') warnings.push(`embedding 不可用,已保留 FTS fallback:${embeddingResult.error ?? 'unknown error'}`)
|
|
2411
|
+
return success(report, warnings)
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
// ------------------------------------------------ embeddings-only rebuild
|
|
2415
|
+
if (this.units.length === 0 && this.graph.nodes.length === 0) {
|
|
2416
|
+
await this.loadCompiledState()
|
|
2417
|
+
}
|
|
2418
|
+
const version = previousVersion + 1
|
|
2419
|
+
const embeddingResult = await this.buildVectorIndex(this.units, version)
|
|
2420
|
+
const compileRunId = `rebuild_${hash(`${startedAt}:${this.cards.length}`)}`
|
|
2421
|
+
await writeTextAtomic(this.safePath('03_compiled', 'build-manifest.json'), JSON.stringify({ version, builtAt: new Date().toISOString(), sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: embeddingResult, compileRunId, lastBuild: { only, incremental: embeddingResult.reused > 0, skipped: false, notesReused: embeddingResult.reused, notesRecompiled: Math.max(0, embeddingResult.count - embeddingResult.reused), ...extractionCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
2422
|
+
const report: RebuildReport = {
|
|
2423
|
+
runId: compileRunId, compileRunId, version, only,
|
|
2424
|
+
cards: this.cards.length, units: this.units.length, notes: this.units.length,
|
|
2425
|
+
graphNodes: this.graph.nodes.length, graphEdges: this.graph.edges.length,
|
|
2426
|
+
embedding: embeddingResult.status, embeddingItems: embeddingResult.count, embeddingReused: embeddingResult.reused,
|
|
2427
|
+
embeddingDimension: embeddingResult.dimension,
|
|
2428
|
+
vectorQuantization: embeddingResult.status === 'ready' ? VECTOR_QUANTIZATION : undefined,
|
|
2429
|
+
embeddingError: embeddingResult.error,
|
|
2430
|
+
incremental: embeddingResult.reused > 0, skipped: false,
|
|
2431
|
+
// `embeddings` 不跑 Pass 0,所以这些计数是真的 0,不是省略。
|
|
2432
|
+
...extractionCounts, relationsExtracted: 0, relationsDropped: 0,
|
|
2433
|
+
...(lineage ? { lineage } : {}), startedAt, finishedAt: new Date().toISOString(),
|
|
2434
|
+
}
|
|
2435
|
+
if (embeddingResult.status === 'unavailable') warnings.push(`embedding 不可用,已保留 FTS fallback:${embeddingResult.error ?? 'unknown error'}`)
|
|
2436
|
+
return success(report, warnings)
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
/**
|
|
2440
|
+
* The staged read path:
|
|
2441
|
+
* 3) recall — multi-route (keyword + vector + knowledge graph) recall over
|
|
2442
|
+
* wikiNotes, bounded by `recallTopK`, then resolve each hit's wiki and
|
|
2443
|
+
* its 1-hop graph;
|
|
2444
|
+
* 4) rerank — BERT cross-encoder, then expand only `contextTopN` notes
|
|
2445
|
+
* into the context pack handed to the model.
|
|
2446
|
+
*/
|
|
2447
|
+
// ------------------------------------------------------------- 召回(0031)
|
|
2448
|
+
|
|
2449
|
+
/**
|
|
2450
|
+
* 编码器健康门(0031 的 P0)。
|
|
2451
|
+
*
|
|
2452
|
+
* 0030 的真实故障:WeMM 经 `SentenceTransformer.encode` 对**任何输入返回同一个向量**
|
|
2453
|
+
* (实测四段无关文本两两 `cos = 1.000000`),而整条链路照样上报
|
|
2454
|
+
* `routes: ["fts","vector","graph"]` —— 谎报「三路都在跑」。索引里 44 行有 39 行逐字节
|
|
2455
|
+
* 相同,排序退化成 `unitId` 字典序,而界面上看不出任何异常。
|
|
2456
|
+
*
|
|
2457
|
+
* 所以「能不能用」不能靠「跑没跑」来判断,要**实测**:编码两段无关文本,要求余弦远小于 1。
|
|
2458
|
+
* 不通过就关掉向量路并如实说明 —— 一条会撒谎的路比没有这条路更坏。
|
|
2459
|
+
*
|
|
2460
|
+
* 代价:第一次用到向量路时要起一次 worker(模型常驻之后就是一次往返)。这个代价是
|
|
2461
|
+
* 必要的,而且它和「第一次召回本来就要加载模型」重叠。
|
|
2462
|
+
*/
|
|
2463
|
+
private encoderHealth: { ok: boolean; reason?: string } | undefined
|
|
2464
|
+
|
|
2465
|
+
private async checkEncoder(): Promise<{ ok: boolean; reason?: string }> {
|
|
2466
|
+
if (this.encoderHealth) return this.encoderHealth
|
|
2467
|
+
if (this.config.embeddingProvider === 'none') {
|
|
2468
|
+
this.encoderHealth = { ok: false, reason: '嵌入已关闭(embeddingProvider = none)' }
|
|
2469
|
+
return this.encoderHealth
|
|
2470
|
+
}
|
|
2471
|
+
try {
|
|
2472
|
+
const probes = [
|
|
2473
|
+
'召回链路自检:指标口径与计算公式。',
|
|
2474
|
+
'Sanity probe: an entirely unrelated sentence about cooking pasta.',
|
|
2475
|
+
]
|
|
2476
|
+
const encoded = await this.encodeEmbeddings(probes.map(text => ({ text })))
|
|
2477
|
+
const left = encoded.vectors[0]
|
|
2478
|
+
const right = encoded.vectors[1]
|
|
2479
|
+
if (!left || !right || left.length !== right.length || left.length === 0) throw new Error('ENCODER_PROBE_SHAPE_INVALID')
|
|
2480
|
+
let dot = 0
|
|
2481
|
+
let leftNorm = 0
|
|
2482
|
+
let rightNorm = 0
|
|
2483
|
+
for (let index = 0; index < left.length; index += 1) {
|
|
2484
|
+
dot += left[index]! * right[index]!
|
|
2485
|
+
leftNorm += left[index]! * left[index]!
|
|
2486
|
+
rightNorm += right[index]! * right[index]!
|
|
2487
|
+
}
|
|
2488
|
+
const cosine = dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm) || 1)
|
|
2489
|
+
if (!Number.isFinite(cosine) || cosine > ENCODER_SELF_SIMILARITY_MAX) {
|
|
2490
|
+
this.encoderHealth = { ok: false, reason: `编码器对两段无关文本给出同一个向量(cos = ${cosine.toFixed(4)}),向量路已停用` }
|
|
2491
|
+
return this.encoderHealth
|
|
2492
|
+
}
|
|
2493
|
+
this.encoderHealth = { ok: true }
|
|
2494
|
+
return this.encoderHealth
|
|
2495
|
+
} catch (cause) {
|
|
2496
|
+
this.encoderHealth = { ok: false, reason: `编码器自检失败:${cause instanceof Error ? cause.message : String(cause)}` }
|
|
2497
|
+
return this.encoderHealth
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
/** 关键词路:FTS5 + BM25(0031)。索引不在时如实降级,不退回旧的计数打分。 */
|
|
2502
|
+
private async ftsRoute(query: string, eligible: WikiNote[], pool: number, threshold: number, titleOf: (sourceId: string) => string | undefined): Promise<{ ranking: RouteRanking[]; diagnostic: RecallRouteDiagnostic }> {
|
|
2503
|
+
const eligibleIds = new Set(eligible.map(note => note.unitId))
|
|
2504
|
+
const sqlite = await loadSqlite()
|
|
2505
|
+
if (!sqlite) return { ranking: [], diagnostic: { route: 'fts', status: 'degraded', reason: '当前 Node 没有 node:sqlite(需要 22.5+),关键词路已降级' } }
|
|
2506
|
+
const index = await FtsIndex.open(this.safePath('03_compiled', FTS_FILE), sqlite.DatabaseSync)
|
|
2507
|
+
if (!index) return { ranking: [], diagnostic: { route: 'fts', status: 'degraded', reason: `关键词索引不存在(03_compiled/${FTS_FILE}),先编译一次` } }
|
|
2508
|
+
try {
|
|
2509
|
+
const found = index.search(query, pool).filter(hit => eligibleIds.has(hit.unitId))
|
|
2510
|
+
/**
|
|
2511
|
+
* 覆盖率门槛(见 `MIN_QUERY_COVERAGE`):只蹭到一两个高频单字的命中在这里淘汰,
|
|
2512
|
+
* 于是「整批都弱」也能弃答 —— 这是相对阈值做不到的那一半。
|
|
2513
|
+
*
|
|
2514
|
+
* **分母只数多字词**(中文双字词与 ASCII 词),不数单个汉字。单个汉字在中文里
|
|
2515
|
+
* 从来不是证据:`的` 到处都是,而 `GMV 怎么算` 这种问句里 怎/么/算 都是噪声。
|
|
2516
|
+
* 把它们算进分母会同时犯两种错 —— 实测:
|
|
2517
|
+
* - 全算 → `GMV 怎么算` 的分母塞进 5 个噪声词,只剩 1 个真命中,覆盖率 0.17,
|
|
2518
|
+
* 正确答案被丢掉;
|
|
2519
|
+
* 只算「命中过任何文档」的词 → `完全无关的词汇zzz` 只剩 `的`,覆盖率 1.0,
|
|
2520
|
+
* 该弃答的反而过了。
|
|
2521
|
+
*
|
|
2522
|
+
* 只数多字词同时修好两边:`GMV 怎么算` → 分母 {gmv, 怎么, 么算},命中 gmv = 0.33 过线;
|
|
2523
|
+
* `完全无关的词汇zzz` → 分母是 6 个双字词,一个都不命中 = 0 淘汰。
|
|
2524
|
+
*/
|
|
2525
|
+
const coverageTerms = [...new Set(segment(query))].filter(term => term.length > 1)
|
|
2526
|
+
if (coverageTerms.length === 0) {
|
|
2527
|
+
// 单字查询(如 `表`)是刻意收窄的问法,没有「多字词覆盖率」可言:不拦它,
|
|
2528
|
+
// 交给阈值。这里如实返回空排名之外的结果需要另走一条路,所以直接用全部命中。
|
|
2529
|
+
const ranking = found.map(hit => ({ unitId: hit.unitId, score: hit.score }))
|
|
2530
|
+
const { kept, dropped, topScore } = applyThreshold(ranking, threshold)
|
|
2531
|
+
return {
|
|
2532
|
+
ranking: kept,
|
|
2533
|
+
diagnostic: {
|
|
2534
|
+
route: 'fts', status: 'ran', candidates: ranking.length, kept: kept.length, topScore, threshold,
|
|
2535
|
+
...(dropped > 0 ? { reason: `${dropped} 条低于关键词阈值 ${threshold} 被丢弃` } : {}),
|
|
2536
|
+
},
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
const notesById = new Map(eligible.map(note => [note.unitId, note]))
|
|
2540
|
+
const scored = found.map(hit => {
|
|
2541
|
+
const note = notesById.get(hit.unitId)
|
|
2542
|
+
const noteTokens = note ? tokenSet(noteIndexText(note, titleOf)) : new Set<string>()
|
|
2543
|
+
let matched = 0
|
|
2544
|
+
for (const term of coverageTerms) if (noteTokens.has(term)) matched += 1
|
|
2545
|
+
return { unitId: hit.unitId, score: hit.score, coverage: matched / coverageTerms.length }
|
|
2546
|
+
})
|
|
2547
|
+
const covered = scored.filter(hit => hit.coverage >= MIN_QUERY_COVERAGE)
|
|
2548
|
+
const ranking = covered.map(hit => ({ unitId: hit.unitId, score: hit.score }))
|
|
2549
|
+
const { kept, dropped, topScore } = applyThreshold(ranking, threshold)
|
|
2550
|
+
const lowCoverage = scored.length - covered.length
|
|
2551
|
+
const reasons: string[] = []
|
|
2552
|
+
if (lowCoverage > 0) reasons.push(`${lowCoverage} 条查询词覆盖率低于 ${MIN_QUERY_COVERAGE} 被丢弃`)
|
|
2553
|
+
if (dropped > 0) reasons.push(`${dropped} 条低于关键词阈值 ${threshold} 被丢弃`)
|
|
2554
|
+
return {
|
|
2555
|
+
ranking: kept,
|
|
2556
|
+
diagnostic: {
|
|
2557
|
+
route: 'fts', status: 'ran', candidates: ranking.length, kept: kept.length, topScore, threshold,
|
|
2558
|
+
...(reasons.length > 0 ? { reason: reasons.join(';') } : {}),
|
|
2559
|
+
},
|
|
2560
|
+
}
|
|
2561
|
+
} finally {
|
|
2562
|
+
index.close()
|
|
2563
|
+
}
|
|
2564
|
+
}
|
|
2565
|
+
|
|
2566
|
+
/**
|
|
2567
|
+
* 向量路:健康门 → 编码一次 → 原始余弦 → 阈值。
|
|
2568
|
+
*
|
|
2569
|
+
* 分数**不再**做 `(cosine + 1) / 2` 的映射:那个映射把余弦压进 0..1 之后,`0.15` 这样的
|
|
2570
|
+
* 阈值就完全失去意义(余弦 0 会被映射成 0.5,轻松过线)。原始余弦是绝对量,可以直接
|
|
2571
|
+
* 用来判「这段内容和问题到底像不像」—— 弃答主要靠它和重排阈值。
|
|
2572
|
+
*/
|
|
2573
|
+
private async vectorRoute(query: string, eligibleIds: Set<string>, pool: number, threshold: number): Promise<{ ranking: RouteRanking[]; diagnostic: RecallRouteDiagnostic; health: { ok: boolean; reason?: string } }> {
|
|
2574
|
+
const health = await this.checkEncoder()
|
|
2575
|
+
if (!health.ok) return { ranking: [], diagnostic: { route: 'vector', status: 'degraded', reason: health.reason }, health }
|
|
2576
|
+
try {
|
|
2577
|
+
const scores = await this.vectorRecall(query, eligibleIds, pool)
|
|
2578
|
+
const ranking = [...scores.entries()].map(([unitId, score]) => ({ unitId, score })).sort((left, right) => right.score - left.score || left.unitId.localeCompare(right.unitId))
|
|
2579
|
+
const { kept, dropped, topScore } = applyThreshold(ranking, threshold)
|
|
2580
|
+
return {
|
|
2581
|
+
ranking: kept,
|
|
2582
|
+
health,
|
|
2583
|
+
diagnostic: {
|
|
2584
|
+
route: 'vector', status: 'ran', candidates: ranking.length, kept: kept.length, topScore, threshold,
|
|
2585
|
+
...(dropped > 0 ? { reason: `${dropped} 条低于向量阈值 ${threshold} 被丢弃` } : {}),
|
|
2586
|
+
},
|
|
2587
|
+
}
|
|
2588
|
+
} catch (cause) {
|
|
2589
|
+
const error = cause instanceof Error ? cause.message : String(cause)
|
|
2590
|
+
this.vectorError = error
|
|
2591
|
+
return { ranking: [], diagnostic: { route: 'vector', status: 'degraded', reason: `向量召回不可用:${error}` }, health }
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
|
|
2595
|
+
/**
|
|
2596
|
+
* 图谱的**名次表**(不是一路分数,见 `graphBoost`)。
|
|
2597
|
+
*
|
|
2598
|
+
* 匹配逻辑与 0030 相同(词面匹配节点 title / id 与关系词,命中节点带出 1 跳邻居并衰减),
|
|
2599
|
+
* 但返回值只用于给**已经在候选池里**的条目加权 —— 它自己不能制造候选。
|
|
2600
|
+
*/
|
|
2601
|
+
private graphBoostRanking(query: string, eligible: WikiNote[], includeProblems: boolean): RouteRanking[] {
|
|
2602
|
+
if (this.graph.nodes.length === 0) return []
|
|
2603
|
+
const queryTokens = segment(query)
|
|
2604
|
+
if (queryTokens.length === 0) return []
|
|
2605
|
+
const byNodeKey = new Map<string, number>()
|
|
2606
|
+
const boost = (key: string, score: number): void => { byNodeKey.set(key, Math.max(byNodeKey.get(key) ?? 0, score)) }
|
|
2607
|
+
const visible = (node: Record<string, unknown>): boolean => includeProblems || node.status !== 'problem'
|
|
2608
|
+
const nodeKey = (node: Record<string, unknown>): string => `${String(node.type)}:${String(node.id)}`
|
|
2609
|
+
|
|
2610
|
+
for (const node of this.graph.nodes) {
|
|
2611
|
+
if (!visible(node)) continue
|
|
2612
|
+
const title = String(node.title ?? '').toLocaleLowerCase()
|
|
2613
|
+
const id = String(node.id ?? '').toLocaleLowerCase()
|
|
2614
|
+
let score = 0
|
|
2615
|
+
for (const token of queryTokens) {
|
|
2616
|
+
if (title.includes(token)) score += 3
|
|
2617
|
+
else if (id.includes(token)) score += 2
|
|
2618
|
+
}
|
|
2619
|
+
if (score > 0) boost(nodeKey(node), score)
|
|
2620
|
+
}
|
|
2621
|
+
for (const edge of this.graph.edges) {
|
|
2622
|
+
const type = String(edge.type ?? '')
|
|
2623
|
+
const relation = (RELATION_QUERY_WORDS[type] ?? type.toLocaleLowerCase().replace(/_/g, ' ')).toLocaleLowerCase()
|
|
2624
|
+
let score = 0
|
|
2625
|
+
for (const token of queryTokens) if (relation.includes(token)) score += 1
|
|
2626
|
+
if (score > 0) { boost(String(edge.from), score); boost(String(edge.to), score) }
|
|
2627
|
+
}
|
|
2628
|
+
if (byNodeKey.size === 0) return []
|
|
2629
|
+
const expanded = new Map<string, number>(byNodeKey)
|
|
2630
|
+
for (const [key, score] of byNodeKey) {
|
|
2631
|
+
for (const edge of this.graph.edges) {
|
|
2632
|
+
const from = String(edge.from)
|
|
2633
|
+
const to = String(edge.to)
|
|
2634
|
+
if (from === key) expanded.set(to, Math.max(expanded.get(to) ?? 0, score * 0.5))
|
|
2635
|
+
else if (to === key) expanded.set(from, Math.max(expanded.get(from) ?? 0, score * 0.5))
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2638
|
+
const byWiki = new Map<string, number>()
|
|
2639
|
+
for (const note of eligible) {
|
|
2640
|
+
const score = expanded.get(`${note.entityType}:${note.wikiId}`)
|
|
2641
|
+
if (score === undefined) continue
|
|
2642
|
+
// 一张 wiki 只留它最好的一节:分节片段不该各自再占一个名次。
|
|
2643
|
+
byWiki.set(note.wikiId, Math.max(byWiki.get(note.wikiId) ?? 0, score))
|
|
2644
|
+
}
|
|
2645
|
+
const primary = new Map(eligible.map(note => [note.wikiId, note.unitId]))
|
|
2646
|
+
return [...byWiki.entries()]
|
|
2647
|
+
.map(([wikiId, score]) => ({ unitId: primary.get(wikiId) ?? '', score }))
|
|
2648
|
+
.filter(item => item.unitId !== '')
|
|
2649
|
+
.sort((left, right) => right.score - left.score || left.unitId.localeCompare(right.unitId))
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
/** SourceRaw 正文,按 sourceId 索引;证据片段从这里取。写入后失效。 */
|
|
2653
|
+
private sourceCorpusCache: Map<string, { title: string; content: string }> | undefined
|
|
2654
|
+
|
|
2655
|
+
private async sourceCorpus(): Promise<Map<string, { title: string; content: string }>> {
|
|
2656
|
+
if (this.sourceCorpusCache) return this.sourceCorpusCache
|
|
2657
|
+
const corpus = new Map<string, { title: string; content: string }>()
|
|
2658
|
+
for (const source of await this.loadSources()) corpus.set(source.sourceId, { title: source.title, content: source.content })
|
|
2659
|
+
this.sourceCorpusCache = corpus
|
|
2660
|
+
return corpus
|
|
2661
|
+
}
|
|
2662
|
+
|
|
2663
|
+
/**
|
|
2664
|
+
* 给命中挂上**证据片段**(0031)。
|
|
2665
|
+
*
|
|
2666
|
+
* `sourceRefs` 只说了「这条结论出自哪几份 SourceRaw」,而模型要核对一个数字时需要的
|
|
2667
|
+
* 是**那一段原文**。这里把它解析出来放进上下文,于是「每个数字都能指回它的口径定义与
|
|
2668
|
+
* 原始文档」在上下文层面成立,而不是靠模型自觉再调一次工具。
|
|
2669
|
+
*/
|
|
2670
|
+
private async attachEvidence(hits: RecallHit[], queryTerms: string[], corpus: Map<string, { title: string; content: string }>): Promise<void> {
|
|
2671
|
+
if (hits.length === 0 || queryTerms.length === 0) return
|
|
2672
|
+
for (const hit of hits) {
|
|
2673
|
+
const evidence: RecallEvidence[] = []
|
|
2674
|
+
for (const ref of hit.sourceRefs) {
|
|
2675
|
+
const source = corpus.get(ref)
|
|
2676
|
+
if (!source) continue
|
|
2677
|
+
evidence.push(...evidenceExcerpts(ref, source.title, source.content, queryTerms))
|
|
2678
|
+
}
|
|
2679
|
+
if (evidence.length > 0) {
|
|
2680
|
+
hit.evidence = evidence
|
|
2681
|
+
.sort((left, right) => right.coverage - left.coverage || left.sourceId.localeCompare(right.sourceId))
|
|
2682
|
+
.slice(0, EVIDENCE_PER_HIT)
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
async recall(
|
|
2688
|
+
query: string,
|
|
2689
|
+
limit?: number,
|
|
2690
|
+
sourceTypes: string[] = [],
|
|
2691
|
+
includeProblems = false,
|
|
2692
|
+
mode?: RecallMode,
|
|
2693
|
+
lineage?: LineageContext,
|
|
2694
|
+
options: { contextTopN?: number; rerank?: boolean } = {},
|
|
2695
|
+
): Promise<Envelope<RecallResult>> {
|
|
2696
|
+
const recallId = generatedId('recall')
|
|
2697
|
+
const recallMode = mode ?? this.config.recallMode
|
|
2698
|
+
const recallTopK = limit ?? this.config.recallTopK
|
|
2699
|
+
const requestedTopN = options.contextTopN ?? this.config.contextTopN
|
|
2700
|
+
const contextTopN = Math.min(requestedTopN, Number.isInteger(recallTopK) && recallTopK > 0 ? recallTopK : requestedTopN)
|
|
2701
|
+
/**
|
|
2702
|
+
* 候选池:融合与重排都在它上面做,`recallTopK` 只决定最终给模型几条。
|
|
2703
|
+
*
|
|
2704
|
+
* 0014 早就点明「`limit` 同时承担候选数与最终上下文数」是缺陷:重排作用在已经截断的
|
|
2705
|
+
* 列表上,只能换顺序,救不回被融合排到窗口外的那一条。所以两者彻底分开,而候选池
|
|
2706
|
+
* 默认 50(与 WeKnora 的 `EmbeddingTopK` 同量级)。
|
|
2707
|
+
*/
|
|
2708
|
+
const candidatePool = Math.max(recallTopK, Math.min(200, Number.isInteger(this.config.recallCandidatePool) && this.config.recallCandidatePool > 0 ? this.config.recallCandidatePool : 50))
|
|
2709
|
+
const emptyDiagnostics = (reason?: string): RecallDiagnostics => ({
|
|
2710
|
+
routes: [], fusion: { method: 'rrf', k: RRF_K, weights: RRF_WEIGHTS }, graphBoost: { applied: 0, weight: GRAPH_BOOST_WEIGHT },
|
|
2711
|
+
abstention: 'relative-only', ...(reason ? { reason } : {}),
|
|
2712
|
+
})
|
|
2713
|
+
const empty = (): RecallResult => ({
|
|
2714
|
+
hits: [], context: [], wiki: [], graph: { nodes: [], edges: [] }, backend: 'fts', routes: [], recallMode,
|
|
2715
|
+
rerank: { status: 'skipped', provider: this.config.rerankProvider },
|
|
2716
|
+
recallTopK: Number.isInteger(recallTopK) ? recallTopK : this.config.recallTopK,
|
|
2717
|
+
candidatePool, contextTopN: Number.isInteger(contextTopN) ? contextTopN : this.config.contextTopN,
|
|
2718
|
+
insufficientEvidence: true, diagnostics: emptyDiagnostics(), recallId,
|
|
2719
|
+
})
|
|
2720
|
+
if (!query?.trim()) return failure(empty(), 'QUERY_REQUIRED', 'query 不能为空', true)
|
|
2721
|
+
if (!Number.isInteger(recallTopK) || recallTopK < 1 || recallTopK > 50) return failure(empty(), 'LIMIT_INVALID', '召回 top_k 必须是 1 到 50 的整数', false)
|
|
2722
|
+
if (!Number.isInteger(requestedTopN) || requestedTopN < 1 || (options.contextTopN !== undefined && requestedTopN > recallTopK)) return failure(empty(), 'CONTEXT_TOP_N_INVALID', `上下文 top_n 必须是 1 到 ${recallTopK} 的整数`, false)
|
|
2723
|
+
if (this.units.length === 0) {
|
|
2724
|
+
await this.loadCompiledState()
|
|
2725
|
+
if (this.units.length === 0) await this.rebuild('all', lineage)
|
|
2726
|
+
}
|
|
2727
|
+
const eventLineage = withRun(lineage, 'recall')
|
|
2728
|
+
const indexVersion = (await this.loadVectorIndex())?.manifest.version
|
|
2729
|
+
let compileRunId: string | undefined
|
|
2730
|
+
try {
|
|
2731
|
+
const manifest = JSON.parse(await readFile(this.safePath('03_compiled', 'build-manifest.json'), 'utf8')) as { compileRunId?: unknown }
|
|
2732
|
+
if (typeof manifest.compileRunId === 'string' && manifest.compileRunId.trim()) compileRunId = manifest.compileRunId
|
|
2733
|
+
} catch { /* the first build has not happened yet */ }
|
|
2734
|
+
|
|
2735
|
+
const wanted = new Set(sourceTypes.filter(Boolean))
|
|
2736
|
+
const eligible = this.units.filter(unit => includeProblems || unit.status !== 'problem').filter(unit => wanted.size === 0 || unit.sourceTypes.some(sourceType => wanted.has(sourceType)))
|
|
2737
|
+
const eligibleIds = new Set(eligible.map(unit => unit.unitId))
|
|
2738
|
+
const queryTerms = [...new Set(segment(query))]
|
|
2739
|
+
const warnings: string[] = []
|
|
2740
|
+
// 一份 SourceRaw 语料,两处要用:关键词路的**查询词覆盖率**必须对着「建索引时那份
|
|
2741
|
+
// 文本」算(含来源标题),而证据片段要对着来源正文取。按 runtime 缓存,写入后失效。
|
|
2742
|
+
const corpus = await this.sourceCorpus()
|
|
2743
|
+
const titleOf = (sourceId: string): string | undefined => corpus.get(sourceId)?.title
|
|
2744
|
+
|
|
2745
|
+
// --- R1 · 候选生成(每一路各自超采、各自过阈值)------------------------
|
|
2746
|
+
// 图谱不再是「第三路」:它不能制造候选,只在 R2 里给已有候选加权(见 `graphBoost`)。
|
|
2747
|
+
// 所以 `mode: 'graph'` 与 `hybrid` 同义 —— 一个「只靠图谱」的召回在设计上不成立。
|
|
2748
|
+
const useFts = recallMode !== 'vector'
|
|
2749
|
+
const useVector = recallMode !== 'fts'
|
|
2750
|
+
const useGraphBoost = recallMode === 'hybrid' || recallMode === 'graph'
|
|
2751
|
+
|
|
2752
|
+
const ftsOutcome = useFts
|
|
2753
|
+
? await this.ftsRoute(query, eligible, candidatePool, this.config.keywordThreshold, titleOf)
|
|
2754
|
+
: { ranking: [] as RouteRanking[], diagnostic: { route: 'fts' as const, status: 'skipped' as const, reason: '当前模式未启用关键词路' } }
|
|
2755
|
+
const vectorOutcome = useVector
|
|
2756
|
+
? await this.vectorRoute(query, eligibleIds, candidatePool, this.config.vectorThreshold)
|
|
2757
|
+
: { ranking: [] as RouteRanking[], diagnostic: { route: 'vector' as const, status: 'skipped' as const, reason: '当前模式未启用向量路' }, health: { ok: false, reason: '当前模式未启用向量路' } }
|
|
2758
|
+
const graphRanking = useGraphBoost ? this.graphBoostRanking(query, eligible, includeProblems) : []
|
|
2759
|
+
|
|
2760
|
+
if (ftsOutcome.diagnostic.status === 'degraded' && ftsOutcome.diagnostic.reason) warnings.push(ftsOutcome.diagnostic.reason)
|
|
2761
|
+
if (vectorOutcome.diagnostic.status === 'degraded' && vectorOutcome.diagnostic.reason) warnings.push(vectorOutcome.diagnostic.reason)
|
|
2762
|
+
|
|
2763
|
+
// --- R2 · 名次级融合(RRF)+ 图谱 boost ---------------------------------
|
|
2764
|
+
const fused = rrfFuse({ fts: ftsOutcome.ranking, vector: vectorOutcome.ranking }, { k: RRF_K, weights: RRF_WEIGHTS })
|
|
2765
|
+
const boosted = graphBoost(fused, graphRanking, { k: RRF_K, weights: RRF_WEIGHTS })
|
|
2766
|
+
|
|
2767
|
+
const unitById = new Map(eligible.map(unit => [unit.unitId, unit]))
|
|
2768
|
+
let hits: RecallHit[] = boosted.hits
|
|
2769
|
+
.filter(hit => unitById.has(hit.unitId))
|
|
2770
|
+
.map(hit => ({ ...unitById.get(hit.unitId)!, score: hit.score, match: matchOf(hit.ranks), ranks: hit.ranks }))
|
|
2771
|
+
|
|
2772
|
+
// --- R3 · 对**候选池**重排,再过阈值 ------------------------------------
|
|
2773
|
+
let rerank: RerankResult
|
|
2774
|
+
if (options.rerank === false || this.config.rerankProvider === 'none') {
|
|
2775
|
+
rerank = { status: 'skipped', provider: this.config.rerankProvider }
|
|
2776
|
+
hits = hits.slice(0, recallTopK)
|
|
2777
|
+
} else {
|
|
2778
|
+
const reranked = await this.rerankNotes(query, hits, this.config.rerankThreshold)
|
|
2779
|
+
hits = reranked.hits.slice(0, recallTopK)
|
|
2780
|
+
rerank = reranked.result
|
|
2781
|
+
if (reranked.warning) warnings.push(reranked.warning)
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
// --- R4 · 证据片段 -----------------------------------------------------
|
|
2785
|
+
await this.attachEvidence(hits, queryTerms, corpus)
|
|
2786
|
+
|
|
2787
|
+
// --- R5 · 弃答 ---------------------------------------------------------
|
|
2788
|
+
// 一路都没过阈值 = 这个知识库里没有能回答这个问题的东西。与其凑满 topN 让模型编,
|
|
2789
|
+
// 不如明确说「没有」:这是产品诚实性的一部分,也是 eval 里「不应召回」样本的落点。
|
|
2790
|
+
const insufficientEvidence = hits.length === 0
|
|
2791
|
+
const ranRoutes = ([ftsOutcome, vectorOutcome] as const).filter(outcome => outcome.diagnostic.status === 'ran').map(outcome => outcome.diagnostic.route)
|
|
2792
|
+
const backend = ranRoutes.includes('vector') ? (ranRoutes.includes('fts') ? 'hybrid' : 'vector') : 'fts'
|
|
2793
|
+
/**
|
|
2794
|
+
* 弃答的**判据强度**要如实标注 —— 说「没有」的时候,读者有权知道凭什么。
|
|
2795
|
+
*
|
|
2796
|
+
* 三种情况差别很大:重排阈值与向量余弦都是绝对判据;而只剩相对阈值时,它挡得住弱命中、
|
|
2797
|
+
* 挡不住「整批都弱」。
|
|
2798
|
+
*
|
|
2799
|
+
* 归因顺序也修过一次。原来只看「重排跑没跑」就报 `rerank-threshold`,是**过度声称**:
|
|
2800
|
+
* 当时实测 `dropped` 恒为 0、logit 挤在 0.68–0.73 —— 那不是「重排很保守」,而是
|
|
2801
|
+
* 分词器词表是空的(见 `RERANK_TOKENIZER_DEGENERATE`),模型在给 `<unk>` 排序。
|
|
2802
|
+
* 现在重排真的会丢东西,所以判据可以按「谁真的丢了」来归因,而不是按「谁跑过」。
|
|
2803
|
+
*/
|
|
2804
|
+
const vectorDropped = (vectorOutcome.diagnostic.candidates ?? 0) > 0 && (vectorOutcome.diagnostic.kept ?? 0) < (vectorOutcome.diagnostic.candidates ?? 0)
|
|
2805
|
+
const rerankDropped = rerank.status === 'ready' && (rerank.dropped ?? 0) > 0
|
|
2806
|
+
const abstention: RecallDiagnostics['abstention'] = !insufficientEvidence
|
|
2807
|
+
? 'not-needed'
|
|
2808
|
+
: rerankDropped ? 'rerank-threshold'
|
|
2809
|
+
: vectorDropped ? 'vector-threshold'
|
|
2810
|
+
: 'relative-only'
|
|
2811
|
+
const diagnostics: RecallDiagnostics = {
|
|
2812
|
+
routes: [ftsOutcome.diagnostic, vectorOutcome.diagnostic],
|
|
2813
|
+
fusion: { method: 'rrf', k: RRF_K, weights: RRF_WEIGHTS },
|
|
2814
|
+
graphBoost: { applied: boosted.applied, weight: GRAPH_BOOST_WEIGHT },
|
|
2815
|
+
abstention,
|
|
2816
|
+
encoder: vectorOutcome.health,
|
|
2817
|
+
...(insufficientEvidence ? {
|
|
2818
|
+
reason: rerankDropped
|
|
2819
|
+
? `没有内容达到重排阈值 ${this.config.rerankThreshold}(候选 ${fused.length} 条,全部被丢弃;最高 logit ${(rerank.topLogit ?? 0).toFixed(2)})`
|
|
2820
|
+
: `没有内容达到召回阈值(关键词 ${this.config.keywordThreshold} / 向量余弦 ${this.config.vectorThreshold})`,
|
|
2821
|
+
} : {}),
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
// --- 组装:片段 → 整篇 → 证据 -------------------------------------------
|
|
2825
|
+
const cards = this.cards.length > 0 ? this.cards : await this.loadCards()
|
|
2826
|
+
const cardByKey = new Map(cards.map(card => [`${card.entityType}:${card.entityId}`, card]))
|
|
2827
|
+
const wikiKey = (note: Pick<RecallHit, 'entityType' | 'entityId'>) => `${note.entityType}:${note.entityId}`
|
|
2828
|
+
const wiki = [...new Set(hits.map(wikiKey))].map(key => cardByKey.get(key)).filter((card): card is KnowledgeCard => card !== undefined)
|
|
2829
|
+
const graph = this.oneHopGraph(new Set(hits.map(wikiKey)), includeProblems)
|
|
2830
|
+
// 上下文条目的选择**不是**简单切片:每张 wiki 封顶 + MMR 去冗余,否则 5 条上下文
|
|
2831
|
+
// 可能全是同一张卡的不同小节(一屏塞满、信息量为 1)。
|
|
2832
|
+
const contextHits = selectContext(hits, {
|
|
2833
|
+
relevance: hit => hit.rerankScore ?? hit.score,
|
|
2834
|
+
tokens: hit => tokenSet(`${hit.title}\n${hit.content}`),
|
|
2835
|
+
wikiKey: hit => hit.entityId,
|
|
2836
|
+
limit: contextTopN,
|
|
2837
|
+
perWikiCap: CONTEXT_PER_WIKI_CAP,
|
|
2838
|
+
})
|
|
2839
|
+
const context: ContextItem[] = contextHits.map(note => ({
|
|
2840
|
+
note,
|
|
2841
|
+
wiki: cardByKey.get(wikiKey(note)) ?? null,
|
|
2842
|
+
graph: this.oneHopGraph(new Set([wikiKey(note)]), includeProblems),
|
|
2843
|
+
...(note.evidence ? { evidence: note.evidence } : {}),
|
|
2844
|
+
}))
|
|
2845
|
+
|
|
2846
|
+
const data: RecallResult = {
|
|
2847
|
+
hits, context, wiki, graph, backend, routes: ranRoutes, recallMode, rerank, recallTopK, candidatePool, contextTopN,
|
|
2848
|
+
insufficientEvidence, diagnostics, recallId,
|
|
2849
|
+
...(indexVersion === undefined ? {} : { indexVersion }),
|
|
2850
|
+
...(compileRunId === undefined ? {} : { compileRunId }),
|
|
2851
|
+
...(eventLineage ? { lineage: eventLineage } : {}),
|
|
2852
|
+
}
|
|
2853
|
+
await this.audit('knowledge.recall', {
|
|
2854
|
+
query: query.slice(0, 4000), count: hits.length, context: context.length, backend, routes: ranRoutes,
|
|
2855
|
+
rerank: rerank.status, rerankThreshold: this.config.rerankThreshold, candidatePool,
|
|
2856
|
+
...(rerank.dropped === undefined ? {} : { rerankDropped: rerank.dropped }),
|
|
2857
|
+
...(rerank.topLogit === undefined ? {} : { rerankTopLogit: Number(rerank.topLogit.toFixed(4)) }),
|
|
2858
|
+
...(rerank.vocabSize === undefined ? {} : { rerankVocabSize: rerank.vocabSize }),
|
|
2859
|
+
insufficientEvidence, routesDetail: diagnostics.routes,
|
|
2860
|
+
...(indexVersion === undefined ? {} : { indexVersion }), ...(compileRunId === undefined ? {} : { compileRunId }),
|
|
2861
|
+
...(eventLineage ? { lineage: eventLineage } : {}),
|
|
2862
|
+
})
|
|
2863
|
+
return success(data, warnings)
|
|
2864
|
+
}
|
|
2865
|
+
|
|
2866
|
+
/**
|
|
2867
|
+
* 向量路的原始余弦(0031 修了两个数)。
|
|
2868
|
+
*
|
|
2869
|
+
* 1. **分母**:向量在入库与查询时都已归一(实测 `|v| = 0.9992`),量化是 `round(v × 127)`,
|
|
2870
|
+
* 所以 `Σ q_i·s_i ≈ 127²·cos`,正确分母是 `127²`。0030 写的是 `127² × dimension`,
|
|
2871
|
+
* 多除了一个 1024 —— 任何余弦都被压进 ±0.001,再经 `(cos+1)/2` 之后所有候选都落在
|
|
2872
|
+
* 0.5 附近,**排序完全没有区分度**(实测:某行自相似度算出来是 0.000975,应为 1.0)。
|
|
2873
|
+
* 2. **量纲**:返回**原始余弦**,不再做 `(cos+1)/2` 的映射 —— 那个映射让 `0.15` 这类阈值
|
|
2874
|
+
* 失去意义(余弦 0 → 0.5,轻松过线)。原始余弦是绝对量,可以直接用于弃答。
|
|
2875
|
+
*/
|
|
2876
|
+
private async vectorRecall(query: string, eligibleIds: Set<string>, candidateLimit: number): Promise<Map<string, number>> {
|
|
2877
|
+
const index = await this.loadVectorIndex()
|
|
2878
|
+
if (!index || index.manifest.itemCount === 0) throw new Error('VECTOR_INDEX_NOT_READY')
|
|
2879
|
+
if (index.manifest.provider !== this.config.embeddingProvider || index.manifest.model !== this.embeddingModelId || index.manifest.dimension !== this.config.embeddingDimension) throw new Error('VECTOR_INDEX_CONFIG_MISMATCH')
|
|
2880
|
+
const encoded = await this.encodeEmbeddings([{ text: query }])
|
|
2881
|
+
const queryBinary = binaryVector(encoded.vectors[0]!, index.manifest.dimension)
|
|
2882
|
+
const queryInt8 = int8Vector(encoded.vectors[0]!, index.manifest.dimension)
|
|
2883
|
+
const binaryWidth = Math.ceil(index.manifest.dimension / 8)
|
|
2884
|
+
const candidates: Array<{ row: number; distance: number }> = []
|
|
2885
|
+
for (let row = 0; row < index.manifest.itemCount; row += 1) {
|
|
2886
|
+
const item = index.manifest.items[row]
|
|
2887
|
+
if (!item || !eligibleIds.has(item.unitId)) continue
|
|
2888
|
+
let distance = 0
|
|
2889
|
+
const offset = row * binaryWidth
|
|
2890
|
+
for (let byte = 0; byte < binaryWidth; byte += 1) distance += BIT_COUNTS[queryBinary[byte]! ^ index.binary[offset + byte]!]!
|
|
2891
|
+
if (candidates.length < candidateLimit || distance < candidates[candidates.length - 1]!.distance) {
|
|
2892
|
+
candidates.push({ row, distance })
|
|
2893
|
+
candidates.sort((left, right) => left.distance - right.distance || left.row - right.row)
|
|
2894
|
+
if (candidates.length > candidateLimit) candidates.pop()
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
const scores = new Map<string, number>()
|
|
2898
|
+
for (const candidate of candidates) {
|
|
2899
|
+
const item = index.manifest.items[candidate.row]!
|
|
2900
|
+
const offset = candidate.row * index.manifest.dimension
|
|
2901
|
+
let dot = 0
|
|
2902
|
+
for (let dimension = 0; dimension < index.manifest.dimension; dimension += 1) dot += signedByte(queryInt8[dimension]!) * signedByte(index.int8[offset + dimension]!)
|
|
2903
|
+
scores.set(item.unitId, Math.max(-1, Math.min(1, dot / (127 * 127))))
|
|
2904
|
+
}
|
|
2905
|
+
return scores
|
|
2906
|
+
}
|
|
2907
|
+
|
|
2908
|
+
/**
|
|
2909
|
+
* 解析后的三个「家当」路径(0029)。
|
|
2910
|
+
*
|
|
2911
|
+
* 设置面板用它显示「自动」到底指向哪里:配置里那三个字段空着就是「跟着根路径
|
|
2912
|
+
* 走」,但用户要看到的是**真实路径**,否则「自动」等于没说。
|
|
2913
|
+
*/
|
|
2914
|
+
resourcePaths(): { python: string; embeddingModel: string; rerankModel: string; embeddingModelId: string } {
|
|
2915
|
+
return {
|
|
2916
|
+
python: this.pythonPath,
|
|
2917
|
+
embeddingModel: this.embeddingModelPath,
|
|
2918
|
+
rerankModel: this.rerankModelPath,
|
|
2919
|
+
embeddingModelId: this.embeddingModelId,
|
|
2920
|
+
}
|
|
2921
|
+
}
|
|
2922
|
+
|
|
2923
|
+
async embeddingHealth(): Promise<Envelope<Record<string, unknown>>> {
|
|
2924
|
+
const index = await this.loadVectorIndex()
|
|
2925
|
+
return success({
|
|
2926
|
+
provider: this.config.embeddingProvider,
|
|
2927
|
+
model: this.embeddingModelPath,
|
|
2928
|
+
configuredDevice: this.config.embeddingDevice,
|
|
2929
|
+
dimension: this.config.embeddingDimension,
|
|
2930
|
+
batchSize: this.config.embeddingBatchSize,
|
|
2931
|
+
worker: this.config.embeddingProvider === 'wemm-python' ? this.vectorWorkerPath() : undefined,
|
|
2932
|
+
vectorIndex: index ? { ready: true, version: index.manifest.version, itemCount: index.manifest.itemCount, dimension: index.manifest.dimension, device: index.manifest.device, quantization: index.manifest.quantization } : { ready: false, itemCount: 0 },
|
|
2933
|
+
lastError: this.vectorError,
|
|
2934
|
+
})
|
|
2935
|
+
}
|
|
2936
|
+
|
|
2937
|
+
/**
|
|
2938
|
+
* Stage 5(旁路):读 SourceRaw 原文,**带配额**。
|
|
2939
|
+
*
|
|
2940
|
+
* 配额是「top `max_chars` 字符」(默认与上限都是 40000),而且是**跨多条来源共享**的,
|
|
2941
|
+
* 所以一份长文档很可能只回来开头。这件事必须**说出来**,不能靠调用方自己猜:
|
|
2942
|
+
*
|
|
2943
|
+
* - 每条带上 `path`(磁盘绝对路径)与 `totalChars`;
|
|
2944
|
+
* - 信封上带 `truncated`。
|
|
2945
|
+
*
|
|
2946
|
+
* 看到 `truncated: true` 或 `returnedChars < totalChars`,就该去读那个文件 —— 这条
|
|
2947
|
+
* 路径是知识库里的真实文件,读它与读这里的内容是同一份数据,而且没有配额。这样
|
|
2948
|
+
* 「一次读不完」就不再等于「读不到」,也不会出现「以为拿到全文、其实只有前 4 万字符」。
|
|
2949
|
+
*/
|
|
2950
|
+
async sourceTruth(query: string, sourceTypes: string[] = [], limit = 20, maxChars = 40000, lineage?: LineageContext): Promise<Envelope<{ sources: SourceRawRead[]; sourceReadId: string; maxChars: number; returnedChars: number; truncated: boolean; lineage?: LineageContext }>> {
|
|
2951
|
+
const sourceReadId = generatedId('source-read')
|
|
2952
|
+
const empty = { sources: [] as SourceRawRead[], sourceReadId, maxChars, returnedChars: 0, truncated: false }
|
|
2953
|
+
if (!this.config.allowSourceTruth) return failure(empty, 'SOURCE_TRUTH_DISABLED', '当前配置禁止访问 SourceTruth', false)
|
|
2954
|
+
if (limit < 1 || limit > 20 || maxChars < 1 || maxChars > 40000) return failure(empty, 'SOURCE_LIMIT_INVALID', 'limit 最大 20,max_chars 最大 40000;需要更多就按返回的 path 直接读文件', false)
|
|
2955
|
+
const wanted = new Set(sourceTypes.filter(Boolean))
|
|
2956
|
+
const queryTokens = tokens(query)
|
|
2957
|
+
const sources = (await this.loadSources()).filter(item => wanted.size === 0 || wanted.has(item.sourceType)).filter(item => !queryTokens.length || queryTokens.some(token => `${item.title}\n${item.content}`.toLocaleLowerCase().includes(token))).slice(0, limit)
|
|
2958
|
+
// 一次把索引取出来(它整段只扫一遍),避免每条来源各扫一次目录。
|
|
2959
|
+
const index = await this.sourceIndexMap()
|
|
2960
|
+
let used = 0
|
|
2961
|
+
let truncated = false
|
|
2962
|
+
const bounded: SourceRawRead[] = []
|
|
2963
|
+
for (const source of sources) {
|
|
2964
|
+
const remaining = Math.max(0, maxChars - used)
|
|
2965
|
+
const content = source.content.slice(0, remaining)
|
|
2966
|
+
if (content.length < source.content.length) truncated = true
|
|
2967
|
+
used += content.length
|
|
2968
|
+
// 配额用尽后剩下的来源直接丢掉(它们只会有空 content),而不是返回一堆空壳。
|
|
2969
|
+
if (content.length === 0 && sources.length > 0) continue
|
|
2970
|
+
bounded.push({
|
|
2971
|
+
...source,
|
|
2972
|
+
content,
|
|
2973
|
+
// 索引命中不了时退回按 id + 分类算出来的路径:那条路径**就是**写入时用的,
|
|
2974
|
+
// 所以它仍然指向真实文件(索引缺失只可能是并发写入还没失效缓存)。
|
|
2975
|
+
path: index.get(slug(source.sourceId)) ?? this.sourcePath(source.sourceId, source.sourceType),
|
|
2976
|
+
returnedChars: content.length,
|
|
2977
|
+
totalChars: source.content.length,
|
|
2978
|
+
})
|
|
2979
|
+
}
|
|
2980
|
+
const eventLineage = withRun(lineage, 'source-read')
|
|
2981
|
+
await this.audit('source_truth.read', { query: query.slice(0, 4000), count: bounded.length, sourceReadId, ...(eventLineage ? { lineage: eventLineage } : {}) })
|
|
2982
|
+
return success({ sources: bounded, sourceReadId, maxChars, returnedChars: used, truncated, ...(eventLineage ? { lineage: eventLineage } : {}) })
|
|
2983
|
+
}
|
|
2984
|
+
|
|
2985
|
+
async graphQuery(action: 'entity' | 'neighbors' | 'path' | 'list' | 'types', entityId?: string, entityType?: string, depth = 1, options: { fromId?: string; toId?: string; status?: string; includeProblem?: boolean; limit?: number } = {}, lineage?: LineageContext): Promise<Envelope<Record<string, unknown>>> {
|
|
2986
|
+
const eventLineage = withRun(lineage, 'graph-query')
|
|
2987
|
+
if (this.units.length === 0) await this.rebuild('all', eventLineage)
|
|
2988
|
+
const attachLineage = (data: Record<string, unknown>): Record<string, unknown> => eventLineage ? { ...data, lineage: eventLineage } : data
|
|
2989
|
+
if (depth < 1 || depth > 3) return failure({}, 'DEPTH_INVALID', 'depth 必须在 1 到 3 之间', false)
|
|
2990
|
+
const limit = options.limit ?? 50
|
|
2991
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 50) return failure({}, 'LIMIT_INVALID', 'limit 必须在 1 到 50 之间', false)
|
|
2992
|
+
if (action === 'types') return success(attachLineage({ types: this.wikiSchema().types.filter(def => def.enabled).map(def => def.id), relationTypes: ['supported_by', RELATES_TO] }))
|
|
2993
|
+
|
|
2994
|
+
const includeProblem = options.includeProblem === true
|
|
2995
|
+
const visible = (node: Record<string, unknown>) => includeProblem || node.status !== 'problem'
|
|
2996
|
+
const allNodes = this.graph.nodes
|
|
2997
|
+
.filter(visible)
|
|
2998
|
+
.filter(node => !options.status || node.status === options.status)
|
|
2999
|
+
const nodes = allNodes.filter(node => !entityType || node.type === entityType)
|
|
3000
|
+
const edges = this.graph.edges.filter(edge => allNodes.some(node => `${node.type}:${node.id}` === String(edge.from)) && allNodes.some(node => `${node.type}:${node.id}` === String(edge.to)))
|
|
3001
|
+
if (action === 'list') return success(attachLineage({ nodes: nodes.slice(0, limit), edges: edges.slice(0, limit) }))
|
|
3002
|
+
|
|
3003
|
+
const nodeKey = (rawId: string | undefined, rawType?: string): string | undefined => {
|
|
3004
|
+
if (!rawId) return undefined
|
|
3005
|
+
if (rawId.includes(':')) return rawId
|
|
3006
|
+
if (rawType) return `${rawType}:${rawId}`
|
|
3007
|
+
const match = allNodes.find(node => node.id === rawId)
|
|
3008
|
+
return match ? `${match.type}:${match.id}` : undefined
|
|
3009
|
+
}
|
|
3010
|
+
const fromKey = nodeKey(options.fromId ?? (action === 'path' ? entityId : undefined), entityType)
|
|
3011
|
+
const toKey = nodeKey(options.toId)
|
|
3012
|
+
if (action === 'path') {
|
|
3013
|
+
if (!fromKey || !toKey) return failure({}, 'PATH_ENDPOINTS_REQUIRED', 'path 需要 from_id 和 to_id(也可用 entity_id 作为 from_id)', true)
|
|
3014
|
+
const queue: Array<{ key: string; path: string[] }> = [{ key: fromKey, path: [fromKey] }]
|
|
3015
|
+
const visited = new Set([fromKey])
|
|
3016
|
+
let found: string[] | undefined
|
|
3017
|
+
while (queue.length > 0 && found === undefined) {
|
|
3018
|
+
const current = queue.shift()!
|
|
3019
|
+
if (current.key === toKey) { found = current.path; break }
|
|
3020
|
+
if (current.path.length - 1 >= depth) continue
|
|
3021
|
+
const nextKeys = edges.flatMap(edge => {
|
|
3022
|
+
const from = String(edge.from); const to = String(edge.to)
|
|
3023
|
+
return from === current.key ? [to] : to === current.key ? [from] : []
|
|
3024
|
+
})
|
|
3025
|
+
for (const next of nextKeys) if (!visited.has(next)) { visited.add(next); queue.push({ key: next, path: [...current.path, next] }) }
|
|
3026
|
+
}
|
|
3027
|
+
if (!found) return success(attachLineage({ path: [], nodes: [], edges: [] }))
|
|
3028
|
+
return success(attachLineage({ path: found, nodes: allNodes.filter(node => found!.includes(`${node.type}:${node.id}`)), edges: edges.filter(edge => found!.includes(String(edge.from)) && found!.includes(String(edge.to))) }))
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
const id = nodeKey(entityId, entityType)
|
|
3032
|
+
if (!id) return failure({}, 'ENTITY_REQUIRED', '该 graph action 需要 entity_id', true)
|
|
3033
|
+
if (action === 'entity') return success(attachLineage({ nodes: nodes.filter(node => `${node.type}:${node.id}` === id).slice(0, limit) }))
|
|
3034
|
+
if (action === 'neighbors') {
|
|
3035
|
+
const ids = new Set([id])
|
|
3036
|
+
for (let index = 0; index < depth; index++) for (const edge of edges) {
|
|
3037
|
+
if (ids.has(String(edge.from))) ids.add(String(edge.to))
|
|
3038
|
+
if (ids.has(String(edge.to))) ids.add(String(edge.from))
|
|
3039
|
+
}
|
|
3040
|
+
return success(attachLineage({ nodes: allNodes.filter(node => ids.has(`${node.type}:${node.id}`)).slice(0, limit), edges: edges.filter(edge => ids.has(String(edge.from)) || ids.has(String(edge.to))).slice(0, limit) }))
|
|
3041
|
+
}
|
|
3042
|
+
return failure({}, 'ACTION_INVALID', `不支持的 graph action: ${action}`, false)
|
|
3043
|
+
}
|
|
3044
|
+
|
|
3045
|
+
async recordFeedback(input: { feedbackId?: string; unitId?: string; rating: number; comment?: string; sessionKey?: string; lineage?: LineageContext }): Promise<Envelope<{ feedbackId: string; lineage?: LineageContext }>> {
|
|
3046
|
+
if (!Number.isInteger(input.rating) || input.rating < 1 || input.rating > 5) return failure({ feedbackId: '' }, 'RATING_INVALID', 'rating 必须为 1 到 5 的整数', false)
|
|
3047
|
+
const feedbackId = input.feedbackId?.trim() || `feedback_${hash(`${Date.now()}:${input.unitId ?? ''}:${input.rating}`)}`
|
|
3048
|
+
await this.ensureLayout()
|
|
3049
|
+
const eventLineage = withRun(input.lineage, 'feedback')
|
|
3050
|
+
await appendFile(this.memoryPath('feedback.ndjson'), `${JSON.stringify({ feedbackId, ...input, ...(eventLineage ? { lineage: eventLineage } : {}), createdAt: new Date().toISOString() })}\n`, 'utf8')
|
|
3051
|
+
return success({ feedbackId, ...(eventLineage ? { lineage: eventLineage } : {}) })
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
async audit(action: string, metadata: Record<string, unknown>): Promise<void> {
|
|
3055
|
+
await this.ensureLayout()
|
|
3056
|
+
await appendFile(this.memoryPath('access-audit.ndjson'), `${JSON.stringify({ action, metadata, at: new Date().toISOString() })}\n`, 'utf8')
|
|
3057
|
+
}
|
|
3058
|
+
}
|
|
3059
|
+
|
|
3060
|
+
/**
|
|
3061
|
+
* @deprecated 0029:知识树的根不再是「一个路径」,而是
|
|
3062
|
+
* `<agentRoot>/domain_knowledge/<知识库 id>`。这里保留旧名字只为让外部调用点
|
|
3063
|
+
* (注册表的默认库、测试)有个可迁移的入口;新代码一律用 `layout.ts` 的
|
|
3064
|
+
* `knowledgeBaseRoot()`。它返回的是**旧布局**的位置,也就是迁移的来源。
|
|
3065
|
+
*/
|
|
3066
|
+
export function defaultKnowledgeRoot(): string {
|
|
3067
|
+
return legacyKnowledgeRoot()
|
|
3068
|
+
}
|
|
3069
|
+
|