dsh-data-domain-knowledge 0.1.0-alpha.11 → 0.1.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -13
- package/eval/README.md +19 -8
- package/eval/evaluate-retrieval.mjs +17 -8
- package/eval/queries.example.jsonl +1 -0
- package/lib/client.js +2294 -2096
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +2 -2
- package/lib/types/axis-registry.d.ts +9 -4
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +62 -23
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/card-organize.d.ts +5 -0
- package/lib/types/card-organize.d.ts.map +1 -1
- package/lib/types/card-organize.js +15 -0
- package/lib/types/card-organize.js.map +1 -1
- package/lib/types/compiler/embedding-identity.d.ts +3 -0
- package/lib/types/compiler/embedding-identity.d.ts.map +1 -0
- package/lib/types/compiler/embedding-identity.js +24 -0
- package/lib/types/compiler/embedding-identity.js.map +1 -0
- package/lib/types/compiler/snapshot.d.ts +16 -0
- package/lib/types/compiler/snapshot.d.ts.map +1 -0
- package/lib/types/compiler/snapshot.js +2 -0
- package/lib/types/compiler/snapshot.js.map +1 -0
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +2 -0
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/environment/install.d.ts +7 -0
- package/lib/types/environment/install.d.ts.map +1 -0
- package/lib/types/environment/install.js +172 -0
- package/lib/types/environment/install.js.map +1 -0
- package/lib/types/environment/manifest.d.ts +208 -0
- package/lib/types/environment/manifest.d.ts.map +1 -0
- package/lib/types/environment/manifest.js +26 -0
- package/lib/types/environment/manifest.js.map +1 -0
- package/lib/types/environment/model-catalog.d.ts +191 -0
- package/lib/types/environment/model-catalog.d.ts.map +1 -0
- package/lib/types/environment/model-catalog.js +91 -0
- package/lib/types/environment/model-catalog.js.map +1 -0
- package/lib/types/environment/process.d.ts +10 -0
- package/lib/types/environment/process.d.ts.map +1 -0
- package/lib/types/environment/process.js +54 -0
- package/lib/types/environment/process.js.map +1 -0
- package/lib/types/environment/selection.d.ts +9 -0
- package/lib/types/environment/selection.d.ts.map +1 -0
- package/lib/types/environment/selection.js +29 -0
- package/lib/types/environment/selection.js.map +1 -0
- package/lib/types/environment/service.d.ts +47 -0
- package/lib/types/environment/service.d.ts.map +1 -0
- package/lib/types/environment/service.js +253 -0
- package/lib/types/environment/service.js.map +1 -0
- package/lib/types/environment/types.d.ts +37 -0
- package/lib/types/environment/types.d.ts.map +1 -0
- package/lib/types/environment/types.js +2 -0
- package/lib/types/environment/types.js.map +1 -0
- package/lib/types/events.d.ts +10 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js +38 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/images.d.ts +26 -0
- package/lib/types/images.d.ts.map +1 -0
- package/lib/types/images.js +196 -0
- package/lib/types/images.js.map +1 -0
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +59 -3
- package/lib/types/index.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts +3 -0
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +9 -0
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge.d.ts +14 -6
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +56 -39
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/layout.d.ts +2 -0
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +2 -0
- package/lib/types/layout.js.map +1 -1
- package/lib/types/retrieval/vector-index.d.ts +31 -0
- package/lib/types/retrieval/vector-index.d.ts.map +1 -0
- package/lib/types/retrieval/vector-index.js +15 -0
- package/lib/types/retrieval/vector-index.js.map +1 -0
- package/lib/types/retrieval/visibility.d.ts +6 -0
- package/lib/types/retrieval/visibility.d.ts.map +1 -0
- package/lib/types/retrieval/visibility.js +11 -0
- package/lib/types/retrieval/visibility.js.map +1 -0
- package/lib/types/runtime.d.ts +36 -6
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +520 -190
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/source-evidence.d.ts +6 -0
- package/lib/types/source-evidence.d.ts.map +1 -0
- package/lib/types/source-evidence.js +28 -0
- package/lib/types/source-evidence.js.map +1 -0
- package/lib/types/store/axis-references.d.ts +5 -0
- package/lib/types/store/axis-references.d.ts.map +1 -0
- package/lib/types/store/axis-references.js +106 -0
- package/lib/types/store/axis-references.js.map +1 -0
- package/lib/types/store/derived.d.ts +11 -0
- package/lib/types/store/derived.d.ts.map +1 -1
- package/lib/types/store/derived.js +31 -6
- package/lib/types/store/derived.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +4 -1
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +15 -9
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/migrate.d.ts.map +1 -1
- package/lib/types/store/migrate.js +16 -5
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/types.d.ts +7 -1
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +3 -4
- package/lib/types/types.js.map +1 -1
- package/package.json +3 -1
- package/python/prepare-models.py +19 -0
- package/python/requirements.txt +8 -6
- package/scripts/doctor.mjs +14 -2
- package/scripts/environment.mjs +29 -0
- package/scripts/setup-embedding.sh +11 -2
- package/src/axis-registry.ts +59 -23
- package/src/card-organize.ts +15 -0
- package/src/client/EnvironmentSetup.tsx +145 -0
- package/src/client/KnowledgePanel.tsx +9 -4
- package/src/compiler/embedding-identity.ts +21 -0
- package/src/compiler/snapshot.ts +17 -0
- package/src/diagnosis/store.ts +1 -0
- package/src/environment/install.ts +129 -0
- package/src/environment/manifest.ts +28 -0
- package/src/environment/model-catalog.ts +105 -0
- package/src/environment/process.ts +54 -0
- package/src/environment/selection.ts +29 -0
- package/src/environment/service.ts +211 -0
- package/src/environment/types.ts +34 -0
- package/src/events.ts +46 -0
- package/src/images.ts +203 -0
- package/src/index.ts +54 -3
- package/src/knowledge-bases.ts +9 -0
- package/src/knowledge.ts +56 -40
- package/src/layout.ts +2 -0
- package/src/retrieval/vector-index.ts +44 -0
- package/src/retrieval/visibility.ts +14 -0
- package/src/runtime.ts +473 -233
- package/src/source-evidence.ts +30 -0
- package/src/store/axis-references.ts +96 -0
- package/src/store/derived.ts +31 -6
- package/src/store/knowledge-db.ts +16 -9
- package/src/store/migrate.ts +15 -5
- package/src/types.ts +10 -5
package/src/runtime.ts
CHANGED
|
@@ -25,7 +25,7 @@ import { MAX_LOG_ITEMS, OperationLog, operationId } from './oplog.js'
|
|
|
25
25
|
import { ExtractionRunStore, extractionRunId, type ExtractionRunPage, type ExtractionRunRecord, type ExtractionRunSummary } from './extraction-runs.js'
|
|
26
26
|
import { AxisRegistry, normalizeAxisName, type AxisEntity, type AxisGuidance, type AxisGuidanceKind, type AxisKind, type AxisRelation, type AxisResolution, type AxisStatus } from './axis-registry.js'
|
|
27
27
|
import { KNOWLEDGE_STATE_LABEL, KnowledgeStore, type Knowledge, type KnowledgeDocumentImpact, type KnowledgeDocumentRemoval, type KnowledgeFacets, type KnowledgeIngest, type KnowledgeIngestResult, type KnowledgeQuery, type KnowledgeState, type KnowledgeVersion } from './knowledge.js'
|
|
28
|
-
import type
|
|
28
|
+
import { compositeEventIdentity, eventEntityFromRecords, type EventEntity, type EventSourceRecord } from './events.js'
|
|
29
29
|
import type { TrialAxis, TrialEvent, TrialKnowledge, TrialSource, TrialTokenUsage } from './wiki-trial-types.js'
|
|
30
30
|
import { buildAliasIndex, candidateEntityId, extractCandidates, normalizeCandidates, RELATES_TO, type NormalizedCandidates } from './extract.js'
|
|
31
31
|
import { skeletonBody, type ModelCall } from './extract-model.js'
|
|
@@ -44,15 +44,23 @@ import {
|
|
|
44
44
|
RELATED_METADATA_KEY, type DeadLink, type RelatedEntry,
|
|
45
45
|
} from './wiki-finalize.js'
|
|
46
46
|
import { applyRerank, applyThreshold, evidenceExcerpts, graphBoost, matchOf, noteIndexText, reviewWeight, rrfFuse, tokenSet, type RouteRanking } from './recall-rank.js'
|
|
47
|
-
import { ASSETS_DIR, KNOWLEDGE_DB_FILE, SOURCE_RAW_DIR, defaultAgentRoot, knowledgeBaseRoot } from './layout.js'
|
|
47
|
+
import { ASSETS_DIR, KNOWLEDGE_DB_FILE, PUBLISHED_IMAGE_MARKER, SOURCE_RAW_DIR, defaultAgentRoot, knowledgeBaseRoot } from './layout.js'
|
|
48
48
|
import { KnowledgeDb, type CardKnowledgeRow, type Journal as StoreJournal, type SourceOriginal } from './store/knowledge-db.js'
|
|
49
49
|
import { exportKnowledgeTree as exportTree, migrateKnowledgeTree } from './store/migrate.js'
|
|
50
|
-
import {
|
|
50
|
+
import { COMPILE_SNAPSHOT, type CompileSnapshot } from './compiler/snapshot.js'
|
|
51
|
+
import { decodeVectorIndex, type LoadedVectorIndex, type VectorManifest } from './retrieval/vector-index.js'
|
|
52
|
+
import { liveGraphKeys, visibleGraph } from './retrieval/visibility.js'
|
|
53
|
+
import { embeddingFingerprint } from './compiler/embedding-identity.js'
|
|
54
|
+
import { DerivedDb, DERIVED_DB_FILE, type DerivedPublication, type FtsHit } from './store/derived.js'
|
|
51
55
|
import { cardVersions as viewCardVersions, documentDetail as viewDocumentDetail, documentList as viewDocumentList, documentVersions as viewDocumentVersions, type ViewHooks } from './store/views.js'
|
|
56
|
+
import { assessKnowledgeEvidence, hasCurrentSupport } from './source-evidence.js'
|
|
57
|
+
import { redirectAxisReferences } from './store/axis-references.js'
|
|
52
58
|
import { unlinkIfExists, writeBufferAtomic, writeTextAtomic } from './fs-atomic.js'
|
|
53
59
|
import { envelopeFailure, envelopeSuccess } from 'dsh-data-contracts/envelope'
|
|
54
60
|
import type { LineageContext, LineageRef } from 'dsh-data-contracts/refs'
|
|
55
61
|
import { analyzeQuery, relationCandidates, type QueryAnalysis } from './query-analysis.js'
|
|
62
|
+
import { readModelSelection, selectedEmbedding } from './environment/selection.js'
|
|
63
|
+
import { rerankModel as rerankModelOption } from './environment/model-catalog.js'
|
|
56
64
|
|
|
57
65
|
// **知识树只有三段**(v1.1 起):`01_source_raw` / `02_wiki` / `03_compiled`。
|
|
58
66
|
//
|
|
@@ -302,34 +310,6 @@ const BIT_COUNTS = Array.from({ length: 256 }, (_, value) => {
|
|
|
302
310
|
return count
|
|
303
311
|
})
|
|
304
312
|
|
|
305
|
-
interface VectorItem {
|
|
306
|
-
unitId: string
|
|
307
|
-
contentHash: string
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
interface VectorManifest {
|
|
311
|
-
version: number
|
|
312
|
-
createdAt: string
|
|
313
|
-
provider: KnowledgeConfig['embeddingProvider']
|
|
314
|
-
model: string
|
|
315
|
-
/**
|
|
316
|
-
* 编码实现的指纹(0031)。旧清单没有这个字段 → 视为不兼容,于是修复编码器之后
|
|
317
|
-
* 一定触发一次真正的重编码,而不是把旧向量按 contentHash 复用回来。
|
|
318
|
-
*/
|
|
319
|
-
encoder?: string
|
|
320
|
-
device: string
|
|
321
|
-
dimension: number
|
|
322
|
-
quantization: typeof VECTOR_QUANTIZATION
|
|
323
|
-
itemCount: number
|
|
324
|
-
items: VectorItem[]
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
interface LoadedVectorIndex {
|
|
328
|
-
manifest: VectorManifest
|
|
329
|
-
binary: Buffer
|
|
330
|
-
int8: Buffer
|
|
331
|
-
}
|
|
332
|
-
|
|
333
313
|
interface EmbeddingInput {
|
|
334
314
|
text: string
|
|
335
315
|
image?: string
|
|
@@ -586,12 +566,18 @@ function withTitleAlias(metadata: Record<string, unknown>, previousTitle: string
|
|
|
586
566
|
* 同一句话在上下文里出现很多遍。完整出处在知识接口里。
|
|
587
567
|
*/
|
|
588
568
|
function recallKnowledgeOf(row: Knowledge, section: string): RecallKnowledge {
|
|
589
|
-
const first = row.sources.find(source => source.
|
|
569
|
+
const first = row.sources.find(source => source.evidenceStatus === 'current' && source.quoteInCurrentVersion)
|
|
570
|
+
?? row.sources.find(source => source.quote.trim() !== '') ?? row.sources[0]
|
|
590
571
|
return {
|
|
591
572
|
id: row.id,
|
|
592
573
|
text: row.text,
|
|
593
574
|
state: row.status,
|
|
594
575
|
section,
|
|
576
|
+
claimVerification: 'unverified',
|
|
577
|
+
requiresReview: !hasCurrentSupport(row),
|
|
578
|
+
...(first?.evidenceStatus ? { evidenceStatus: first.evidenceStatus } : {}),
|
|
579
|
+
...(first?.toVersion === undefined ? {} : { sourceVersion: first.toVersion }),
|
|
580
|
+
...(first?.currentVersion === undefined ? {} : { currentSourceVersion: first.currentVersion }),
|
|
595
581
|
...(first?.quote ? { quote: first.quote } : {}),
|
|
596
582
|
...(first?.documentId ? { sourceId: first.documentId } : {}),
|
|
597
583
|
...(first && first.headingPath.length > 0 ? { headingPath: first.headingPath } : {}),
|
|
@@ -1092,51 +1078,6 @@ function withEventReferences(
|
|
|
1092
1078
|
return `${base ? `${base}\n\n` : ''}## 关联 Event\n\n${bullets.join('\n')}`
|
|
1093
1079
|
}
|
|
1094
1080
|
|
|
1095
|
-
function eventEntityFromRecords(input: {
|
|
1096
|
-
id: string
|
|
1097
|
-
identityKey: string
|
|
1098
|
-
identityType: EventEntity['identityType']
|
|
1099
|
-
records: EventSourceRecord[]
|
|
1100
|
-
createdAt: string
|
|
1101
|
-
updatedAt?: string
|
|
1102
|
-
}): EventEntity {
|
|
1103
|
-
const { id, identityKey, identityType, records } = input
|
|
1104
|
-
const first = records[0]
|
|
1105
|
-
const firstCoordinate = records.find(record =>
|
|
1106
|
-
record.coordinateIds.objectId || record.coordinateIds.actionId || record.coordinateIds.sceneId,
|
|
1107
|
-
)
|
|
1108
|
-
const sources = new Map<string, TrialSource>()
|
|
1109
|
-
for (const record of records) {
|
|
1110
|
-
for (const source of record.sources) sources.set(`${source.documentId}\u0001${source.quote}`, source)
|
|
1111
|
-
}
|
|
1112
|
-
return {
|
|
1113
|
-
id,
|
|
1114
|
-
identityKey,
|
|
1115
|
-
identityType,
|
|
1116
|
-
externalId: records.find(record => record.externalId)?.externalId ?? null,
|
|
1117
|
-
title: first?.title ?? id,
|
|
1118
|
-
summary: records.find(record => record.summary.trim())?.summary ?? first?.title ?? id,
|
|
1119
|
-
coordinate: {
|
|
1120
|
-
object: records.find(record => record.coordinate.object)?.coordinate.object ?? null,
|
|
1121
|
-
action: records.find(record => record.coordinate.action)?.coordinate.action ?? null,
|
|
1122
|
-
scene: records.find(record => record.coordinate.scene)?.coordinate.scene ?? null,
|
|
1123
|
-
},
|
|
1124
|
-
coordinateIds: firstCoordinate?.coordinateIds ?? { objectId: '', actionId: '', sceneId: '' },
|
|
1125
|
-
occurredAt: records.find(record => record.occurredAt)?.occurredAt ?? null,
|
|
1126
|
-
symptom: records.find(record => record.symptom)?.symptom ?? null,
|
|
1127
|
-
timeline: records.flatMap(record => record.timeline.map(step => ({ ...step, sourceId: record.sourceId }))),
|
|
1128
|
-
rootCauses: records.flatMap(record => record.rootCause
|
|
1129
|
-
? [{ text: record.rootCause, status: record.rootCauseStatus, sourceId: record.sourceId }]
|
|
1130
|
-
: []),
|
|
1131
|
-
outcomes: records.flatMap(record => record.outcome ? [{ text: record.outcome, sourceId: record.sourceId }] : []),
|
|
1132
|
-
missing: [...new Set(records.flatMap(record => record.missing))],
|
|
1133
|
-
sources: [...sources.values()],
|
|
1134
|
-
records,
|
|
1135
|
-
createdAt: input.createdAt,
|
|
1136
|
-
updatedAt: input.updatedAt ?? new Date().toISOString(),
|
|
1137
|
-
}
|
|
1138
|
-
}
|
|
1139
|
-
|
|
1140
1081
|
/** 把 `incoming` 按小节折进 `existing`;结构对不上时整段追加。 */
|
|
1141
1082
|
function foldBody(existing: string, incoming: string): string {
|
|
1142
1083
|
if (!existing) return incoming
|
|
@@ -1780,8 +1721,8 @@ export interface WikiSchemaSnapshot {
|
|
|
1780
1721
|
/**
|
|
1781
1722
|
* 解析三个「家当」路径(0029)。
|
|
1782
1723
|
*
|
|
1783
|
-
*
|
|
1784
|
-
*
|
|
1724
|
+
* 资源目录内的隔离解释器优先于 DSH_PYTHON;自定义配置路径仍由构造函数处理。
|
|
1725
|
+
* 未准备资源时回退到 DSH_PYTHON、`python3` 或 HF 模型名。
|
|
1785
1726
|
*
|
|
1786
1727
|
* `modelId` 把本地模型的**身份**与它的位置解耦,避免同一 Agent 根目录下的
|
|
1787
1728
|
* resource 目录变化导致向量清单误判为配置不匹配。
|
|
@@ -1790,6 +1731,7 @@ function resolveResources(resourceRoot: string | undefined): {
|
|
|
1790
1731
|
python: string
|
|
1791
1732
|
embeddingModel: string
|
|
1792
1733
|
rerankModel: string
|
|
1734
|
+
embeddingDimension?: number
|
|
1793
1735
|
modelId: (path: string) => string
|
|
1794
1736
|
} {
|
|
1795
1737
|
const root = resourceRoot ? resolve(resourceRoot) : undefined
|
|
@@ -1798,15 +1740,22 @@ function resolveResources(resourceRoot: string | undefined): {
|
|
|
1798
1740
|
return fallback
|
|
1799
1741
|
}
|
|
1800
1742
|
const python = firstExisting([
|
|
1743
|
+
root && process.platform === 'win32' ? join(root, 'python', 'Scripts', 'python.exe') : undefined,
|
|
1801
1744
|
root ? join(root, 'python', 'bin', 'python') : undefined,
|
|
1802
|
-
], 'python3')
|
|
1745
|
+
], process.env.DSH_PYTHON || 'python3')
|
|
1746
|
+
const selected = selectedEmbedding(root)
|
|
1747
|
+
const selection = root ? readModelSelection(root) : undefined
|
|
1748
|
+
const selectedRerank = root && selection ? join(root, 'models', rerankModelOption(selection.rerank).directory) : undefined
|
|
1803
1749
|
const embeddingModel = firstExisting([
|
|
1750
|
+
selected?.path,
|
|
1804
1751
|
root ? join(root, 'models', PREFERRED_LOCAL_EMBEDDING_MODEL_DIR) : undefined,
|
|
1805
1752
|
root ? join(root, 'models', LOCAL_EMBEDDING_MODEL_DIR) : undefined,
|
|
1806
1753
|
], FALLBACK_EMBEDDING_MODEL)
|
|
1807
1754
|
const rerankModel = firstExisting([
|
|
1755
|
+
selectedRerank,
|
|
1808
1756
|
root ? join(root, 'models', LOCAL_RERANK_MODEL_DIR) : undefined,
|
|
1809
1757
|
], FALLBACK_RERANK_MODEL)
|
|
1758
|
+
const embeddingDimension = selected?.path && embeddingModel === selected.path ? selected.dimension : undefined
|
|
1810
1759
|
const modelId = (path: string): string => {
|
|
1811
1760
|
for (const base of [root]) {
|
|
1812
1761
|
if (!base || !path.startsWith(`${base}${sep}`)) continue
|
|
@@ -1815,10 +1764,19 @@ function resolveResources(resourceRoot: string | undefined): {
|
|
|
1815
1764
|
}
|
|
1816
1765
|
return path
|
|
1817
1766
|
}
|
|
1818
|
-
return { python, embeddingModel, rerankModel, modelId }
|
|
1767
|
+
return { python, embeddingModel, rerankModel, embeddingDimension, modelId }
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
/** An inherited DSH_PYTHON value selects the base interpreter, not the dependency environment. */
|
|
1771
|
+
function resolveConfiguredPython(configured: string | undefined, resourcePython: string): string {
|
|
1772
|
+
const selected = configured?.trim()
|
|
1773
|
+
const hostPython = process.env.DSH_PYTHON?.trim()
|
|
1774
|
+
return selected && selected !== hostPython ? selected : resourcePython
|
|
1819
1775
|
}
|
|
1820
1776
|
|
|
1821
1777
|
export interface KnowledgeRuntimeOptions {
|
|
1778
|
+
/** Production deployment requires a working Python + embedding + rerank stack. */
|
|
1779
|
+
environmentGuard?: () => Promise<void>
|
|
1822
1780
|
/**
|
|
1823
1781
|
* 这一库的「坐标提取备注」(0068),人写的。
|
|
1824
1782
|
*
|
|
@@ -1831,7 +1789,7 @@ export interface KnowledgeRuntimeOptions {
|
|
|
1831
1789
|
* 0029:家当目录(`<agentRoot>/resource`)。
|
|
1832
1790
|
*
|
|
1833
1791
|
* 配置里那三个路径字段(`python` / `embeddingModel` / `rerankModel`)为空时,
|
|
1834
|
-
*
|
|
1792
|
+
* 它们从这里解析。省略时使用 DSH Python、系统命令或模型名作为兜底 —— 独立运行时与单元
|
|
1835
1793
|
* 测试不必先造一个根路径。
|
|
1836
1794
|
*/
|
|
1837
1795
|
resourceRoot?: string
|
|
@@ -2042,6 +2000,7 @@ function blankProgress(
|
|
|
2042
2000
|
}
|
|
2043
2001
|
|
|
2044
2002
|
export class KnowledgeRuntime {
|
|
2003
|
+
private readonly environmentGuard?: () => Promise<void>
|
|
2045
2004
|
/** 当前生效的配置。volatile 改动会就地合并进来(见 `updateConfig`),所以它不是只读的。 */
|
|
2046
2005
|
config: RuntimeKnowledgeConfig
|
|
2047
2006
|
readonly rootDir: string
|
|
@@ -2137,6 +2096,8 @@ export class KnowledgeRuntime {
|
|
|
2137
2096
|
*/
|
|
2138
2097
|
private derived: DerivedDb | null | undefined
|
|
2139
2098
|
private derivedProblem: string | undefined
|
|
2099
|
+
private publication?: DerivedPublication
|
|
2100
|
+
private recallSnapshotCache?: { hash: string; snapshot: CompileSnapshot; vector: LoadedVectorIndex | null }
|
|
2140
2101
|
/** 段内布局整理是否已经跑过(每个 runtime 一次)。 */
|
|
2141
2102
|
private segmentsOrganized = false
|
|
2142
2103
|
/**
|
|
@@ -2170,6 +2131,7 @@ export class KnowledgeRuntime {
|
|
|
2170
2131
|
private vectorIndex: LoadedVectorIndex | null = null
|
|
2171
2132
|
private vectorError: string | undefined
|
|
2172
2133
|
private embeddingWorker: JsonLineWorker | null = null
|
|
2134
|
+
private embeddingWorkerFingerprint?: string
|
|
2173
2135
|
/** Separate process so a rerank model failure cannot take embeddings down. */
|
|
2174
2136
|
private rerankWorker: JsonLineWorker | null = null
|
|
2175
2137
|
/** Cache the first rerank failure; do not respawn a doomed model per recall. */
|
|
@@ -2187,7 +2149,8 @@ export class KnowledgeRuntime {
|
|
|
2187
2149
|
* 清单里的模型名就对不上,整个向量索引被判成「配置不匹配」而作废,用户会莫名
|
|
2188
2150
|
* 收到一次全量重编码。清单该记「是哪一个模型」,不是「它在哪」。
|
|
2189
2151
|
*/
|
|
2190
|
-
private
|
|
2152
|
+
private embeddingModelId: string
|
|
2153
|
+
private readonly resourceRoot?: string
|
|
2191
2154
|
/**
|
|
2192
2155
|
* 这个知识库的记忆段(0032)。
|
|
2193
2156
|
*
|
|
@@ -2210,9 +2173,12 @@ export class KnowledgeRuntime {
|
|
|
2210
2173
|
this.extractionNoteProvider = options.extractionNote
|
|
2211
2174
|
this.memoryDir = options.memoryRoot ? resolve(options.memoryRoot) : join(this.rootDir, '04_memory')
|
|
2212
2175
|
this.staticModelCall = options.modelCall
|
|
2176
|
+
this.environmentGuard = options.environmentGuard
|
|
2213
2177
|
this.modelSeamProvider = options.modelSeam
|
|
2178
|
+
this.resourceRoot = options.resourceRoot
|
|
2214
2179
|
const resources = resolveResources(options.resourceRoot)
|
|
2215
|
-
this.
|
|
2180
|
+
if (!this.config.embeddingModel?.trim() && resources.embeddingDimension) this.config.embeddingDimension = resources.embeddingDimension
|
|
2181
|
+
this.pythonPath = resolveConfiguredPython(config.python, resources.python)
|
|
2216
2182
|
this.embeddingModelPath = config.embeddingModel?.trim() || resources.embeddingModel
|
|
2217
2183
|
this.rerankModelPath = config.rerankModel?.trim() || resources.rerankModel
|
|
2218
2184
|
this.embeddingModelId = resources.modelId(this.embeddingModelPath)
|
|
@@ -2224,6 +2190,10 @@ export class KnowledgeRuntime {
|
|
|
2224
2190
|
* **每次现问**,不是在构造时定死:插件 apply 的时刻早于 `agent-default-model` 挂载,
|
|
2225
2191
|
* 那时问一次只会得到 undefined,而定死之后就再也不会变(见 `modelCallProvider`)。
|
|
2226
2192
|
*/
|
|
2193
|
+
private embeddingFingerprint(): string {
|
|
2194
|
+
return embeddingFingerprint(this.embeddingModelPath, this.resourceRoot, this.config.embeddingProvider === 'ollama' ? this.config.embeddingBaseUrl : undefined)
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2227
2197
|
private modelSeamNow(): ModelSeam {
|
|
2228
2198
|
if (this.modelSeamProvider) return this.modelSeamProvider(this.baseId)
|
|
2229
2199
|
return this.staticModelCall ? { call: this.staticModelCall } : { why: '运行时没有注入模型接缝' }
|
|
@@ -2248,12 +2218,17 @@ export class KnowledgeRuntime {
|
|
|
2248
2218
|
updateConfig(patch: Partial<RuntimeKnowledgeConfig>): void {
|
|
2249
2219
|
const previous = this.config
|
|
2250
2220
|
this.config = { ...previous, ...patch }
|
|
2221
|
+
const resources = resolveResources(this.resourceRoot)
|
|
2222
|
+
// A managed model preset owns its output dimension. Custom model paths keep
|
|
2223
|
+
// the advanced dimension setting from the plugin config.
|
|
2224
|
+
if (!this.config.embeddingModel?.trim() && resources.embeddingDimension) this.config.embeddingDimension = resources.embeddingDimension
|
|
2251
2225
|
|
|
2252
2226
|
const modelChanged =
|
|
2253
2227
|
previous.embeddingProvider !== this.config.embeddingProvider ||
|
|
2254
2228
|
previous.embeddingModel !== this.config.embeddingModel ||
|
|
2255
2229
|
previous.embeddingDimension !== this.config.embeddingDimension ||
|
|
2256
2230
|
previous.embeddingDevice !== this.config.embeddingDevice ||
|
|
2231
|
+
previous.embeddingBaseUrl !== this.config.embeddingBaseUrl ||
|
|
2257
2232
|
previous.python !== this.config.python
|
|
2258
2233
|
const rerankChanged =
|
|
2259
2234
|
previous.rerankProvider !== this.config.rerankProvider ||
|
|
@@ -2266,15 +2241,17 @@ export class KnowledgeRuntime {
|
|
|
2266
2241
|
// 旧索引建在旧模型/旧维度下,留着它只会给出没有意义的相似度。
|
|
2267
2242
|
this.vectorIndex = null
|
|
2268
2243
|
this.vectorError = undefined
|
|
2244
|
+
this.encoderHealth = undefined
|
|
2269
2245
|
}
|
|
2270
2246
|
if (rerankChanged) {
|
|
2271
2247
|
this.rerankWorker?.close()
|
|
2272
2248
|
this.rerankWorker = null
|
|
2273
2249
|
this.rerankUnavailable = undefined
|
|
2274
2250
|
}
|
|
2275
|
-
|
|
2276
|
-
if (patch.embeddingModel !== undefined) this.embeddingModelPath = patch.embeddingModel.trim() ||
|
|
2277
|
-
if (patch.rerankModel !== undefined) this.rerankModelPath = patch.rerankModel.trim() ||
|
|
2251
|
+
this.pythonPath = resolveConfiguredPython(this.config.python, resources.python)
|
|
2252
|
+
if (patch.embeddingModel !== undefined) this.embeddingModelPath = patch.embeddingModel.trim() || resources.embeddingModel
|
|
2253
|
+
if (patch.rerankModel !== undefined) this.rerankModelPath = patch.rerankModel.trim() || resources.rerankModel
|
|
2254
|
+
this.embeddingModelId = resources.modelId(this.embeddingModelPath)
|
|
2278
2255
|
}
|
|
2279
2256
|
|
|
2280
2257
|
// ------------------------------------------------- 固定卡片模板
|
|
@@ -2750,22 +2727,9 @@ export class KnowledgeRuntime {
|
|
|
2750
2727
|
private async loadVectorIndex(): Promise<LoadedVectorIndex | null> {
|
|
2751
2728
|
if (this.vectorIndex) return this.vectorIndex
|
|
2752
2729
|
try {
|
|
2753
|
-
|
|
2754
|
-
if (loaded === undefined) return null
|
|
2755
|
-
const manifest = loaded.manifest as VectorManifest
|
|
2756
|
-
if (manifest.quantization !== VECTOR_QUANTIZATION || !Number.isInteger(manifest.dimension) || !Array.isArray(manifest.items)) return null
|
|
2757
|
-
// 编码实现换了 → 库里这份索引不是「同一个东西」,当作不存在(于是 recall 会如实降级,
|
|
2758
|
-
// 而不是拿一份用别的编码器算出来的向量去排序)。
|
|
2759
|
-
if (manifest.encoder !== ENCODER_FINGERPRINT) return null
|
|
2760
|
-
const binary = loaded.binary
|
|
2761
|
-
const int8 = loaded.int8
|
|
2762
|
-
const binaryWidth = Math.ceil(manifest.dimension / 8)
|
|
2763
|
-
if (manifest.itemCount !== manifest.items.length || binary.length !== manifest.itemCount * binaryWidth || int8.length !== manifest.itemCount * manifest.dimension) return null
|
|
2764
|
-
this.vectorIndex = { manifest, binary, int8 }
|
|
2730
|
+
this.vectorIndex = decodeVectorIndex((await this.derivedHandle())?.vectorIndex(), ENCODER_FINGERPRINT)
|
|
2765
2731
|
return this.vectorIndex
|
|
2766
|
-
} catch {
|
|
2767
|
-
return null
|
|
2768
|
-
}
|
|
2732
|
+
} catch { return null }
|
|
2769
2733
|
}
|
|
2770
2734
|
|
|
2771
2735
|
private vectorWorkerPath(): string {
|
|
@@ -2804,6 +2768,13 @@ export class KnowledgeRuntime {
|
|
|
2804
2768
|
|
|
2805
2769
|
private async encodeEmbeddings(inputs: EmbeddingInput[]): Promise<{ vectors: number[][]; device: string }> {
|
|
2806
2770
|
if (this.config.embeddingProvider === 'none') throw new Error('EMBEDDING_DISABLED')
|
|
2771
|
+
const fingerprint = this.embeddingFingerprint()
|
|
2772
|
+
if (fingerprint !== this.embeddingWorkerFingerprint) {
|
|
2773
|
+
this.embeddingWorker?.close()
|
|
2774
|
+
this.embeddingWorker = null
|
|
2775
|
+
this.embeddingWorkerFingerprint = fingerprint
|
|
2776
|
+
this.encoderHealth = undefined
|
|
2777
|
+
}
|
|
2807
2778
|
// Keep each JSON response bounded for large stores. The WeMM process is
|
|
2808
2779
|
// persistent, so chunking does not reload the 1.8GB model between chunks.
|
|
2809
2780
|
const chunkSize = Math.max(1, this.config.embeddingBatchSize * 64)
|
|
@@ -2832,10 +2803,22 @@ export class KnowledgeRuntime {
|
|
|
2832
2803
|
}
|
|
2833
2804
|
|
|
2834
2805
|
close(): void {
|
|
2806
|
+
for (const task of this.compileTasks.values()) {
|
|
2807
|
+
if (['succeeded', 'failed', 'cancelled'].includes(task.progress.state)) continue
|
|
2808
|
+
task.cancelRequested = true
|
|
2809
|
+
task.pauseRequested = false
|
|
2810
|
+
task.resume?.()
|
|
2811
|
+
}
|
|
2835
2812
|
this.embeddingWorker?.close()
|
|
2836
2813
|
this.embeddingWorker = null
|
|
2837
2814
|
this.rerankWorker?.close()
|
|
2838
2815
|
this.rerankWorker = null
|
|
2816
|
+
this.oplog?.close()
|
|
2817
|
+
this.extractionRunsStore?.close()
|
|
2818
|
+
this.axesStore?.close()
|
|
2819
|
+
this.knowledgeDb?.close()
|
|
2820
|
+
this.derived?.close()
|
|
2821
|
+
this.store?.close()
|
|
2839
2822
|
}
|
|
2840
2823
|
|
|
2841
2824
|
/** Both the encoder and the reranker speak the same JSONL worker protocol. */
|
|
@@ -2880,13 +2863,15 @@ export class KnowledgeRuntime {
|
|
|
2880
2863
|
// 「这份台账派生出了哪些结论」就回到关键词路回答 —— 那本来就是词面问题,不是图问题。
|
|
2881
2864
|
const titleOf = (sourceId: string): string | undefined => byId.get(sourceId)
|
|
2882
2865
|
try {
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
})
|
|
2889
|
-
)
|
|
2866
|
+
const db = await this.derivedHandle()
|
|
2867
|
+
if (!db) throw new Error('DERIVED_STORE_UNAVAILABLE')
|
|
2868
|
+
const documents = units.map(unit => ({
|
|
2869
|
+
unitId: unit.unitId,
|
|
2870
|
+
title: unit.title,
|
|
2871
|
+
content: noteIndexText({ title: unit.title, content: unit.content, sourceRefs: unit.sourceRefs }, titleOf),
|
|
2872
|
+
}))
|
|
2873
|
+
if (this.publication) this.publication.fts = documents
|
|
2874
|
+
else db.rebuildFts(documents)
|
|
2890
2875
|
this.ftsError = undefined
|
|
2891
2876
|
return 'ready'
|
|
2892
2877
|
} catch (cause) {
|
|
@@ -2896,13 +2881,14 @@ export class KnowledgeRuntime {
|
|
|
2896
2881
|
}
|
|
2897
2882
|
|
|
2898
2883
|
private async buildVectorIndex(units: KnowledgeUnit[], version: number): Promise<EmbeddingBuildResult> {
|
|
2884
|
+
if (this.publication) this.publication.vectors = null
|
|
2899
2885
|
if (this.config.embeddingProvider === 'none') return { status: 'skipped', count: 0, reused: 0 }
|
|
2900
2886
|
if (this.config.embeddingDimension % 8 !== 0) return { status: 'unavailable', count: 0, reused: 0, error: 'embeddingDimension 必须是 8 的倍数,才能进行 binary quantization' }
|
|
2901
2887
|
const inputs = await this.vectorInputs(units)
|
|
2902
2888
|
if (inputs.length === 0) return { status: 'ready', count: 0, reused: 0, dimension: this.config.embeddingDimension, device: 'none' }
|
|
2903
2889
|
|
|
2904
2890
|
const previous = await this.loadVectorIndex()
|
|
2905
|
-
const compatiblePrevious = previous && previous.manifest.dimension === this.config.embeddingDimension && previous.manifest.provider === this.config.embeddingProvider && previous.manifest.model === this.embeddingModelId && previous.manifest.encoder === ENCODER_FINGERPRINT ? previous : null
|
|
2891
|
+
const compatiblePrevious = previous && previous.manifest.dimension === this.config.embeddingDimension && previous.manifest.provider === this.config.embeddingProvider && previous.manifest.model === this.embeddingModelId && previous.manifest.modelFingerprint === this.embeddingFingerprint() && previous.manifest.encoder === ENCODER_FINGERPRINT ? previous : null
|
|
2906
2892
|
const previousById = new Map((compatiblePrevious?.manifest.items ?? []).map((item, index) => [item.unitId, { item, index }]))
|
|
2907
2893
|
const binaryWidth = Math.ceil(this.config.embeddingDimension / 8)
|
|
2908
2894
|
const binary = Buffer.alloc(inputs.length * binaryWidth)
|
|
@@ -2951,13 +2937,18 @@ export class KnowledgeRuntime {
|
|
|
2951
2937
|
this.encoderHealth = { ok: false, reason: `编码器退化:${inputs.length} 条 note 只产出 ${distinct.size} 个不同向量` }
|
|
2952
2938
|
throw new Error(`EMBEDDING_INDEX_DEGENERATE: ${inputs.length} 条只产出 ${distinct.size} 个不同向量`)
|
|
2953
2939
|
}
|
|
2954
|
-
const manifest: VectorManifest = { version, createdAt: new Date().toISOString(), provider: this.config.embeddingProvider, model: this.embeddingModelId, encoder: ENCODER_FINGERPRINT, device, dimension: this.config.embeddingDimension, quantization: VECTOR_QUANTIZATION, itemCount: inputs.length, items: inputs.map(item => ({ unitId: item.unit.unitId, contentHash: item.contentHash })) }
|
|
2940
|
+
const manifest: VectorManifest = { version, createdAt: new Date().toISOString(), provider: this.config.embeddingProvider, model: this.embeddingModelId, modelFingerprint: this.embeddingFingerprint(), encoder: ENCODER_FINGERPRINT, device, dimension: this.config.embeddingDimension, quantization: VECTOR_QUANTIZATION, itemCount: inputs.length, items: inputs.map(item => ({ unitId: item.unit.unitId, contentHash: item.contentHash })) }
|
|
2955
2941
|
/*
|
|
2956
2942
|
* 向量、它的量化缓冲与清单**一起**写进派生库:三个东西必须同一次落地,
|
|
2957
2943
|
* 分三次写文件时崩在中间会留下「清单说有 40 条、缓冲只有 30 条」的半成品。
|
|
2958
2944
|
*/
|
|
2959
|
-
|
|
2960
|
-
|
|
2945
|
+
const db = await this.derivedHandle()
|
|
2946
|
+
if (!db) throw new Error('DERIVED_STORE_UNAVAILABLE')
|
|
2947
|
+
if (this.publication) this.publication.vectors = { manifest, binary, int8 }
|
|
2948
|
+
else {
|
|
2949
|
+
db.putVectorIndex(manifest, binary, int8)
|
|
2950
|
+
this.vectorIndex = { manifest, binary, int8 }
|
|
2951
|
+
}
|
|
2961
2952
|
this.vectorError = undefined
|
|
2962
2953
|
return { status: 'ready', count: inputs.length, reused, dimension: this.config.embeddingDimension, device }
|
|
2963
2954
|
} catch (cause) {
|
|
@@ -2973,7 +2964,7 @@ export class KnowledgeRuntime {
|
|
|
2973
2964
|
for (const artifact of [WIKI_NOTES, LEGACY_UNITS]) {
|
|
2974
2965
|
try {
|
|
2975
2966
|
const notes = JSON.parse(await this.readArtifact(artifact))
|
|
2976
|
-
if (Array.isArray(notes)
|
|
2967
|
+
if (Array.isArray(notes)) { this.units = notes as KnowledgeUnit[]; break }
|
|
2977
2968
|
} catch { /* the first build has not happened yet */ }
|
|
2978
2969
|
}
|
|
2979
2970
|
try {
|
|
@@ -3213,7 +3204,10 @@ export class KnowledgeRuntime {
|
|
|
3213
3204
|
*/
|
|
3214
3205
|
let compile: ImportCompileReport | null = null
|
|
3215
3206
|
if (input.compile !== false) {
|
|
3216
|
-
|
|
3207
|
+
let rebuilt: Envelope<RebuildReport> | undefined
|
|
3208
|
+
try { rebuilt = await this.rebuild('wiki', input.lineage) }
|
|
3209
|
+
catch (cause) { warnings.push(`来源已写入,编译未完成:${cause instanceof Error ? cause.message : String(cause)}`) }
|
|
3210
|
+
if (rebuilt === undefined) return success({ written, failed, attempted: items.length, compile }, warnings)
|
|
3217
3211
|
if (rebuilt.ok === false) {
|
|
3218
3212
|
warnings.push(`来源已写入,但编译失败:${rebuilt.error?.message ?? '未知原因'}`)
|
|
3219
3213
|
} else {
|
|
@@ -3514,7 +3508,14 @@ export class KnowledgeRuntime {
|
|
|
3514
3508
|
*
|
|
3515
3509
|
* 失败一律返回 `null`,**不抛** —— 「知识库不可用」是一条要显示的结论,不是一个异常。
|
|
3516
3510
|
*/
|
|
3511
|
+
private storeHandleOpening?: Promise<KnowledgeDb | null>
|
|
3517
3512
|
private async storeHandle(): Promise<KnowledgeDb | null> {
|
|
3513
|
+
if (this.store !== undefined) return this.store
|
|
3514
|
+
this.storeHandleOpening ??= this.openStoreHandle()
|
|
3515
|
+
return this.storeHandleOpening
|
|
3516
|
+
}
|
|
3517
|
+
|
|
3518
|
+
private async openStoreHandle(): Promise<KnowledgeDb | null> {
|
|
3518
3519
|
if (this.store !== undefined) return this.store
|
|
3519
3520
|
const sqlite = await loadSqlite()
|
|
3520
3521
|
if (sqlite === null) {
|
|
@@ -3674,7 +3675,14 @@ export class KnowledgeRuntime {
|
|
|
3674
3675
|
}
|
|
3675
3676
|
|
|
3676
3677
|
/** 派生库(`<库根>/derived.db`):编译产物,**可整体删掉重算**。 */
|
|
3678
|
+
private derivedHandleOpening?: Promise<DerivedDb | null>
|
|
3677
3679
|
private async derivedHandle(): Promise<DerivedDb | null> {
|
|
3680
|
+
if (this.derived !== undefined) return this.derived
|
|
3681
|
+
this.derivedHandleOpening ??= this.openDerivedHandle()
|
|
3682
|
+
return this.derivedHandleOpening
|
|
3683
|
+
}
|
|
3684
|
+
|
|
3685
|
+
private async openDerivedHandle(): Promise<DerivedDb | null> {
|
|
3678
3686
|
if (this.derived !== undefined) return this.derived
|
|
3679
3687
|
const sqlite = await loadSqlite()
|
|
3680
3688
|
if (sqlite === null) { this.derived = null; return null }
|
|
@@ -3701,7 +3709,8 @@ export class KnowledgeRuntime {
|
|
|
3701
3709
|
private async writeArtifact(name: string, body: string): Promise<void> {
|
|
3702
3710
|
const db = await this.derivedHandle()
|
|
3703
3711
|
if (db === null) throw new Error('派生库不可用')
|
|
3704
|
-
|
|
3712
|
+
if (this.publication) this.publication.artifacts.set(name, body)
|
|
3713
|
+
else db.putArtifact(name, body)
|
|
3705
3714
|
}
|
|
3706
3715
|
|
|
3707
3716
|
private async hasArtifact(name: string): Promise<boolean> {
|
|
@@ -4095,7 +4104,29 @@ export class KnowledgeRuntime {
|
|
|
4095
4104
|
}
|
|
4096
4105
|
for (const card of (await this.loadCards()).filter(item => item.sourceRefs.includes(docId))) {
|
|
4097
4106
|
try {
|
|
4098
|
-
|
|
4107
|
+
const placements = db.placementsOf([card.entityId]).get(card.entityId) ?? []
|
|
4108
|
+
const remaining: Array<{ row: CardKnowledgeRow; knowledge: Knowledge }> = []
|
|
4109
|
+
for (const row of placements) {
|
|
4110
|
+
const item = await store?.get(row.knowledgeId)
|
|
4111
|
+
if (item) remaining.push({ row, knowledge: item })
|
|
4112
|
+
}
|
|
4113
|
+
const sections = new Map<string, string[]>()
|
|
4114
|
+
for (const { row, knowledge: item } of remaining.sort((a, b) => a.row.ordinal - b.row.ordinal)) {
|
|
4115
|
+
const heading = row.section || '知识'
|
|
4116
|
+
const lines = sections.get(heading) ?? []
|
|
4117
|
+
lines.push(`${item.text}[^${item.id}]`)
|
|
4118
|
+
sections.set(heading, lines)
|
|
4119
|
+
}
|
|
4120
|
+
const content = sections.size > 0
|
|
4121
|
+
? [...sections].map(([heading, lines]) => `## ${heading}\n\n${lines.join('\n\n')}`).join('\n\n')
|
|
4122
|
+
: card.content
|
|
4123
|
+
const updated = await this.writeCard({
|
|
4124
|
+
...card, content,
|
|
4125
|
+
...(sections.size > 0 ? { summary: remaining[0].knowledge.text, placements: remaining.map(item => item.row) } : {}),
|
|
4126
|
+
metadata: { ...card.metadata, layout: undefined, citations: undefined, evidence_needs_review: sections.size === 0 },
|
|
4127
|
+
sourceRefs: card.sourceRefs.filter(ref => ref !== docId), origin: 'internal',
|
|
4128
|
+
})
|
|
4129
|
+
if (!updated.ok) throw new Error(updated.error.message)
|
|
4099
4130
|
cardsDetached += 1
|
|
4100
4131
|
} catch (cause) { cardFailures += 1; if (firstCardFailure === '') firstCardFailure = cause instanceof Error ? cause.message : String(cause) }
|
|
4101
4132
|
}
|
|
@@ -4353,7 +4384,7 @@ export class KnowledgeRuntime {
|
|
|
4353
4384
|
}))
|
|
4354
4385
|
const hash = layoutHash({
|
|
4355
4386
|
knowledgeIds: knowledge.map(item => item.id),
|
|
4356
|
-
knowledgeContext: knowledge.map(item =>
|
|
4387
|
+
knowledgeContext: listed.knowledge.map(item => JSON.stringify({ id: item.id, text: item.text, version: item.version, headings: item.sources.map(source => source.headingPath), coordinate: input.coordinate })),
|
|
4357
4388
|
profileId: input.profileId,
|
|
4358
4389
|
})
|
|
4359
4390
|
const layout: CardLayoutRecord = { profile: input.profileId, hash, origin: 'model' }
|
|
@@ -4456,7 +4487,7 @@ export class KnowledgeRuntime {
|
|
|
4456
4487
|
raw: raw.trim().replace(/\s+/g, ' ').slice(0, 400),
|
|
4457
4488
|
})
|
|
4458
4489
|
}
|
|
4459
|
-
const checked = checkOrganize(plan, context.knowledge.map(item => item.id), { ...(await this.relatedForCard(card)) })
|
|
4490
|
+
const checked = checkOrganize(plan, context.knowledge.map(item => item.id), { knowledge: context.knowledge, ...(await this.relatedForCard(card)) })
|
|
4460
4491
|
const problems = [...parsed.problems.map(problem => problem.message), ...checked.problems.map(problem => problem.message)]
|
|
4461
4492
|
const placements = problems.length === 0 ? placementsOf(plan, checked.cited) : []
|
|
4462
4493
|
const images = await this.imagesForCard(card)
|
|
@@ -4667,6 +4698,7 @@ export class KnowledgeRuntime {
|
|
|
4667
4698
|
const problems = [...parsed.problems]
|
|
4668
4699
|
if (plan !== undefined) {
|
|
4669
4700
|
const checked = checkOrganize(plan, knowledge.map(item => item.id), {
|
|
4701
|
+
knowledge,
|
|
4670
4702
|
...(input.related === undefined ? {} : { related: input.related }),
|
|
4671
4703
|
})
|
|
4672
4704
|
problems.push(...checked.problems)
|
|
@@ -5036,7 +5068,7 @@ export class KnowledgeRuntime {
|
|
|
5036
5068
|
}
|
|
5037
5069
|
|
|
5038
5070
|
/** 1-hop neighbourhood of a set of wiki cards inside the compiled graph. */
|
|
5039
|
-
private oneHopGraph(keys: Set<string>, includeProblem: boolean): ContextGraph {
|
|
5071
|
+
private oneHopGraph(keys: Set<string>, includeProblem: boolean, graph: ContextGraph = this.graph): ContextGraph {
|
|
5040
5072
|
const visible = (node: Record<string, unknown>) => includeProblem || node.status !== 'problem'
|
|
5041
5073
|
const edges = this.graph.edges.filter(edge => keys.has(String(edge.from)) || keys.has(String(edge.to))).slice(0, 50)
|
|
5042
5074
|
const ids = new Set(keys)
|
|
@@ -5136,7 +5168,7 @@ export class KnowledgeRuntime {
|
|
|
5136
5168
|
for (const artifact of [WIKI_NOTES, LEGACY_UNITS]) {
|
|
5137
5169
|
try {
|
|
5138
5170
|
const notes = JSON.parse(await this.readArtifact(artifact))
|
|
5139
|
-
if (Array.isArray(notes)
|
|
5171
|
+
if (Array.isArray(notes)) return notes as WikiNote[]
|
|
5140
5172
|
} catch { /* the first build has not happened yet */ }
|
|
5141
5173
|
}
|
|
5142
5174
|
return []
|
|
@@ -5160,7 +5192,11 @@ export class KnowledgeRuntime {
|
|
|
5160
5192
|
return contentHashOf(JSON.stringify({
|
|
5161
5193
|
sourceId: source.sourceId,
|
|
5162
5194
|
content: contentHashOf(source.content),
|
|
5163
|
-
|
|
5195
|
+
title: source.title,
|
|
5196
|
+
compiler: 'knowledge-extract-v2',
|
|
5197
|
+
extractWithModel: this.config.extractWithModel,
|
|
5198
|
+
extractionNote: this.extractionNote(),
|
|
5199
|
+
layoutProfile: this.config.layoutProfileId,
|
|
5164
5200
|
declared: contentHashOf(JSON.stringify(declared)),
|
|
5165
5201
|
registry: registryHash,
|
|
5166
5202
|
}))
|
|
@@ -5203,7 +5239,7 @@ export class KnowledgeRuntime {
|
|
|
5203
5239
|
): Promise<ExtractionOutcome> {
|
|
5204
5240
|
const schema = this.wikiSchema()
|
|
5205
5241
|
const enabledTypes = schema.types.filter(def => def.enabled)
|
|
5206
|
-
const
|
|
5242
|
+
const registry = await this.axisRegistry()
|
|
5207
5243
|
const outcome = this.emptyExtractionOutcome({})
|
|
5208
5244
|
/**
|
|
5209
5245
|
* 模型调用**每轮解析一次**,而且解析不到时**如实报出来**(0044)。
|
|
@@ -5214,6 +5250,7 @@ export class KnowledgeRuntime {
|
|
|
5214
5250
|
*/
|
|
5215
5251
|
const seam = this.config.extractWithModel ? this.modelSeamNow() : {}
|
|
5216
5252
|
const call = seam.call
|
|
5253
|
+
const registryHash = contentHashOf(JSON.stringify({ types: enabledTypes, axesRevision: registry?.inputRevision() ?? 0, model: { available: Boolean(call), provider: seam.provider, model: seam.model, reasoningEffort: seam.reasoningEffort } }))
|
|
5217
5254
|
if (this.config.extractWithModel && !call) {
|
|
5218
5255
|
outcome.warnings.push(
|
|
5219
5256
|
`模型抽取已开启,但取不到可用的模型调用:${seam.why ?? '原因未知'} —— 本次退回规则抽取:`
|
|
@@ -5241,9 +5278,9 @@ export class KnowledgeRuntime {
|
|
|
5241
5278
|
for (const source of ordered) {
|
|
5242
5279
|
if (runId) await this.checkpointCompileTask(runId)
|
|
5243
5280
|
const fingerprint = this.extractionFingerprint(source, registryHash)
|
|
5244
|
-
outcome.extractionHashes[source.sourceId] = fingerprint
|
|
5245
5281
|
if (previousHashes[source.sourceId] === fingerprint) {
|
|
5246
5282
|
outcome.sourcesSkipped += 1
|
|
5283
|
+
outcome.extractionHashes[source.sourceId] = fingerprint
|
|
5247
5284
|
const kept = persisted[source.sourceId] ?? []
|
|
5248
5285
|
if (kept.length > 0) {
|
|
5249
5286
|
outcome.relationsBySource[source.sourceId] = kept
|
|
@@ -5257,7 +5294,8 @@ export class KnowledgeRuntime {
|
|
|
5257
5294
|
try {
|
|
5258
5295
|
// 同一轮里后一份来源应当能看到前一份来源刚抽出来的实体,所以判重索引
|
|
5259
5296
|
// 每份来源都用「磁盘上的卡片 + 这一轮 pending 的卡片」重建。
|
|
5260
|
-
await this.compileSource(source, byKey, pending, enabledTypes, outcome, call, knowledgeIds, runId, seam)
|
|
5297
|
+
const complete = await this.compileSource(source, byKey, pending, enabledTypes, outcome, call, knowledgeIds, runId, seam)
|
|
5298
|
+
if (complete !== false) outcome.extractionHashes[source.sourceId] = fingerprint
|
|
5261
5299
|
} catch (cause) {
|
|
5262
5300
|
if (cause instanceof CompileTaskCancelledError) throw cause
|
|
5263
5301
|
outcome.warnings.push(`来源 ${source.sourceId} 的候选抽取失败,已跳过该来源:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
@@ -5284,7 +5322,7 @@ export class KnowledgeRuntime {
|
|
|
5284
5322
|
* 的卡片不算「变了」,所以重复编译不会反复写盘。
|
|
5285
5323
|
*/
|
|
5286
5324
|
const folded = new Map<string, string>()
|
|
5287
|
-
for (const [key, item] of pending) folded.set(key,
|
|
5325
|
+
for (const [key, item] of pending) folded.set(key, item.content)
|
|
5288
5326
|
const view: Array<{ key: string; content: string; sourceRefs: string[] }> = []
|
|
5289
5327
|
for (const [key, card] of byKey) view.push({ key, content: card.content, sourceRefs: card.sourceRefs })
|
|
5290
5328
|
for (const [key, item] of pending) view.push({ key, content: folded.get(key) ?? '', sourceRefs: item.sourceRefs })
|
|
@@ -5358,9 +5396,14 @@ export class KnowledgeRuntime {
|
|
|
5358
5396
|
const contentBeforeEvents = preserveModelLayout ? existing.content : content
|
|
5359
5397
|
const finalContent = withEventReferences(contentBeforeEvents, item.metadata.event_refs, eventsById)
|
|
5360
5398
|
const finalSummary = preserveModelLayout ? existing.summary : item.summary
|
|
5361
|
-
|
|
5399
|
+
let finalPlacements = preserveModelLayout || item.placements === undefined
|
|
5362
5400
|
? undefined
|
|
5363
5401
|
: translatePlacements(item.placements, knowledgeIds)
|
|
5402
|
+
if (finalPlacements !== undefined && existing) {
|
|
5403
|
+
const db = await this.storeHandle()
|
|
5404
|
+
const previous = db?.placementsOf([existing.entityId]).get(existing.entityId) ?? []
|
|
5405
|
+
finalPlacements = [...new Map([...previous, ...finalPlacements].map(row => [row.knowledgeId, row])).values()]
|
|
5406
|
+
}
|
|
5364
5407
|
const titledMetadata = withTitleAlias(metadata, existing?.title, finalTitle)
|
|
5365
5408
|
const finalMetadata = titledMetadata
|
|
5366
5409
|
|
|
@@ -5388,6 +5431,7 @@ export class KnowledgeRuntime {
|
|
|
5388
5431
|
origin: 'compile',
|
|
5389
5432
|
})
|
|
5390
5433
|
if (written.ok === false) {
|
|
5434
|
+
for (const ref of item.sourceRefs) delete outcome.extractionHashes[ref]
|
|
5391
5435
|
outcome.warnings.push(`卡片 ${item.entityId} 写入失败(${written.error.code}):${written.error.message}`)
|
|
5392
5436
|
continue
|
|
5393
5437
|
}
|
|
@@ -5650,7 +5694,8 @@ export class KnowledgeRuntime {
|
|
|
5650
5694
|
knowledgeIds: Record<string, string>,
|
|
5651
5695
|
compileRunId?: string,
|
|
5652
5696
|
modelInfo?: ModelSeam,
|
|
5653
|
-
): Promise<
|
|
5697
|
+
): Promise<boolean> {
|
|
5698
|
+
let extractionComplete = call === undefined
|
|
5654
5699
|
// 判重索引要看到「这一轮前面几份来源刚抽出来的卡片」,否则同一实体跨两份文档
|
|
5655
5700
|
// 会各建一张 —— 而那正是这次改造要修掉的那个 bug。
|
|
5656
5701
|
const cards = [...byKey.values()]
|
|
@@ -5683,6 +5728,9 @@ export class KnowledgeRuntime {
|
|
|
5683
5728
|
/** 撞名报了没有 —— 同一个名字被三个坐标撞到时报三次是噪声。 */
|
|
5684
5729
|
const collisionReported = new Set<string>()
|
|
5685
5730
|
|
|
5731
|
+
const sourceDb = await this.storeHandle()
|
|
5732
|
+
const sourceHash = createHash('sha256').update(source.content, 'utf8').digest('hex')
|
|
5733
|
+
const sourceVersion = sourceDb?.versions('source', source.sourceId).find(row => row.blobHash === sourceHash)?.version
|
|
5686
5734
|
const ruleCandidates = extractCandidates(source, enabledTypes)
|
|
5687
5735
|
outcome.candidatesExtracted += ruleCandidates.length
|
|
5688
5736
|
const ruleNormalized = normalizeCandidates(ruleCandidates, buildAliasIndex(cards))
|
|
@@ -5786,9 +5834,10 @@ export class KnowledgeRuntime {
|
|
|
5786
5834
|
})
|
|
5787
5835
|
|
|
5788
5836
|
/**
|
|
5789
|
-
*
|
|
5790
|
-
*
|
|
5837
|
+
* 记录抽取轨迹,再写入事实真源。轨迹是审计;事实写入不完整时不能保存成功指纹,
|
|
5838
|
+
* 下一次增量编译必须能够重试。
|
|
5791
5839
|
*/
|
|
5840
|
+
extractionComplete = report.model.ok
|
|
5792
5841
|
const summary = trialSummary(report)
|
|
5793
5842
|
await this.recordExtractionRun({
|
|
5794
5843
|
id: runId, docId: source.sourceId, title: source.title, summary,
|
|
@@ -5797,6 +5846,7 @@ export class KnowledgeRuntime {
|
|
|
5797
5846
|
if (compileRunId) await this.checkpointCompileTask(compileRunId)
|
|
5798
5847
|
const ingested = await this.ingestTrialKnowledge({
|
|
5799
5848
|
runId, knowledge: report.knowledge,
|
|
5849
|
+
sourceVersions: { [source.sourceId]: sourceVersion },
|
|
5800
5850
|
objects: report.objects, actions: report.actions, scenes: report.scenes,
|
|
5801
5851
|
})
|
|
5802
5852
|
let eventIngested: Awaited<ReturnType<KnowledgeRuntime['ingestTrialEvents']>> = {
|
|
@@ -5815,6 +5865,7 @@ export class KnowledgeRuntime {
|
|
|
5815
5865
|
outcome.warnings.push(`Event 没有写入知识库:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
5816
5866
|
}
|
|
5817
5867
|
outcome.warnings.push(...eventIngested.warnings)
|
|
5868
|
+
if (ingested.skipped > 0 || eventIngested.skipped > 0) extractionComplete = false
|
|
5818
5869
|
// 落位要用库内的 id。这一份报告里每一条知识的映射都并进整轮那一份 ——
|
|
5819
5870
|
// 后面的来源可能命中同一张卡,那时落位指的行也得是活的。
|
|
5820
5871
|
Object.assign(knowledgeIds, ingested.ids)
|
|
@@ -5832,7 +5883,7 @@ export class KnowledgeRuntime {
|
|
|
5832
5883
|
outcome.axesAmbiguous += axes.filter(axis => axis.outcome === 'ambiguous').length
|
|
5833
5884
|
outcome.knowledgeMerged += ingested.merged
|
|
5834
5885
|
if (ingested.skipped > 0) {
|
|
5835
|
-
outcome.warnings.push(`${ingested.skipped}
|
|
5886
|
+
outcome.warnings.push(`${ingested.skipped} 条知识未进入真源(出处核验或写入失败),已保留抽取轨迹,下一次编译会重试`)
|
|
5836
5887
|
}
|
|
5837
5888
|
if (eventIngested.skipped > 0) {
|
|
5838
5889
|
outcome.warnings.push(`${eventIngested.skipped} 个 Event 没有可核验出处,未进入 Event 真源`)
|
|
@@ -6063,6 +6114,7 @@ export class KnowledgeRuntime {
|
|
|
6063
6114
|
...(outcome.relationsBySource[source.sourceId] ?? []),
|
|
6064
6115
|
...ruleNormalized.relations,
|
|
6065
6116
|
]
|
|
6117
|
+
return extractionComplete
|
|
6066
6118
|
}
|
|
6067
6119
|
|
|
6068
6120
|
/**
|
|
@@ -6204,7 +6256,7 @@ export class KnowledgeRuntime {
|
|
|
6204
6256
|
* 调用点必须在**按卡片去重之后**:先扩展再去重会把邻居与命中混在一起,
|
|
6205
6257
|
* 而上下文里必须分得清「这是召回来的」还是「这是延伸出来的」。
|
|
6206
6258
|
*/
|
|
6207
|
-
private expandWiki(seedKeys: string[], cardByKey: Map<string, KnowledgeCard>, includeProblems: boolean): RecallExpansion {
|
|
6259
|
+
private expandWiki(seedKeys: string[], cardByKey: Map<string, KnowledgeCard>, includeProblems: boolean, graph: ContextGraph = this.graph): RecallExpansion {
|
|
6208
6260
|
const seeds = new Set(seedKeys)
|
|
6209
6261
|
const visible = (card: KnowledgeCard): boolean => includeProblems || card.status !== 'problem'
|
|
6210
6262
|
const via = new Map<string, string[]>()
|
|
@@ -6214,7 +6266,7 @@ export class KnowledgeRuntime {
|
|
|
6214
6266
|
via.set(key, bucket)
|
|
6215
6267
|
}
|
|
6216
6268
|
const edges: Array<Record<string, unknown>> = []
|
|
6217
|
-
for (const edge of
|
|
6269
|
+
for (const edge of graph.edges) {
|
|
6218
6270
|
const type = String(edge.type)
|
|
6219
6271
|
if (type === 'supported_by') continue
|
|
6220
6272
|
const from = String(edge.from)
|
|
@@ -6733,14 +6785,22 @@ export class KnowledgeRuntime {
|
|
|
6733
6785
|
* 与抽取轨迹**同一个数据库文件**(`<memoryDir>/knowledge.sqlite`):它们是同一层
|
|
6734
6786
|
* (领域实体与它的审计),而解析一条知识要同时读两边 —— 分两个文件只是多一次打开。
|
|
6735
6787
|
*/
|
|
6788
|
+
private axisRegistryOpening?: Promise<AxisRegistry | null>
|
|
6736
6789
|
async axisRegistry(): Promise<AxisRegistry | null> {
|
|
6790
|
+
this.axisRegistryOpening ??= this.openAxisRegistry()
|
|
6791
|
+
return this.axisRegistryOpening
|
|
6792
|
+
}
|
|
6793
|
+
|
|
6794
|
+
private async openAxisRegistry(): Promise<AxisRegistry | null> {
|
|
6737
6795
|
if (this.axesReady !== null) return this.axesReady ? this.axesStore : null
|
|
6738
6796
|
/*
|
|
6739
6797
|
* 0062 v2:轴注册表与 Knowledge 搬进**知识库文件**(`<库根>/knowledge.db`),不再是
|
|
6740
6798
|
* 记忆段里那个 `knowledge.sqlite`。理由是同一条:它们是**知识**(不可重建的命名决定
|
|
6741
6799
|
* 与人的裁决),必须跟着知识库一起备份、迁移、删除;而记忆段只装交互记录。
|
|
6742
6800
|
*/
|
|
6743
|
-
const
|
|
6801
|
+
const db = await this.storeHandle()
|
|
6802
|
+
if (db === null) return null
|
|
6803
|
+
const registry = new AxisRegistry(this.safePath(KNOWLEDGE_DB_FILE), db.connection())
|
|
6744
6804
|
const ok = await registry.open()
|
|
6745
6805
|
this.axesStore = ok ? registry : null
|
|
6746
6806
|
this.axesReady = ok
|
|
@@ -7258,7 +7318,11 @@ export class KnowledgeRuntime {
|
|
|
7258
7318
|
/** 并两个实体。被并的那个不删,转成 `merged` 并指向目标。 */
|
|
7259
7319
|
async mergeAxes(fromId: string, intoId: string): Promise<boolean> {
|
|
7260
7320
|
const registry = await this.axisRegistry()
|
|
7261
|
-
|
|
7321
|
+
const db = await this.storeHandle()
|
|
7322
|
+
if (!registry || !db) return false
|
|
7323
|
+
const merged = await registry.merge(fromId, intoId, kind => redirectAxisReferences(db, kind, fromId, intoId))
|
|
7324
|
+
if (merged) this.cards = await this.loadCards()
|
|
7325
|
+
return merged
|
|
7262
7326
|
}
|
|
7263
7327
|
|
|
7264
7328
|
/** 加一个别名。`scopeId` 空串 = 全局。 */
|
|
@@ -7276,14 +7340,22 @@ export class KnowledgeRuntime {
|
|
|
7276
7340
|
/**
|
|
7277
7341
|
* Knowledge 库(0063 §2):事实真源。
|
|
7278
7342
|
*
|
|
7279
|
-
*
|
|
7343
|
+
* 与轴注册表共享经过校验的 `knowledge.db` 连接,使用各自的表。
|
|
7280
7344
|
* 两个模块互不认识,靠 id 字符串耦合 —— 代价是没有数据库级引用完整性,
|
|
7281
7345
|
* 收益是「一条知识挂了一个不存在的坐标」会被 §5.7 的坐标核验**报出来**,
|
|
7282
7346
|
* 而不是被数据库静默拒绝。
|
|
7283
7347
|
*/
|
|
7348
|
+
private knowledgeStoreOpening?: Promise<KnowledgeStore | null>
|
|
7284
7349
|
async knowledgeStore(): Promise<KnowledgeStore | null> {
|
|
7350
|
+
this.knowledgeStoreOpening ??= this.openKnowledgeStore()
|
|
7351
|
+
return this.knowledgeStoreOpening
|
|
7352
|
+
}
|
|
7353
|
+
|
|
7354
|
+
private async openKnowledgeStore(): Promise<KnowledgeStore | null> {
|
|
7285
7355
|
if (this.knowledgeReady !== null) return this.knowledgeReady ? this.knowledgeDb : null
|
|
7286
|
-
const
|
|
7356
|
+
const db = await this.storeHandle()
|
|
7357
|
+
if (db === null) return null
|
|
7358
|
+
const store = new KnowledgeStore(this.safePath(KNOWLEDGE_DB_FILE), db.connection())
|
|
7287
7359
|
const ok = await store.open()
|
|
7288
7360
|
this.knowledgeDb = ok ? store : null
|
|
7289
7361
|
this.knowledgeReady = ok
|
|
@@ -7306,13 +7378,12 @@ export class KnowledgeRuntime {
|
|
|
7306
7378
|
/*
|
|
7307
7379
|
* 给每一处出处补上**它依据的是哪一版**(0062 v2)。
|
|
7308
7380
|
*
|
|
7309
|
-
*
|
|
7310
|
-
*
|
|
7311
|
-
* 来源被人改过之后,没有任何东西能回答那条知识还算不算数。
|
|
7381
|
+
* 抽取链路带实际读取版本;直接写入没有指定版本时,补当前来源版本。
|
|
7382
|
+
* 版本用于追踪证据变化,不能据此裁定 Knowledge 是否失效。
|
|
7312
7383
|
*/
|
|
7313
7384
|
const db = await this.storeHandle()
|
|
7314
7385
|
const sources = (input.sources ?? []).map(source => {
|
|
7315
|
-
if (!source.documentId) return source
|
|
7386
|
+
if (!source.documentId || source.toVersion !== undefined) return source
|
|
7316
7387
|
const version = db?.source(source.documentId)?.version
|
|
7317
7388
|
return version === undefined ? source : { ...source, toVersion: version }
|
|
7318
7389
|
})
|
|
@@ -7333,7 +7404,11 @@ export class KnowledgeRuntime {
|
|
|
7333
7404
|
*
|
|
7334
7405
|
* 调用方可以显式传 `false` 关掉它(试跑那条链路不需要这个字段)。
|
|
7335
7406
|
*/
|
|
7336
|
-
try {
|
|
7407
|
+
try {
|
|
7408
|
+
const page = await store.list({ withCards: true, ...query })
|
|
7409
|
+
const db = await this.storeHandle()
|
|
7410
|
+
return db ? { ...page, knowledge: page.knowledge.map(row => assessKnowledgeEvidence(row, db)) } : page
|
|
7411
|
+
} catch { return null }
|
|
7337
7412
|
}
|
|
7338
7413
|
|
|
7339
7414
|
/** 读一条 Knowledge 的当前内容与版本时间线,供内容侧边栏使用。 */
|
|
@@ -7343,7 +7418,8 @@ export class KnowledgeRuntime {
|
|
|
7343
7418
|
try {
|
|
7344
7419
|
const knowledge = await store.get(id)
|
|
7345
7420
|
if (knowledge === null) return null
|
|
7346
|
-
|
|
7421
|
+
const db = await this.storeHandle()
|
|
7422
|
+
return { knowledge: db ? assessKnowledgeEvidence(knowledge, db) : knowledge, versions: await store.versions(id) }
|
|
7347
7423
|
} catch { return null }
|
|
7348
7424
|
}
|
|
7349
7425
|
|
|
@@ -7398,7 +7474,7 @@ export class KnowledgeRuntime {
|
|
|
7398
7474
|
const cardOrphans = cards.filter(card => card.sourceRefs.every(ref => !live.has(ref))).length
|
|
7399
7475
|
const events = (await this.loadEvents()).filter(event => event.records.some(record => !live.has(record.sourceId))).length
|
|
7400
7476
|
/**
|
|
7401
|
-
*
|
|
7477
|
+
* 未被引用的来源图片也进这张账(2026-10-05)。`images.publish` 写入的独立图片会被识别并保留。
|
|
7402
7478
|
*
|
|
7403
7479
|
* 少了它,「来源与卡片都清完了、只剩一堆没人引用的图」这种状态在界面上**一个字都没有**
|
|
7404
7480
|
* —— 清理孤儿的入口因此不出现,而用户唯一能看到的读数(「收录 N 个文件」)又把它算成
|
|
@@ -7406,7 +7482,7 @@ export class KnowledgeRuntime {
|
|
|
7406
7482
|
* 「按一下能清掉几个」是同几个。
|
|
7407
7483
|
*/
|
|
7408
7484
|
const names = await this.assetBucketNames()
|
|
7409
|
-
const assets = names.length === 0 ? 0 : this.unreferencedBuckets(names, sources, cards).length
|
|
7485
|
+
const assets = names.length === 0 ? 0 : (await this.unreferencedBuckets(names, sources, cards)).length
|
|
7410
7486
|
return { knowledge, cards: cardOrphans, events, assets }
|
|
7411
7487
|
}
|
|
7412
7488
|
|
|
@@ -7434,8 +7510,9 @@ export class KnowledgeRuntime {
|
|
|
7434
7510
|
}
|
|
7435
7511
|
|
|
7436
7512
|
/**
|
|
7437
|
-
* 还有没有人在引用这个桶 ——
|
|
7438
|
-
*
|
|
7513
|
+
* 还有没有人在引用这个桶 —— 判据是来源正文、来源 front matter 的 `metadata.assets`、
|
|
7514
|
+
* 卡片正文出现过 `_assets/<bucket>/`,或者它是 `images.publish` 写入的独立图床图片。
|
|
7515
|
+
* 独立图片没有来源/卡片引用是正常状态,必须保留。
|
|
7439
7516
|
*
|
|
7440
7517
|
* 为什么三处都要看:桶名是**内容哈希的前 12 位**(`convert/ingest.ts`),不是来源 id。
|
|
7441
7518
|
* 同一份内容以两个分类导入会共用同一个桶;而卡片正文会把图片引用**复制进去**
|
|
@@ -7443,7 +7520,7 @@ export class KnowledgeRuntime {
|
|
|
7443
7520
|
*
|
|
7444
7521
|
* 不看 `03_compiled/`:那是派生物,下一次 `rebuild` 会把指向已删图片的引用一起扔掉。
|
|
7445
7522
|
*/
|
|
7446
|
-
private unreferencedBuckets(names: string[], sources: SourceRaw[], cards: KnowledgeCard[]): string[] {
|
|
7523
|
+
private async unreferencedBuckets(names: string[], sources: SourceRaw[], cards: KnowledgeCard[]): Promise<string[]> {
|
|
7447
7524
|
const referenced = new Set<string>()
|
|
7448
7525
|
/**
|
|
7449
7526
|
* **两种引用形式都算**:`assets/<桶>/`(新写的)与 `_assets/<桶>/`(存量)。
|
|
@@ -7462,7 +7539,38 @@ export class KnowledgeRuntime {
|
|
|
7462
7539
|
if (Array.isArray(source.metadata.assets)) for (const item of source.metadata.assets) if (typeof item === 'string') collect(item)
|
|
7463
7540
|
}
|
|
7464
7541
|
for (const card of cards) collect(card.content)
|
|
7465
|
-
|
|
7542
|
+
const unreferenced: string[] = []
|
|
7543
|
+
for (const name of names) {
|
|
7544
|
+
if (referenced.has(name)) continue
|
|
7545
|
+
if (await this.isPublishedImageBucket(name)) continue
|
|
7546
|
+
unreferenced.push(name)
|
|
7547
|
+
}
|
|
7548
|
+
return unreferenced
|
|
7549
|
+
}
|
|
7550
|
+
|
|
7551
|
+
/**
|
|
7552
|
+
* `images.publish` 的图片不需要被来源或卡片引用。新写入的桶有明确标记;同时兼容
|
|
7553
|
+
* 标记加入前已发布的图片:它们以图片自身的 SHA-256 前 12 位作为桶名。
|
|
7554
|
+
*/
|
|
7555
|
+
private async isPublishedImageBucket(name: string): Promise<boolean> {
|
|
7556
|
+
for (const root of this.assetRoots()) {
|
|
7557
|
+
if (existsSync(join(root, name, PUBLISHED_IMAGE_MARKER))) return true
|
|
7558
|
+
}
|
|
7559
|
+
|
|
7560
|
+
// 旧版本 images.publish 只会写入知识库根目录的 assets/,因此只在这里做哈希识别。
|
|
7561
|
+
if (!/^[a-f0-9]{12}$/i.test(name)) return false
|
|
7562
|
+
const dir = join(this.rootDir, ASSETS_DIR, name)
|
|
7563
|
+
try {
|
|
7564
|
+
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
7565
|
+
if (!entry.isFile() || !/\.(?:png|jpe?g|gif|webp|avif|bmp|svg)$/i.test(entry.name)) continue
|
|
7566
|
+
const path = join(dir, entry.name)
|
|
7567
|
+
const details = await stat(path)
|
|
7568
|
+
if (details.size > 20 * 1024 * 1024) continue
|
|
7569
|
+
const bytes = await readFile(path)
|
|
7570
|
+
if (createHash('sha256').update(bytes).digest('hex').slice(0, 12) === name) return true
|
|
7571
|
+
}
|
|
7572
|
+
} catch { /* 桶目录可能在扫描期间被删除 */ }
|
|
7573
|
+
return false
|
|
7466
7574
|
}
|
|
7467
7575
|
|
|
7468
7576
|
/**
|
|
@@ -7532,7 +7640,7 @@ export class KnowledgeRuntime {
|
|
|
7532
7640
|
if (names.length === 0) return { buckets: 0, files: 0, problems: [] }
|
|
7533
7641
|
const problems: string[] = []
|
|
7534
7642
|
const cards = this.cards.length > 0 ? this.cards : await this.loadCards()
|
|
7535
|
-
const doomed = this.unreferencedBuckets(names, await this.loadSources(), cards)
|
|
7643
|
+
const doomed = await this.unreferencedBuckets(names, await this.loadSources(), cards)
|
|
7536
7644
|
let buckets = 0
|
|
7537
7645
|
let files = 0
|
|
7538
7646
|
for (const name of doomed) {
|
|
@@ -7670,6 +7778,7 @@ export class KnowledgeRuntime {
|
|
|
7670
7778
|
async ingestTrialKnowledge(input: {
|
|
7671
7779
|
runId: string
|
|
7672
7780
|
knowledge: TrialKnowledge[]
|
|
7781
|
+
sourceVersions?: Record<string, number | undefined>
|
|
7673
7782
|
objects: TrialAxis[]
|
|
7674
7783
|
actions: TrialAxis[]
|
|
7675
7784
|
scenes: TrialAxis[]
|
|
@@ -7689,7 +7798,7 @@ export class KnowledgeRuntime {
|
|
|
7689
7798
|
ids: Record<string, string>
|
|
7690
7799
|
}> {
|
|
7691
7800
|
const store = await this.knowledgeStore()
|
|
7692
|
-
if (store === null) return { saved: 0, merged: 0, skipped:
|
|
7801
|
+
if (store === null) return { saved: 0, merged: 0, skipped: input.knowledge.length, ids: {} }
|
|
7693
7802
|
const axes = { objects: input.objects, actions: input.actions, scenes: input.scenes }
|
|
7694
7803
|
let saved = 0
|
|
7695
7804
|
let merged = 0
|
|
@@ -7700,7 +7809,7 @@ export class KnowledgeRuntime {
|
|
|
7700
7809
|
if (usable.length === 0) { skipped += 1; continue }
|
|
7701
7810
|
try {
|
|
7702
7811
|
const coordinate = coordinateIdsOf(axes, item.object, item.action, item.scene)
|
|
7703
|
-
const result = await
|
|
7812
|
+
const result = await this.ingestKnowledge({
|
|
7704
7813
|
text: item.text,
|
|
7705
7814
|
objectId: coordinate.objectId,
|
|
7706
7815
|
actionId: coordinate.actionId,
|
|
@@ -7708,11 +7817,13 @@ export class KnowledgeRuntime {
|
|
|
7708
7817
|
runId: input.runId,
|
|
7709
7818
|
sources: usable.map(source => ({
|
|
7710
7819
|
documentId: source.documentId,
|
|
7820
|
+
...(input.sourceVersions?.[source.documentId] === undefined ? {} : { toVersion: input.sourceVersions[source.documentId] }),
|
|
7711
7821
|
quote: source.quote,
|
|
7712
7822
|
headingPath: source.headingPath,
|
|
7713
7823
|
...(source.locator ? { locator: source.locator } : {}),
|
|
7714
7824
|
})),
|
|
7715
7825
|
})
|
|
7826
|
+
if (result === null) { skipped += 1; continue }
|
|
7716
7827
|
if (result.merged) merged += 1
|
|
7717
7828
|
else saved += 1
|
|
7718
7829
|
/*
|
|
@@ -7901,13 +8012,7 @@ export class KnowledgeRuntime {
|
|
|
7901
8012
|
}
|
|
7902
8013
|
}
|
|
7903
8014
|
} else if (preciseTime(event.occurredAt) && hasCompleteCoordinate(coordinateIds) && event.symptom?.trim()) {
|
|
7904
|
-
identityKey =
|
|
7905
|
-
event.occurredAt,
|
|
7906
|
-
coordinateIds.objectId,
|
|
7907
|
-
coordinateIds.actionId,
|
|
7908
|
-
coordinateIds.sceneId,
|
|
7909
|
-
normalizeKnowledge(event.symptom),
|
|
7910
|
-
]))
|
|
8015
|
+
identityKey = compositeEventIdentity(event.occurredAt, coordinateIds, event.symptom)
|
|
7911
8016
|
identityType = 'composite'
|
|
7912
8017
|
} else {
|
|
7913
8018
|
identityKey = 'source:' + input.sourceId + ':' + quoteKeyOf(event, evidence)
|
|
@@ -8010,7 +8115,12 @@ export class KnowledgeRuntime {
|
|
|
8010
8115
|
*/
|
|
8011
8116
|
async rebuild(only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
8012
8117
|
const runId = newCompileRunId()
|
|
8013
|
-
|
|
8118
|
+
let result: Envelope<RebuildReport>
|
|
8119
|
+
try { result = await this.enqueueCompile(runId, only, lineage, options) }
|
|
8120
|
+
catch (cause) {
|
|
8121
|
+
await this.settleProgressFailure(runId, cause)
|
|
8122
|
+
throw cause
|
|
8123
|
+
}
|
|
8014
8124
|
/*
|
|
8015
8125
|
* 同步调用方拿的是返回值,但**别人可能在轮询同一条进度记录**(面板正开着编译对话框,
|
|
8016
8126
|
* 而 Agent 起了这一次)。结论两个出口都要有。
|
|
@@ -8108,7 +8218,7 @@ export class KnowledgeRuntime {
|
|
|
8108
8218
|
queued: false,
|
|
8109
8219
|
finishedAt: current.finishedAt ?? new Date().toISOString(),
|
|
8110
8220
|
failed: true,
|
|
8111
|
-
outcome:
|
|
8221
|
+
outcome: `编译失败:${message}`,
|
|
8112
8222
|
})
|
|
8113
8223
|
await this.patchCompileOperation(runId, 'failed', `编译失败:${message}`)
|
|
8114
8224
|
}
|
|
@@ -8162,6 +8272,49 @@ export class KnowledgeRuntime {
|
|
|
8162
8272
|
* 它」—— 那两件事分别在 `enqueueCompile()` 与 `rebuild()` / `startCompile()` 里。
|
|
8163
8273
|
*/
|
|
8164
8274
|
private async compileOnce(runId: string, only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
8275
|
+
await this.environmentGuard?.()
|
|
8276
|
+
this.publication = { artifacts: new Map() }
|
|
8277
|
+
const configAtStart = JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() })
|
|
8278
|
+
try {
|
|
8279
|
+
await this.loadCompiledState()
|
|
8280
|
+
const result = await this.prepareCompile(runId, only, lineage, options)
|
|
8281
|
+
await this.checkpointCompileTask(runId)
|
|
8282
|
+
if (result.ok) {
|
|
8283
|
+
const db = await this.derivedHandle()
|
|
8284
|
+
if (!db) throw new Error('DERIVED_STORE_UNAVAILABLE')
|
|
8285
|
+
const truth = await this.storeHandle()
|
|
8286
|
+
const knowledge = await this.knowledgeStore()
|
|
8287
|
+
const rows: Knowledge[] = []
|
|
8288
|
+
if (knowledge) {
|
|
8289
|
+
for (let offset = 0;; offset += 500) {
|
|
8290
|
+
const page = await knowledge.list({ limit: 500, offset })
|
|
8291
|
+
rows.push(...page.knowledge)
|
|
8292
|
+
if (offset + page.knowledge.length >= page.total) break
|
|
8293
|
+
}
|
|
8294
|
+
}
|
|
8295
|
+
// Embeddings-only rebuild keeps the published card and knowledge generation.
|
|
8296
|
+
const previous = db.artifact(COMPILE_SNAPSHOT)
|
|
8297
|
+
const snapshot: CompileSnapshot = only === 'embeddings' && previous
|
|
8298
|
+
? { ...JSON.parse(previous), version: result.data.version, compileRunId: runId }
|
|
8299
|
+
: {
|
|
8300
|
+
version: result.data.version, compileRunId: runId,
|
|
8301
|
+
cards: this.cards, notes: this.units, graph: this.graph,
|
|
8302
|
+
diagnosisGraph: this.diagnosisGraph,
|
|
8303
|
+
placements: [...(truth?.placements() ?? new Map())], knowledge: rows,
|
|
8304
|
+
}
|
|
8305
|
+
this.publication.artifacts.set(COMPILE_SNAPSHOT, JSON.stringify(snapshot))
|
|
8306
|
+
await this.checkpointCompileTask(runId)
|
|
8307
|
+
if (JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() }) !== configAtStart) throw new Error('COMPILE_CONFIG_CHANGED: 编译期间设置已变化,请重新编译')
|
|
8308
|
+
db.publish(this.publication)
|
|
8309
|
+
this.vectorIndex = null
|
|
8310
|
+
}
|
|
8311
|
+
return result
|
|
8312
|
+
} finally {
|
|
8313
|
+
this.publication = undefined
|
|
8314
|
+
}
|
|
8315
|
+
}
|
|
8316
|
+
|
|
8317
|
+
private async prepareCompile(runId: string, only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
8165
8318
|
/**
|
|
8166
8319
|
* 进度记录**先占位**:`runId` 一落下去,这一趟就有了名字 —— 之后无论从哪个出口结束
|
|
8167
8320
|
* (三种 return,或者抛异常),收口都能认出这条记录属于谁。
|
|
@@ -8224,23 +8377,16 @@ export class KnowledgeRuntime {
|
|
|
8224
8377
|
this.writeCompileProgress({ ...(this.compileProgress ?? blankProgress(runId, only, startedAt, 'running', queuedAt)), ...patch })
|
|
8225
8378
|
}
|
|
8226
8379
|
/**
|
|
8227
|
-
*
|
|
8228
|
-
*
|
|
8229
|
-
* 它在 `rebuild()` 的**每一个** return 之前调用(一共三处出口)。少了任何一处,
|
|
8230
|
-
* 那次编译的进度就会永远停在「进行中」—— 而界面拿它当「正在跑」,于是按钮一直灰着。
|
|
8380
|
+
* 四段准备完成,保存结论。事务发布之前仍是 running;发布成功后由
|
|
8381
|
+
* settleProgressReport 收口,发布失败则由 settleProgressFailure 收口。
|
|
8231
8382
|
*/
|
|
8232
8383
|
const finishProgress = (outcome: string): void => {
|
|
8233
8384
|
const current = this.compileProgress
|
|
8234
8385
|
if (current === null) return
|
|
8235
|
-
|
|
8236
|
-
* 收口时**立刻存档**,不只写槽位:槽位会被下一趟(排在后面的那一趟)顶掉,
|
|
8237
|
-
* 而存档按编号留着 —— 发起编译的那一方无论何时来读,都读得到自己那一份。
|
|
8238
|
-
*/
|
|
8239
|
-
this.rememberCompile({
|
|
8386
|
+
this.writeCompileProgress({
|
|
8240
8387
|
...current,
|
|
8241
|
-
state: '
|
|
8388
|
+
state: 'running',
|
|
8242
8389
|
queued: false,
|
|
8243
|
-
finishedAt: new Date().toISOString(),
|
|
8244
8390
|
sourcesDone: Math.max(current.sourcesDone, current.sourcesTotal),
|
|
8245
8391
|
steps: current.steps.map(step => ({ ...step, state: 'done' })),
|
|
8246
8392
|
outcome,
|
|
@@ -8269,7 +8415,7 @@ export class KnowledgeRuntime {
|
|
|
8269
8415
|
if (adoption.adopted > 0) this.cards = await this.loadCards()
|
|
8270
8416
|
|
|
8271
8417
|
// ------------------------------------------------ previous build state
|
|
8272
|
-
let previousManifest: { version?: number; builtAt?: string; cardHashes?: Record<string, string>; extractionHashes?: Record<string, string>; embedding?: unknown; diagnosisHash?: string; indexHash?: string } = {}
|
|
8418
|
+
let previousManifest: { version?: number; builtAt?: string; cardHashes?: Record<string, string>; extractionHashes?: Record<string, string>; embedding?: unknown; embeddingIdentity?: string; diagnosisHash?: string; indexHash?: string } = {}
|
|
8273
8419
|
try {
|
|
8274
8420
|
previousManifest = JSON.parse(await this.readArtifact('build-manifest.json')) as typeof previousManifest
|
|
8275
8421
|
} catch { /* first build */ }
|
|
@@ -8502,7 +8648,8 @@ export class KnowledgeRuntime {
|
|
|
8502
8648
|
*/
|
|
8503
8649
|
const incremental = only === 'wiki'
|
|
8504
8650
|
if (only === 'embeddings') {
|
|
8505
|
-
|
|
8651
|
+
await this.loadCompiledState()
|
|
8652
|
+
Object.assign(cardHashes, previousHashes)
|
|
8506
8653
|
} else {
|
|
8507
8654
|
const previousNotes = incremental ? await this.readNotesArtifact() : []
|
|
8508
8655
|
const previousByWiki = new Map<string, WikiNote[]>()
|
|
@@ -8574,7 +8721,9 @@ export class KnowledgeRuntime {
|
|
|
8574
8721
|
const diagnosis = await this.diagnosisStore().readGraph()
|
|
8575
8722
|
const diagnosisHash = contentHashOf(JSON.stringify({ nodes: diagnosis.nodes, edges: diagnosis.edges }))
|
|
8576
8723
|
const diagnosisChanged = diagnosisHash !== previousManifest.diagnosisHash
|
|
8577
|
-
const
|
|
8724
|
+
const embeddingIdentity = JSON.stringify({ provider: this.config.embeddingProvider, model: this.embeddingModelId, fingerprint: this.embeddingFingerprint(), dimension: this.config.embeddingDimension, encoder: ENCODER_FINGERPRINT, baseUrl: this.config.embeddingBaseUrl })
|
|
8725
|
+
const embeddingChanged = previousManifest.embeddingIdentity !== embeddingIdentity
|
|
8726
|
+
const corpusChanged = embeddingChanged || !incremental || previousVersion === 0 || cardsRecompiled > 0 || cardsRemoved > 0 || diagnosisChanged || derivedRefreshed || indexChanged
|
|
8578
8727
|
if (!corpusChanged) {
|
|
8579
8728
|
await this.checkpointCompileTask(runId)
|
|
8580
8729
|
// 这一趟的编号是调用方给的(0066):进度记录、报告、manifest 用同一个编号。
|
|
@@ -8597,7 +8746,7 @@ export class KnowledgeRuntime {
|
|
|
8597
8746
|
// 用户刚点了编译却看到上次的统计,会以为按钮没生效。version 不变,所以
|
|
8598
8747
|
// 这个写入既不改索引也不让向量缓存失效。extractionHashes 必须跟着更新:
|
|
8599
8748
|
// 来源变了但没抽出卡片时,不落新指纹会让下一次又把它重抽一遍。
|
|
8600
|
-
await this.writeArtifact('build-manifest.json', JSON.stringify({ version: previousVersion, builtAt: previousManifest.builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: previousManifest.embedding, diagnosisHash, indexHash, compileRunId, lastBuild: { only, incremental: true, skipped: true, cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0, notesReused, notesRecompiled: 0, ...extractionCounts, ...finalizeCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
8749
|
+
await this.writeArtifact('build-manifest.json', JSON.stringify({ version: previousVersion, builtAt: previousManifest.builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embeddingIdentity: JSON.stringify({ provider: this.config.embeddingProvider, model: this.embeddingModelId, fingerprint: this.embeddingFingerprint(), dimension: this.config.embeddingDimension, encoder: ENCODER_FINGERPRINT, baseUrl: this.config.embeddingBaseUrl }), embedding: previousManifest.embedding, diagnosisHash, indexHash, compileRunId, lastBuild: { only, incremental: true, skipped: true, cardsReused, cardsRecompiled: 0, cardsAdded: 0, cardsRemoved: 0, notesReused, notesRecompiled: 0, ...extractionCounts, ...finalizeCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
8601
8750
|
const detail = detailOf({ skipped: true, version: previousVersion, cards: { created: [], updated: [], reused: reusedTitles.slice(0, MAX_LOG_ITEMS), removed: [] } })
|
|
8602
8751
|
report.detail = detail
|
|
8603
8752
|
await settleOp(detail, `没有变化:复用 ${notesReused} 条召回片段,索引仍为 v${previousVersion}。`, 'done')
|
|
@@ -8668,6 +8817,7 @@ export class KnowledgeRuntime {
|
|
|
8668
8817
|
await this.writeArtifact(DIAGNOSIS_GRAPH, JSON.stringify({ version, builtAt, nodes: diagnosis.nodes, edges: diagnosis.edges }, null, 2))
|
|
8669
8818
|
await this.checkpointCompileTask(runId)
|
|
8670
8819
|
const embeddingResult = await this.buildVectorIndex(this.units, version)
|
|
8820
|
+
if (this.environmentGuard && embeddingResult.status === 'unavailable') throw new Error(`向量索引构建失败:${embeddingResult.error ?? '未知原因'}。请重新自检环境后重试编译`)
|
|
8671
8821
|
await this.checkpointCompileTask(runId)
|
|
8672
8822
|
setStep('build', 'done', [
|
|
8673
8823
|
`知识图谱 ${this.graph.nodes.length} 节点 / ${this.graph.edges.length} 边`
|
|
@@ -8680,7 +8830,7 @@ export class KnowledgeRuntime {
|
|
|
8680
8830
|
])
|
|
8681
8831
|
// 这一趟的编号是调用方给的(0066):进度记录、报告、manifest 用同一个编号。
|
|
8682
8832
|
const compileRunId = runId
|
|
8683
|
-
await this.writeArtifact('build-manifest.json', JSON.stringify({ version, builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: embeddingResult, fts: ftsResult, compileRunId, diagnosisHash, indexHash, diagnosisGraph: { nodes: diagnosis.nodes.length, edges: diagnosis.edges.length }, lastBuild: { only, incremental: cardsReused > 0, skipped: false, cardsReused, cardsRecompiled, cardsAdded, cardsRemoved: removedCount, notesReused, notesRecompiled, ...extractionCounts, ...finalizeCounts, relationsExtracted, relationsDropped }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
8833
|
+
await this.writeArtifact('build-manifest.json', JSON.stringify({ version, builtAt, sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embeddingIdentity: JSON.stringify({ provider: this.config.embeddingProvider, model: this.embeddingModelId, fingerprint: this.embeddingFingerprint(), dimension: this.config.embeddingDimension, encoder: ENCODER_FINGERPRINT, baseUrl: this.config.embeddingBaseUrl }), embedding: embeddingResult, fts: ftsResult, compileRunId, diagnosisHash, indexHash, diagnosisGraph: { nodes: diagnosis.nodes.length, edges: diagnosis.edges.length }, lastBuild: { only, incremental: cardsReused > 0, skipped: false, cardsReused, cardsRecompiled, cardsAdded, cardsRemoved: removedCount, notesReused, notesRecompiled, ...extractionCounts, ...finalizeCounts, relationsExtracted, relationsDropped }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
8684
8834
|
const report: RebuildReport = {
|
|
8685
8835
|
runId: compileRunId, compileRunId, version, only,
|
|
8686
8836
|
cards: this.cards.length, units: this.units.length, notes: this.units.length,
|
|
@@ -8732,6 +8882,7 @@ export class KnowledgeRuntime {
|
|
|
8732
8882
|
const version = previousVersion + 1
|
|
8733
8883
|
await this.checkpointCompileTask(runId)
|
|
8734
8884
|
const embeddingResult = await this.buildVectorIndex(this.units, version)
|
|
8885
|
+
if (this.environmentGuard && embeddingResult.status === 'unavailable') throw new Error(`向量索引构建失败:${embeddingResult.error ?? '未知原因'}。请重新自检环境后重试编译`)
|
|
8735
8886
|
await this.checkpointCompileTask(runId)
|
|
8736
8887
|
// 只重建向量这一趟只跑第五段(其余四段在开跑时就标成「本次不涉及」,见 `blankSteps`)。
|
|
8737
8888
|
setStep('build', 'done', [
|
|
@@ -8742,7 +8893,7 @@ export class KnowledgeRuntime {
|
|
|
8742
8893
|
])
|
|
8743
8894
|
// 这一趟的编号是调用方给的(0066):进度记录、报告、manifest 用同一个编号。
|
|
8744
8895
|
const compileRunId = runId
|
|
8745
|
-
await this.writeArtifact('build-manifest.json', JSON.stringify({ version, builtAt: new Date().toISOString(), sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embedding: embeddingResult, compileRunId, lastBuild: { only, incremental: embeddingResult.reused > 0, skipped: false, notesReused: embeddingResult.reused, notesRecompiled: Math.max(0, embeddingResult.count - embeddingResult.reused), ...extractionCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
8896
|
+
await this.writeArtifact('build-manifest.json', JSON.stringify({ ...previousManifest, version, builtAt: new Date().toISOString(), sourceCount: sources.length, cardCount: this.cards.length, noteCount: this.units.length, cardHashes, extractionHashes: extraction.extractionHashes, embeddingIdentity: JSON.stringify({ provider: this.config.embeddingProvider, model: this.embeddingModelId, fingerprint: this.embeddingFingerprint(), dimension: this.config.embeddingDimension, encoder: ENCODER_FINGERPRINT, baseUrl: this.config.embeddingBaseUrl }), embedding: embeddingResult, compileRunId, lastBuild: { only, incremental: embeddingResult.reused > 0, skipped: false, notesReused: embeddingResult.reused, notesRecompiled: Math.max(0, embeddingResult.count - embeddingResult.reused), ...extractionCounts, relationsExtracted: 0, relationsDropped: 0 }, ...(lineage ? { lineage } : {}) }, null, 2))
|
|
8746
8897
|
const report: RebuildReport = {
|
|
8747
8898
|
runId: compileRunId, compileRunId, version, only,
|
|
8748
8899
|
cards: this.cards.length, units: this.units.length, notes: this.units.length,
|
|
@@ -8821,6 +8972,7 @@ export class KnowledgeRuntime {
|
|
|
8821
8972
|
rightNorm += right[index]! * right[index]!
|
|
8822
8973
|
}
|
|
8823
8974
|
const cosine = dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm) || 1)
|
|
8975
|
+
if (leftNorm < 1e-12 || rightNorm < 1e-12) throw new Error('ENCODER_PROBE_ZERO_VECTOR: 编码器返回零向量')
|
|
8824
8976
|
if (!Number.isFinite(cosine) || cosine > ENCODER_SELF_SIMILARITY_MAX) {
|
|
8825
8977
|
this.encoderHealth = { ok: false, reason: `编码器对两段无关文本给出同一个向量(cos = ${cosine.toFixed(4)}),向量路已停用` }
|
|
8826
8978
|
return this.encoderHealth
|
|
@@ -8974,13 +9126,13 @@ export class KnowledgeRuntime {
|
|
|
8974
9126
|
}
|
|
8975
9127
|
|
|
8976
9128
|
/** 关键词路:FTS5 + BM25(0031)。索引不在时如实降级,不退回旧的计数打分。 */
|
|
8977
|
-
private async ftsRoute(query: string, eligible: WikiNote[], pool: number, threshold: number, titleOf: (sourceId: string) => string | undefined): Promise<{ ranking: RouteRanking[]; rows: Array<RouteRanking & { kept: boolean }>; diagnostic: RecallRouteDiagnostic }> {
|
|
9129
|
+
private async ftsRoute(query: string, eligible: WikiNote[], pool: number, threshold: number, titleOf: (sourceId: string) => string | undefined, captured?: { ready: boolean; hits: FtsHit[] }): Promise<{ ranking: RouteRanking[]; rows: Array<RouteRanking & { kept: boolean }>; diagnostic: RecallRouteDiagnostic }> {
|
|
8978
9130
|
const eligibleIds = new Set(eligible.map(note => note.unitId))
|
|
8979
9131
|
const derived = await this.derivedHandle()
|
|
8980
9132
|
if (derived === null) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '当前 Node 没有 node:sqlite(需要 22.5+),关键词路已降级' } }
|
|
8981
|
-
if (!derived.hasFts()) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '关键词索引还没建(在 derived.db 里),先编译一次' } }
|
|
9133
|
+
if (!(captured?.ready ?? derived.hasFts())) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '关键词索引还没建(在 derived.db 里),先编译一次' } }
|
|
8982
9134
|
try {
|
|
8983
|
-
const found = derived.searchFts(query, pool).filter(hit => eligibleIds.has(hit.unitId))
|
|
9135
|
+
const found = (captured?.hits ?? derived.searchFts(query, pool)).filter(hit => eligibleIds.has(hit.unitId))
|
|
8984
9136
|
/**
|
|
8985
9137
|
* 覆盖率门槛(见 `MIN_QUERY_COVERAGE`):只蹭到一两个高频单字的命中在这里淘汰,
|
|
8986
9138
|
* 于是「整批都弱」也能弃答 —— 这是相对阈值做不到的那一半。
|
|
@@ -9046,8 +9198,8 @@ export class KnowledgeRuntime {
|
|
|
9046
9198
|
* 阈值就完全失去意义(余弦 0 会被映射成 0.5,轻松过线)。原始余弦是绝对量,可以直接
|
|
9047
9199
|
* 用来判「这段内容和问题到底像不像」—— 弃答主要靠它和重排阈值。
|
|
9048
9200
|
*/
|
|
9049
|
-
private async vectorRoute(query: string, eligibleIds: Set<string>, pool: number, threshold: number, expectedVersion?: number): Promise<{ ranking: RouteRanking[]; rows: Array<RouteRanking & { kept: boolean }>; diagnostic: RecallRouteDiagnostic; health: { ok: boolean; reason?: string }; timings?: { embedMs: number; scanMs: number } }> {
|
|
9050
|
-
const index = await this.loadVectorIndex()
|
|
9201
|
+
private async vectorRoute(query: string, eligibleIds: Set<string>, pool: number, threshold: number, expectedVersion?: number, captured?: LoadedVectorIndex | null): Promise<{ ranking: RouteRanking[]; rows: Array<RouteRanking & { kept: boolean }>; diagnostic: RecallRouteDiagnostic; health: { ok: boolean; reason?: string }; timings?: { embedMs: number; scanMs: number } }> {
|
|
9202
|
+
const index = captured === undefined ? await this.loadVectorIndex() : captured
|
|
9051
9203
|
if (!index) {
|
|
9052
9204
|
const reason = '向量索引不存在或损坏,请执行 embeddings 编译重建'
|
|
9053
9205
|
return { ranking: [], rows: [], diagnostic: { route: 'vector', status: 'degraded', reason }, health: { ok: false, reason } }
|
|
@@ -9059,7 +9211,7 @@ export class KnowledgeRuntime {
|
|
|
9059
9211
|
const health = await this.checkEncoder()
|
|
9060
9212
|
if (!health.ok) return { ranking: [], rows: [], diagnostic: { route: 'vector', status: 'degraded', reason: health.reason }, health }
|
|
9061
9213
|
try {
|
|
9062
|
-
const { scores, embedMs, scanMs } = await this.vectorRecall(query, eligibleIds, pool)
|
|
9214
|
+
const { scores, embedMs, scanMs } = await this.vectorRecall(query, eligibleIds, pool, index)
|
|
9063
9215
|
const ranking = [...scores.entries()].map(([unitId, score]) => ({ unitId, score })).sort((left, right) => right.score - left.score || left.unitId.localeCompare(right.unitId))
|
|
9064
9216
|
const { kept, dropped, topScore, rows } = applyThreshold(ranking, threshold)
|
|
9065
9217
|
return {
|
|
@@ -9101,10 +9253,10 @@ export class KnowledgeRuntime {
|
|
|
9101
9253
|
* 是因为两张图的节点含义不同(一个是卡片,一个是字段级节点),合成一张会让
|
|
9102
9254
|
* 图谱 Tab 看到一张卡片散成七个点。
|
|
9103
9255
|
*/
|
|
9104
|
-
private diagnosisHits(queryTokens: string[]): Map<string, number> {
|
|
9256
|
+
private diagnosisHits(queryTokens: string[], graph: ContextGraph = this.diagnosisGraph): Map<string, number> {
|
|
9105
9257
|
const seed = new Map<string, number>()
|
|
9106
|
-
if (
|
|
9107
|
-
for (const node of
|
|
9258
|
+
if (graph.nodes.length === 0) return seed
|
|
9259
|
+
for (const node of graph.nodes) {
|
|
9108
9260
|
const title = String(node.title ?? '').toLocaleLowerCase()
|
|
9109
9261
|
const id = String(node.id ?? '').toLocaleLowerCase()
|
|
9110
9262
|
let score = 0
|
|
@@ -9115,7 +9267,7 @@ export class KnowledgeRuntime {
|
|
|
9115
9267
|
if (score > 0) seed.set(String(node.id), score)
|
|
9116
9268
|
}
|
|
9117
9269
|
// 关系词命中:两端各得 1 分。中文提问不会写 `refutes`,所以走 RELATION_QUERY_WORDS。
|
|
9118
|
-
for (const edge of
|
|
9270
|
+
for (const edge of graph.edges) {
|
|
9119
9271
|
const type = String(edge.kind ?? '')
|
|
9120
9272
|
const relation = (RELATION_QUERY_WORDS[type] ?? type.toLocaleLowerCase().replace(/_/g, ' ')).toLocaleLowerCase()
|
|
9121
9273
|
let score = 0
|
|
@@ -9128,7 +9280,7 @@ export class KnowledgeRuntime {
|
|
|
9128
9280
|
if (seed.size === 0) return seed
|
|
9129
9281
|
const expanded = new Map<string, number>(seed)
|
|
9130
9282
|
for (const [id, score] of seed) {
|
|
9131
|
-
for (const edge of
|
|
9283
|
+
for (const edge of graph.edges) {
|
|
9132
9284
|
const from = String(edge.source)
|
|
9133
9285
|
const to = String(edge.target)
|
|
9134
9286
|
if (from === id) expanded.set(to, Math.max(expanded.get(to) ?? 0, score * 0.5))
|
|
@@ -9138,8 +9290,8 @@ export class KnowledgeRuntime {
|
|
|
9138
9290
|
return expanded
|
|
9139
9291
|
}
|
|
9140
9292
|
|
|
9141
|
-
private async graphBoostRanking(query: string, eligible: WikiNote[], includeProblems: boolean): Promise<RouteRanking[]> {
|
|
9142
|
-
if (
|
|
9293
|
+
private async graphBoostRanking(query: string, eligible: WikiNote[], includeProblems: boolean, graph: ContextGraph = this.graph, diagnosisGraph: ContextGraph = this.diagnosisGraph): Promise<RouteRanking[]> {
|
|
9294
|
+
if (graph.nodes.length === 0) return []
|
|
9143
9295
|
const queryTokens = segment(query)
|
|
9144
9296
|
if (queryTokens.length === 0) return []
|
|
9145
9297
|
const byNodeKey = new Map<string, number>()
|
|
@@ -9148,9 +9300,9 @@ export class KnowledgeRuntime {
|
|
|
9148
9300
|
const nodeKey = (node: Record<string, unknown>): string => `${String(node.type)}:${String(node.id)}`
|
|
9149
9301
|
/** 卡片级节点 id → 它的 `entity_type`。归因图的节点 id 与卡片 id 相等(§14.1)。 */
|
|
9150
9302
|
const typeByEntityId = new Map<string, string>()
|
|
9151
|
-
for (const node of
|
|
9303
|
+
for (const node of graph.nodes) typeByEntityId.set(String(node.id), String(node.type))
|
|
9152
9304
|
|
|
9153
|
-
for (const node of
|
|
9305
|
+
for (const node of graph.nodes) {
|
|
9154
9306
|
if (!visible(node)) continue
|
|
9155
9307
|
const title = String(node.title ?? '').toLocaleLowerCase()
|
|
9156
9308
|
const id = String(node.id ?? '').toLocaleLowerCase()
|
|
@@ -9161,7 +9313,7 @@ export class KnowledgeRuntime {
|
|
|
9161
9313
|
}
|
|
9162
9314
|
if (score > 0) boost(nodeKey(node), score)
|
|
9163
9315
|
}
|
|
9164
|
-
for (const edge of
|
|
9316
|
+
for (const edge of graph.edges) {
|
|
9165
9317
|
const type = String(edge.type ?? '')
|
|
9166
9318
|
const relation = (RELATION_QUERY_WORDS[type] ?? type.toLocaleLowerCase().replace(/_/g, ' ')).toLocaleLowerCase()
|
|
9167
9319
|
let score = 0
|
|
@@ -9176,14 +9328,14 @@ export class KnowledgeRuntime {
|
|
|
9176
9328
|
* 归因图上命中的节点,映射回同 id 的卡片级节点加权 —— **不需要合图**,
|
|
9177
9329
|
* 因为两张图的节点 id 本来就相等(§14.1)。
|
|
9178
9330
|
*/
|
|
9179
|
-
for (const [id, score] of this.diagnosisHits(queryTokens)) {
|
|
9331
|
+
for (const [id, score] of this.diagnosisHits(queryTokens, diagnosisGraph)) {
|
|
9180
9332
|
const type = typeByEntityId.get(id)
|
|
9181
9333
|
if (type) boost(`${type}:${id}`, score)
|
|
9182
9334
|
}
|
|
9183
9335
|
if (byNodeKey.size === 0) return []
|
|
9184
9336
|
const expanded = new Map<string, number>(byNodeKey)
|
|
9185
9337
|
for (const [key, score] of byNodeKey) {
|
|
9186
|
-
for (const edge of
|
|
9338
|
+
for (const edge of graph.edges) {
|
|
9187
9339
|
const from = String(edge.from)
|
|
9188
9340
|
const to = String(edge.to)
|
|
9189
9341
|
if (from === key) expanded.set(to, Math.max(expanded.get(to) ?? 0, score * 0.5))
|
|
@@ -9263,7 +9415,7 @@ export class KnowledgeRuntime {
|
|
|
9263
9415
|
* 这次召回少一条证据,而正确的答案是换成它被并进的那一条(`mergedInto` 只跳一跳:
|
|
9264
9416
|
* 并进的目标自己不会再被并 —— 那正是「并」这个动作的定义)。
|
|
9265
9417
|
*/
|
|
9266
|
-
private async attachKnowledge(hits: RecallHit[]): Promise<void> {
|
|
9418
|
+
private async attachKnowledge(hits: RecallHit[], snapshot?: CompileSnapshot): Promise<void> {
|
|
9267
9419
|
if (hits.length === 0) return
|
|
9268
9420
|
const store = await this.storeHandle()
|
|
9269
9421
|
if (store === null) return
|
|
@@ -9274,7 +9426,7 @@ export class KnowledgeRuntime {
|
|
|
9274
9426
|
* 只取**命中这几张卡**的落位。用 `placements()`(全表)在这里是错的量级:
|
|
9275
9427
|
* 一次召回十来张卡,而库里可能有上千张。
|
|
9276
9428
|
*/
|
|
9277
|
-
try { placements = store.placementsOf([...new Set(hits.map(hit => hit.entityId))]) } catch { return }
|
|
9429
|
+
try { placements = snapshot?.placements.length ? new Map(snapshot.placements) : store.placementsOf([...new Set(hits.map(hit => hit.entityId))]) } catch { return }
|
|
9278
9430
|
/**
|
|
9279
9431
|
* 每条命中要哪几条 Knowledge:以 Card 正文切出的 heading 为准。
|
|
9280
9432
|
*
|
|
@@ -9290,7 +9442,14 @@ export class KnowledgeRuntime {
|
|
|
9290
9442
|
})
|
|
9291
9443
|
const wanted = new Set(perHit.flat().map(row => row.knowledgeId))
|
|
9292
9444
|
if (wanted.size === 0) return
|
|
9293
|
-
const rows =
|
|
9445
|
+
const rows = snapshot && snapshot.knowledge.length > 0
|
|
9446
|
+
? new Map(snapshot.knowledge.filter(row => wanted.has(row.id) && knowledge !== null)
|
|
9447
|
+
.flatMap(row => {
|
|
9448
|
+
// Deleted facts never reappear through a historical snapshot.
|
|
9449
|
+
const live = store.raw('SELECT status FROM knowledge WHERE id = ?').get(row.id) as { status: KnowledgeState } | undefined
|
|
9450
|
+
return live ? [[row.id, recallKnowledgeOf(assessKnowledgeEvidence({ ...row, status: live.status }, store), '')] as [string, RecallKnowledge]] : []
|
|
9451
|
+
}))
|
|
9452
|
+
: await this.readKnowledgeRows(knowledge, [...wanted])
|
|
9294
9453
|
if (rows.size === 0) return
|
|
9295
9454
|
hits.forEach((hit, index) => {
|
|
9296
9455
|
const collected = new Map<string, RecallKnowledge>()
|
|
@@ -9318,11 +9477,11 @@ export class KnowledgeRuntime {
|
|
|
9318
9477
|
const followup = new Map<string, string>()
|
|
9319
9478
|
for (const row of rows.knowledge) {
|
|
9320
9479
|
if (row.mergedInto !== '') followup.set(row.mergedInto, row.id)
|
|
9321
|
-
else out.set(row.id, recallKnowledgeOf(row, ''))
|
|
9480
|
+
else out.set(row.id, recallKnowledgeOf(assessKnowledgeEvidence(row, (await this.storeHandle())!), ''))
|
|
9322
9481
|
}
|
|
9323
9482
|
if (followup.size > 0) {
|
|
9324
9483
|
rows = await store.list({ ids: [...followup.keys()], limit: 500 })
|
|
9325
|
-
for (const row of rows.knowledge) out.set(row.id, recallKnowledgeOf(row, ''))
|
|
9484
|
+
for (const row of rows.knowledge) out.set(row.id, recallKnowledgeOf(assessKnowledgeEvidence(row, (await this.storeHandle())!), ''))
|
|
9326
9485
|
// 目标也没了(例如它的来源被删了):那条落位就是悬空的,如实**不显示**,
|
|
9327
9486
|
// 而不是把一条已经不存在的事实塞进上下文。
|
|
9328
9487
|
}
|
|
@@ -9377,25 +9536,59 @@ export class KnowledgeRuntime {
|
|
|
9377
9536
|
if (!query?.trim()) return failure(empty(), 'QUERY_REQUIRED', 'query 不能为空', true)
|
|
9378
9537
|
if (!Number.isInteger(recallTopK) || recallTopK < 1 || recallTopK > 50) return failure(empty(), 'LIMIT_INVALID', '召回 top_k 必须是 1 到 50 的整数', false)
|
|
9379
9538
|
if (!Number.isInteger(requestedTopN) || requestedTopN < 1 || (options.contextTopN !== undefined && requestedTopN > recallTopK)) return failure(empty(), 'CONTEXT_TOP_N_INVALID', `上下文 top_n 必须是 1 到 ${recallTopK} 的整数`, false)
|
|
9380
|
-
|
|
9381
|
-
|
|
9382
|
-
|
|
9383
|
-
|
|
9384
|
-
|
|
9385
|
-
|
|
9539
|
+
try { await this.environmentGuard?.() }
|
|
9540
|
+
catch (cause) { return failure(empty(), 'ENVIRONMENT_NOT_READY', cause instanceof Error ? cause.message : String(cause), true, '在 Tita 设置中完成环境准备并重新自检。') }
|
|
9541
|
+
const derived = await this.derivedHandle()
|
|
9542
|
+
const liveCards = await this.loadCards()
|
|
9543
|
+
const liveKeys = new Set(liveCards.map(card => cardKey(card)))
|
|
9544
|
+
const snapshotHash = derived?.artifactHash(COMPILE_SNAPSHOT)
|
|
9545
|
+
const cached = snapshotHash && this.recallSnapshotCache?.hash === snapshotHash ? this.recallSnapshotCache : undefined
|
|
9546
|
+
const raw = cached ? undefined : derived?.artifact(COMPILE_SNAPSHOT)
|
|
9547
|
+
let snapshot: CompileSnapshot
|
|
9548
|
+
if (cached) snapshot = cached.snapshot
|
|
9549
|
+
else if (raw) snapshot = JSON.parse(raw)
|
|
9550
|
+
else {
|
|
9551
|
+
// Upgrade path for stores built before compiled snapshots existed.
|
|
9552
|
+
const manifest = JSON.parse(derived?.artifact('build-manifest.json') ?? '{}')
|
|
9553
|
+
snapshot = {
|
|
9554
|
+
version: manifest.version ?? 0, compileRunId: manifest.compileRunId,
|
|
9555
|
+
notes: JSON.parse(derived?.artifact(WIKI_NOTES) ?? derived?.artifact(LEGACY_UNITS) ?? '[]'),
|
|
9556
|
+
cards: liveCards, graph: JSON.parse(derived?.artifact('knowledge_graph.json') ?? '{"nodes":[],"edges":[]}'),
|
|
9557
|
+
diagnosisGraph: JSON.parse(derived?.artifact(DIAGNOSIS_GRAPH) ?? '{"nodes":[],"edges":[]}'),
|
|
9558
|
+
placements: [], knowledge: [],
|
|
9559
|
+
}
|
|
9560
|
+
}
|
|
9561
|
+
if (snapshot.notes.length === 0) return failure(empty(), 'INDEX_NOT_READY', '知识索引尚未构建,请先编译知识库', true, '先执行 rebuild_knowledge,再进行召回。')
|
|
9562
|
+
const capturedFts = { ready: derived?.hasFts() ?? false, hits: derived?.searchFts(query, candidatePool) ?? [] }
|
|
9563
|
+
const capturedVector = cached ? cached.vector : decodeVectorIndex(derived?.vectorIndex(), ENCODER_FINGERPRINT)
|
|
9564
|
+
if (this.environmentGuard && recallMode !== 'fts' && (!capturedVector
|
|
9565
|
+
|| capturedVector.manifest.provider !== this.config.embeddingProvider
|
|
9566
|
+
|| capturedVector.manifest.dimension !== this.config.embeddingDimension
|
|
9567
|
+
|| capturedVector.manifest.model !== this.embeddingModelId
|
|
9568
|
+
|| capturedVector.manifest.modelFingerprint !== this.embeddingFingerprint())) {
|
|
9569
|
+
return failure(empty(), 'INDEX_NOT_READY', '环境已准备,但当前模型的向量索引尚未构建,请重新编译知识库', true, '执行 rebuild_knowledge,完成后再召回。')
|
|
9570
|
+
}
|
|
9571
|
+
if (snapshotHash) this.recallSnapshotCache = { hash: snapshotHash, snapshot, vector: capturedVector }
|
|
9386
9572
|
const eventLineage = withRun(lineage, 'recall')
|
|
9387
|
-
const indexVersion =
|
|
9388
|
-
|
|
9389
|
-
|
|
9390
|
-
|
|
9391
|
-
|
|
9392
|
-
|
|
9393
|
-
|
|
9394
|
-
|
|
9395
|
-
|
|
9396
|
-
|
|
9573
|
+
const indexVersion = capturedVector?.manifest.version
|
|
9574
|
+
const compileRunId = snapshot.compileRunId
|
|
9575
|
+
const knowledgeIndexVersion = snapshot.version
|
|
9576
|
+
const truthAtRead = await this.storeHandle()
|
|
9577
|
+
const liveSourceIds = new Set((truthAtRead?.raw('SELECT entity_id FROM source').all() as Array<{ entity_id: string }> | undefined)?.map(row => row.entity_id) ?? [])
|
|
9578
|
+
const snapshotCardByKey = new Map(snapshot.cards.map(card => [cardKey(card), card]))
|
|
9579
|
+
for (const key of liveKeys) {
|
|
9580
|
+
if (snapshotCardByKey.get(key)?.sourceRefs.some(ref => !liveSourceIds.has(ref))) liveKeys.delete(key)
|
|
9581
|
+
}
|
|
9582
|
+
let publishedGraph = visibleGraph(snapshot.graph, truthAtRead ? liveGraphKeys(truthAtRead) : new Set())
|
|
9583
|
+
const currentByKey = new Map(liveCards.map(card => [cardKey(card), card]))
|
|
9584
|
+
let cards: KnowledgeCard[] = snapshot.cards.filter(card => liveKeys.has(cardKey(card))).map(card => {
|
|
9585
|
+
const current = currentByKey.get(cardKey(card))!
|
|
9586
|
+
return { ...card, status: current.status, annotations: current.annotations, review: current.review,
|
|
9587
|
+
metadata: { ...card.metadata, ...(current.metadata.evidence_needs_review === true ? { evidence_needs_review: true } : {}) } }
|
|
9588
|
+
})
|
|
9589
|
+
const eligible = snapshot.notes.filter(unit => liveKeys.has(`${unit.entityType}:${unit.entityId}`)
|
|
9590
|
+
&& (includeProblems || currentByKey.get(`${unit.entityType}:${unit.entityId}`)?.status !== 'problem'))
|
|
9397
9591
|
const eligibleIds = new Set(eligible.map(unit => unit.unitId))
|
|
9398
|
-
const cards = this.cards.length > 0 ? this.cards : await this.loadCards()
|
|
9399
9592
|
const queryTerms = [...new Set(segment(query))]
|
|
9400
9593
|
const warnings: string[] = []
|
|
9401
9594
|
// 一份 SourceRaw 语料,两处要用:关键词路的**查询词覆盖率**必须对着「建索引时那份
|
|
@@ -9429,16 +9622,19 @@ export class KnowledgeRuntime {
|
|
|
9429
9622
|
|
|
9430
9623
|
const ftsStarted = performance.now()
|
|
9431
9624
|
const ftsOutcome = useFts
|
|
9432
|
-
? await this.ftsRoute(query, eligible, candidatePool, this.config.keywordThreshold, titleOf)
|
|
9625
|
+
? await this.ftsRoute(query, eligible, candidatePool, this.config.keywordThreshold, titleOf, capturedFts)
|
|
9433
9626
|
: { ranking: [] as RouteRanking[], rows: [] as Array<RouteRanking & { kept: boolean }>, diagnostic: { route: 'fts' as const, status: 'skipped' as const, reason: '当前模式未启用关键词路' } }
|
|
9434
9627
|
const ftsMs = performance.now() - ftsStarted
|
|
9435
9628
|
const vectorStarted = performance.now()
|
|
9436
9629
|
const vectorOutcome = useVector
|
|
9437
|
-
? await this.vectorRoute(query, eligibleIds, candidatePool, this.config.vectorThreshold, knowledgeIndexVersion)
|
|
9630
|
+
? await this.vectorRoute(query, eligibleIds, candidatePool, this.config.vectorThreshold, knowledgeIndexVersion, capturedVector)
|
|
9438
9631
|
: { ranking: [] as RouteRanking[], rows: [] as Array<RouteRanking & { kept: boolean }>, diagnostic: { route: 'vector' as const, status: 'skipped' as const, reason: '当前模式未启用向量路' }, health: { ok: false, reason: '当前模式未启用向量路' } }
|
|
9632
|
+
if (this.environmentGuard && useVector && vectorOutcome.diagnostic.status === 'degraded') {
|
|
9633
|
+
return failure(empty(), 'ENVIRONMENT_NOT_READY', `向量推理失败:${vectorOutcome.diagnostic.reason ?? '未知原因'}`, true, '在 Tita 设置中重新自检环境后再召回。')
|
|
9634
|
+
}
|
|
9439
9635
|
const vectorMs = performance.now() - vectorStarted
|
|
9440
9636
|
const graphStarted = performance.now()
|
|
9441
|
-
const graphRanking = useGraphBoost ? await this.graphBoostRanking(query, eligible, includeProblems) : []
|
|
9637
|
+
const graphRanking = useGraphBoost ? await this.graphBoostRanking(query, eligible, includeProblems, publishedGraph, snapshot.diagnosisGraph) : []
|
|
9442
9638
|
const graphMs = performance.now() - graphStarted
|
|
9443
9639
|
|
|
9444
9640
|
if (coordinateForRecall.diagnostic.status === 'degraded' && coordinateForRecall.diagnostic.reason) warnings.push(coordinateForRecall.diagnostic.reason)
|
|
@@ -9508,11 +9704,19 @@ export class KnowledgeRuntime {
|
|
|
9508
9704
|
rerank = reranked.result
|
|
9509
9705
|
if (reranked.warning) warnings.push(reranked.warning)
|
|
9510
9706
|
}
|
|
9707
|
+
if (this.environmentGuard && rerank.status === 'unavailable') {
|
|
9708
|
+
return failure(empty(), 'ENVIRONMENT_NOT_READY', `重排推理失败:${rerank.error ?? '未知原因'}`, true, '在 Tita 设置中重新自检环境后再召回。')
|
|
9709
|
+
}
|
|
9511
9710
|
// 重排换的是相关性顺序,评分加权仍然要压在最上面 —— 否则「review 过的排前面一点」
|
|
9512
9711
|
// 只在关掉重排时成立,而重排默认是开着的。
|
|
9513
9712
|
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId))
|
|
9514
9713
|
|
|
9515
9714
|
// --- R4 · 证据片段 -----------------------------------------------------
|
|
9715
|
+
const truth = await this.storeHandle()
|
|
9716
|
+
cards = cards.filter(card => truth?.card(card.entityId) !== undefined && card.sourceRefs.every(ref => truth?.source(ref) !== undefined))
|
|
9717
|
+
const finalKeys = new Set(cards.map(card => cardKey(card)))
|
|
9718
|
+
publishedGraph = visibleGraph(publishedGraph, truth ? liveGraphKeys(truth) : new Set())
|
|
9719
|
+
hits = hits.filter(hit => finalKeys.has(`${hit.entityType}:${hit.entityId}`))
|
|
9516
9720
|
await this.attachEvidence(hits, queryTerms, corpus)
|
|
9517
9721
|
/**
|
|
9518
9722
|
* --- R4.5 · 支撑这一段的知识(0069)--------------------------------------
|
|
@@ -9520,7 +9724,10 @@ export class KnowledgeRuntime {
|
|
|
9520
9724
|
* 排在证据**之后**、去重**之前**:它挂在每条 note 上,按卡片去重时自然跟着
|
|
9521
9725
|
* 被选中的那一条走(`ContextItem.knowledge` 读的就是 `notes[0].knowledge`)。
|
|
9522
9726
|
*/
|
|
9523
|
-
await this.attachKnowledge(hits)
|
|
9727
|
+
await this.attachKnowledge(hits, snapshot)
|
|
9728
|
+
if (cards.some(card => card.metadata.evidence_needs_review === true)) warnings.push('部分卡片缺少知识落位记录,删除来源后无法核对每段正文的归属,需要人工复核。')
|
|
9729
|
+
if (hits.some(hit => hit.knowledge?.some(row => row.requiresReview))) warnings.push('部分知识依据历史或未记录版本的证据,需人工复核;证据变化不表示知识已失效。')
|
|
9730
|
+
if (hits.some(hit => hit.knowledge?.length)) warnings.push('引用核验仅检查来源定位;引用完整不代表陈述已通过语义核验。')
|
|
9524
9731
|
|
|
9525
9732
|
// --- R5 · 弃答 ---------------------------------------------------------
|
|
9526
9733
|
// 一路都没过阈值 = 这个知识库里没有能回答这个问题的东西。与其凑满 topN 让模型编,
|
|
@@ -9589,7 +9796,7 @@ export class KnowledgeRuntime {
|
|
|
9589
9796
|
.map(([key, notes]) => ({ key, notes, card: cardByKey.get(key) }))
|
|
9590
9797
|
.slice(0, contextTopN)
|
|
9591
9798
|
const wiki = hitCards.map(item => item.card).filter((card): card is KnowledgeCard => card !== undefined)
|
|
9592
|
-
const graph = this.oneHopGraph(new Set(hitCards.map(item => item.key)), includeProblems)
|
|
9799
|
+
const graph = this.oneHopGraph(new Set(hitCards.map(item => item.key)), includeProblems, publishedGraph)
|
|
9593
9800
|
|
|
9594
9801
|
/**
|
|
9595
9802
|
* **在去重之后**做 1 跳扩展(0050 的第二半)。
|
|
@@ -9597,7 +9804,7 @@ export class KnowledgeRuntime {
|
|
|
9597
9804
|
* 反过来(先扩展再去重)会把邻居与被命中的卡片混在一起,分不清「这是召回来的」
|
|
9598
9805
|
* 还是「这是邻居」—— 而这两件事在上下文里必须能分开,前者是证据,后者是延伸。
|
|
9599
9806
|
*/
|
|
9600
|
-
const expansion = this.expandWiki(hitCards.map(item => item.key), cardByKey, includeProblems)
|
|
9807
|
+
const expansion = this.expandWiki(hitCards.map(item => item.key), cardByKey, includeProblems, publishedGraph)
|
|
9601
9808
|
|
|
9602
9809
|
/** 上下文条目:先命中的卡片(带它们的 note),再扩展来的卡片。 */
|
|
9603
9810
|
const items: ContextItem[] = [
|
|
@@ -9605,7 +9812,7 @@ export class KnowledgeRuntime {
|
|
|
9605
9812
|
wiki: item.card ?? null,
|
|
9606
9813
|
notes: item.notes,
|
|
9607
9814
|
note: item.notes[0]!,
|
|
9608
|
-
graph: this.oneHopGraph(new Set([item.key]), includeProblems),
|
|
9815
|
+
graph: this.oneHopGraph(new Set([item.key]), includeProblems, publishedGraph),
|
|
9609
9816
|
reason: 'hit' as const,
|
|
9610
9817
|
...(item.notes[0]!.evidence ? { evidence: item.notes[0]!.evidence } : {}),
|
|
9611
9818
|
...(item.notes[0]!.knowledge ? { knowledge: item.notes[0]!.knowledge } : {}),
|
|
@@ -9618,7 +9825,7 @@ export class KnowledgeRuntime {
|
|
|
9618
9825
|
return {
|
|
9619
9826
|
wiki: card,
|
|
9620
9827
|
notes: [],
|
|
9621
|
-
graph: this.oneHopGraph(new Set([key]), includeProblems),
|
|
9828
|
+
graph: this.oneHopGraph(new Set([key]), includeProblems, publishedGraph),
|
|
9622
9829
|
reason: 'expansion' as const,
|
|
9623
9830
|
...(from === undefined ? {} : { via: { entityType: from.entityType, entityId: from.entityId, title: from.title, relation: relation ?? '' } }),
|
|
9624
9831
|
}
|
|
@@ -9729,10 +9936,10 @@ export class KnowledgeRuntime {
|
|
|
9729
9936
|
* 2. **量纲**:返回**原始余弦**,不再做 `(cos+1)/2` 的映射 —— 那个映射让 `0.15` 这类阈值
|
|
9730
9937
|
* 失去意义(余弦 0 → 0.5,轻松过线)。原始余弦是绝对量,可以直接用于弃答。
|
|
9731
9938
|
*/
|
|
9732
|
-
private async vectorRecall(query: string, eligibleIds: Set<string>, candidateLimit: number): Promise<{ scores: Map<string, number>; embedMs: number; scanMs: number }> {
|
|
9733
|
-
const index = await this.loadVectorIndex()
|
|
9939
|
+
private async vectorRecall(query: string, eligibleIds: Set<string>, candidateLimit: number, captured?: LoadedVectorIndex): Promise<{ scores: Map<string, number>; embedMs: number; scanMs: number }> {
|
|
9940
|
+
const index = captured ?? await this.loadVectorIndex()
|
|
9734
9941
|
if (!index || index.manifest.itemCount === 0) throw new Error('VECTOR_INDEX_NOT_READY')
|
|
9735
|
-
if (index.manifest.provider !== this.config.embeddingProvider || index.manifest.model !== this.embeddingModelId || index.manifest.dimension !== this.config.embeddingDimension) throw new Error('VECTOR_INDEX_CONFIG_MISMATCH')
|
|
9942
|
+
if (index.manifest.provider !== this.config.embeddingProvider || index.manifest.model !== this.embeddingModelId || index.manifest.modelFingerprint !== this.embeddingFingerprint() || index.manifest.dimension !== this.config.embeddingDimension) throw new Error('VECTOR_INDEX_CONFIG_MISMATCH')
|
|
9736
9943
|
// 查询编码单独计时:模型没加载时这一段是 8 秒,加载之后是 0.3 秒 —— 这一段的
|
|
9737
9944
|
// 数字就是「测试为什么慢」的答案本身,混进扫描时间里就看不出来了。
|
|
9738
9945
|
const embedStarted = performance.now()
|
|
@@ -9781,6 +9988,35 @@ export class KnowledgeRuntime {
|
|
|
9781
9988
|
}
|
|
9782
9989
|
}
|
|
9783
9990
|
|
|
9991
|
+
/** Installed resources become available without restarting the host. */
|
|
9992
|
+
reloadResources(): void {
|
|
9993
|
+
this.embeddingWorker?.close(); this.embeddingWorker = null
|
|
9994
|
+
this.rerankWorker?.close(); this.rerankWorker = null
|
|
9995
|
+
this.encoderHealth = undefined; this.rerankUnavailable = undefined
|
|
9996
|
+
this.updateConfig({ python: this.config.python, embeddingModel: this.config.embeddingModel, rerankModel: this.config.rerankModel })
|
|
9997
|
+
}
|
|
9998
|
+
|
|
9999
|
+
async environmentProbe(kind: 'embedding' | 'rerank'): Promise<{ ok: boolean; message: string }> {
|
|
10000
|
+
await mkdir(this.rootDir, { recursive: true })
|
|
10001
|
+
if (kind === 'embedding') {
|
|
10002
|
+
this.encoderHealth = undefined
|
|
10003
|
+
const health = await this.checkEncoder()
|
|
10004
|
+
return { ok: health.ok, message: health.ok ? `已完成 ${this.config.embeddingDimension} 维向量推理,未发现零向量或相同向量退化` : health.reason ?? '向量推理自检失败' }
|
|
10005
|
+
}
|
|
10006
|
+
if (this.config.rerankProvider === 'none') return { ok: false, message: '重排模型未启用' }
|
|
10007
|
+
try {
|
|
10008
|
+
if (!this.rerankWorker) this.rerankWorker = this.spawnWorker('RERANK')
|
|
10009
|
+
const response = await this.rerankWorker.request({ task: 'rerank', model: this.rerankModelPath, device: this.config.embeddingDevice,
|
|
10010
|
+
query: '销售额按签收时间统计', documents: ['销售额按签收时间统计,签收后计入销售额。', '烹饪意大利面需要煮沸清水并加入面条。'] }, this.config.rerankTimeoutMs)
|
|
10011
|
+
const scores: unknown = response?.scores
|
|
10012
|
+
if (!Array.isArray(scores) || scores.length !== 2 || scores.some(value => typeof value !== 'number' || !Number.isFinite(value))) throw new Error('重排自检返回分数无效')
|
|
10013
|
+
if (Number(response.vocabSize) < 1000 || !Number.isFinite(Number(response.vocabSize))) throw new Error('重排模型分词器词表缺失或退化')
|
|
10014
|
+
if (scores[0] <= scores[1]) throw new Error('重排模型未能把直接相关文本排在无关文本前')
|
|
10015
|
+
this.rerankUnavailable = undefined
|
|
10016
|
+
return { ok: true, message: '分词器与实际重排推理通过自检' }
|
|
10017
|
+
} catch (cause) { return { ok: false, message: cause instanceof Error ? cause.message : String(cause) } }
|
|
10018
|
+
}
|
|
10019
|
+
|
|
9784
10020
|
async embeddingHealth(): Promise<Envelope<Record<string, unknown>>> {
|
|
9785
10021
|
const index = await this.loadVectorIndex()
|
|
9786
10022
|
return success({
|
|
@@ -9887,8 +10123,12 @@ export class KnowledgeRuntime {
|
|
|
9887
10123
|
* 「盘上还没有编译产物」(`needsBuild`),要不要编由界面和 Agent 自己决定 ——
|
|
9888
10124
|
* 与 `recall` 同一条规矩:先读盘,读不到再说。
|
|
9889
10125
|
*/
|
|
9890
|
-
|
|
9891
|
-
const
|
|
10126
|
+
const derived = await this.derivedHandle()
|
|
10127
|
+
const snapshot = derived?.artifact(COMPILE_SNAPSHOT)
|
|
10128
|
+
const rawGraph: ContextGraph = snapshot ? JSON.parse(snapshot).graph : JSON.parse(derived?.artifact('knowledge_graph.json') ?? '{"nodes":[],"edges":[]}')
|
|
10129
|
+
const truth = await this.storeHandle()
|
|
10130
|
+
const graph = visibleGraph(rawGraph, truth ? liveGraphKeys(truth) : new Set())
|
|
10131
|
+
const needsBuild = !snapshot && graph.nodes.length === 0
|
|
9892
10132
|
const attachLineage = (data: Record<string, unknown>): Record<string, unknown> => eventLineage ? { ...data, lineage: eventLineage } : data
|
|
9893
10133
|
/** 空图 + 一句为什么空:调用方(尤其是 Agent)不能把「还没编译」读成「这个库没有知识」。 */
|
|
9894
10134
|
const attachBuild = (data: Record<string, unknown>): Record<string, unknown> => needsBuild ? { ...data, needsBuild: true } : data
|
|
@@ -9899,11 +10139,11 @@ export class KnowledgeRuntime {
|
|
|
9899
10139
|
|
|
9900
10140
|
const includeProblem = options.includeProblem === true
|
|
9901
10141
|
const visible = (node: Record<string, unknown>) => includeProblem || node.status !== 'problem'
|
|
9902
|
-
const allNodes =
|
|
10142
|
+
const allNodes = graph.nodes
|
|
9903
10143
|
.filter(visible)
|
|
9904
10144
|
.filter(node => !options.status || node.status === options.status)
|
|
9905
10145
|
const nodes = allNodes.filter(node => !entityType || node.type === entityType)
|
|
9906
|
-
const edges =
|
|
10146
|
+
const edges = graph.edges.filter(edge => allNodes.some(node => `${node.type}:${node.id}` === String(edge.from)) && allNodes.some(node => `${node.type}:${node.id}` === String(edge.to)))
|
|
9907
10147
|
if (action === 'list') return success(attachLineage(attachBuild({ nodes: nodes.slice(0, limit), edges: edges.slice(0, limit) })))
|
|
9908
10148
|
|
|
9909
10149
|
const nodeKey = (rawId: string | undefined, rawType?: string): string | undefined => {
|