dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -56
- package/cordis.patch.yml +36 -129
- package/eval/benchmark-extraction-format.mjs +20 -0
- package/lib/client.js +3262 -4142
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +1147 -364
- package/lib/types/axis-registry.d.ts +8 -1
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +62 -7
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +3 -0
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +9 -0
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +68 -9
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/diagnosis/cards.d.ts +7 -3
- package/lib/types/diagnosis/cards.d.ts.map +1 -1
- package/lib/types/diagnosis/cards.js +13 -8
- package/lib/types/diagnosis/cards.js.map +1 -1
- package/lib/types/diagnosis/run.d.ts +4 -13
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +26 -16
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/schema.js +2 -2
- package/lib/types/diagnosis/schema.js.map +1 -1
- package/lib/types/diagnosis/seed-source.d.ts +1 -1
- package/lib/types/diagnosis/seed-source.d.ts.map +1 -1
- package/lib/types/diagnosis/seed-source.js +5 -4
- package/lib/types/diagnosis/seed-source.js.map +1 -1
- package/lib/types/diagnosis/seed.d.ts +4 -4
- package/lib/types/diagnosis/seed.d.ts.map +1 -1
- package/lib/types/diagnosis/seed.js +6 -8
- package/lib/types/diagnosis/seed.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +18 -17
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +94 -66
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +10 -16
- package/lib/types/diagnosis/types.d.ts.map +1 -1
- package/lib/types/diagnosis/types.js.map +1 -1
- package/lib/types/environment/dsh-python.d.ts +77 -0
- package/lib/types/environment/dsh-python.d.ts.map +1 -0
- package/lib/types/environment/dsh-python.js +139 -0
- package/lib/types/environment/dsh-python.js.map +1 -0
- package/lib/types/environment/install.d.ts +9 -1
- package/lib/types/environment/install.d.ts.map +1 -1
- package/lib/types/environment/install.js +149 -24
- package/lib/types/environment/install.js.map +1 -1
- package/lib/types/environment/model-catalog.d.ts +34 -2
- package/lib/types/environment/model-catalog.d.ts.map +1 -1
- package/lib/types/environment/model-catalog.js +40 -2
- package/lib/types/environment/model-catalog.js.map +1 -1
- package/lib/types/environment/service.d.ts +13 -3
- package/lib/types/environment/service.d.ts.map +1 -1
- package/lib/types/environment/service.js +127 -23
- package/lib/types/environment/service.js.map +1 -1
- package/lib/types/environment/types.d.ts +9 -0
- package/lib/types/environment/types.d.ts.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js +13 -0
- package/lib/types/events.js.map +1 -1
- package/lib/types/extract-contract.d.ts +1 -1
- package/lib/types/extract-contract.d.ts.map +1 -1
- package/lib/types/extract-contract.js +34 -31
- package/lib/types/extract-contract.js.map +1 -1
- package/lib/types/extract-model.d.ts +2 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract-table-contract.d.ts +3 -0
- package/lib/types/extract-table-contract.d.ts.map +1 -0
- package/lib/types/extract-table-contract.js +30 -0
- package/lib/types/extract-table-contract.js.map +1 -0
- package/lib/types/extract-table.d.ts +8 -0
- package/lib/types/extract-table.d.ts.map +1 -0
- package/lib/types/extract-table.js +151 -0
- package/lib/types/extract-table.js.map +1 -0
- package/lib/types/extraction-runs.d.ts +30 -6
- package/lib/types/extraction-runs.d.ts.map +1 -1
- package/lib/types/extraction-runs.js +69 -8
- package/lib/types/extraction-runs.js.map +1 -1
- package/lib/types/fs-atomic.d.ts +0 -7
- package/lib/types/fs-atomic.d.ts.map +1 -1
- package/lib/types/fs-atomic.js +17 -2
- package/lib/types/fs-atomic.js.map +1 -1
- package/lib/types/images.d.ts +1 -1
- package/lib/types/images.d.ts.map +1 -1
- package/lib/types/images.js +2 -2
- package/lib/types/images.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +232 -50
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/scope.d.ts.map +1 -1
- package/lib/types/intent/scope.js +1 -0
- package/lib/types/intent/scope.js.map +1 -1
- package/lib/types/intent/tools.d.ts +19 -0
- package/lib/types/intent/tools.d.ts.map +1 -1
- package/lib/types/intent/tools.js +37 -1
- package/lib/types/intent/tools.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +5 -16
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge.d.ts +3 -0
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +21 -5
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/materialize-snapshot.d.ts +49 -0
- package/lib/types/materialize-snapshot.d.ts.map +1 -0
- package/lib/types/materialize-snapshot.js +2 -0
- package/lib/types/materialize-snapshot.js.map +1 -0
- package/lib/types/model-text.d.ts +1 -1
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +5 -4
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/oplog.d.ts.map +1 -1
- package/lib/types/oplog.js +7 -0
- package/lib/types/oplog.js.map +1 -1
- package/lib/types/pipeline/engine.d.ts +60 -0
- package/lib/types/pipeline/engine.d.ts.map +1 -0
- package/lib/types/pipeline/engine.js +338 -0
- package/lib/types/pipeline/engine.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +73 -0
- package/lib/types/pipeline/store.d.ts.map +1 -0
- package/lib/types/pipeline/store.js +143 -0
- package/lib/types/pipeline/store.js.map +1 -0
- package/lib/types/pipeline/types.d.ts +101 -0
- package/lib/types/pipeline/types.d.ts.map +1 -0
- package/lib/types/pipeline/types.js +5 -0
- package/lib/types/pipeline/types.js.map +1 -0
- package/lib/types/present.d.ts +2 -0
- package/lib/types/present.d.ts.map +1 -1
- package/lib/types/present.js +9 -4
- package/lib/types/present.js.map +1 -1
- package/lib/types/retention.d.ts +39 -0
- package/lib/types/retention.d.ts.map +1 -0
- package/lib/types/retention.js +42 -0
- package/lib/types/retention.js.map +1 -0
- package/lib/types/runtime.d.ts +115 -5
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +862 -132
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/source-evidence.d.ts +9 -1
- package/lib/types/source-evidence.d.ts.map +1 -1
- package/lib/types/source-evidence.js +13 -5
- package/lib/types/source-evidence.js.map +1 -1
- package/lib/types/store/axis-references.js +3 -3
- package/lib/types/store/axis-references.js.map +1 -1
- package/lib/types/store/derived.d.ts +1 -1
- package/lib/types/store/derived.d.ts.map +1 -1
- package/lib/types/store/derived.js +41 -7
- package/lib/types/store/derived.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +18 -0
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +111 -6
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/migrate.js +1 -1
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/trajectory-types.d.ts +172 -0
- package/lib/types/trajectory-types.d.ts.map +1 -0
- package/lib/types/trajectory-types.js +36 -0
- package/lib/types/trajectory-types.js.map +1 -0
- package/lib/types/types.d.ts +3 -4
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-finalize.d.ts +16 -0
- package/lib/types/wiki-finalize.d.ts.map +1 -1
- package/lib/types/wiki-finalize.js +26 -0
- package/lib/types/wiki-finalize.js.map +1 -1
- package/lib/types/wiki-materialize.d.ts.map +1 -1
- package/lib/types/wiki-materialize.js +3 -15
- package/lib/types/wiki-materialize.js.map +1 -1
- package/lib/types/wiki-trial-types.d.ts +3 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial.d.ts +2 -0
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +31 -7
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +6 -3
- package/preset/tita-data-agent/agent.cordis.yml +36 -129
- package/src/axis-registry.ts +57 -9
- package/src/client/AttributionEventGuide.tsx +10 -9
- package/src/client/CompileDialogs.tsx +10 -2
- package/src/client/EnvironmentSetup.tsx +76 -16
- package/src/client/ExpansionGraph.tsx +13 -7
- package/src/client/ExtractionTasksPage.tsx +543 -156
- package/src/client/ImportFilesDialog.tsx +2 -13
- package/src/client/IntentTab.tsx +3 -1
- package/src/client/KnowledgeBaseTab.tsx +139 -122
- package/src/client/KnowledgeLayerSection.tsx +19 -2
- package/src/client/KnowledgePanel.tsx +17 -3
- package/src/client/ListPager.tsx +15 -5
- package/src/client/MaterializeTrajectoryView.tsx +124 -0
- package/src/client/NodeSettings.tsx +90 -25
- package/src/client/NormalizeTrajectoryView.tsx +136 -0
- package/src/client/PipelineNotice.tsx +26 -0
- package/src/client/RecallPanel.tsx +55 -25
- package/src/client/TabGuide.tsx +4 -4
- package/src/client/TrialReportView.tsx +61 -16
- package/src/client/TypeDefinitionDrawer.tsx +33 -9
- package/src/client/WikiCardPreview.tsx +43 -10
- package/src/client/WikiFlowPanel.tsx +4 -4
- package/src/client/architecture.tsx +1 -1
- package/src/client/attribution/AttributionTab.tsx +37 -36
- package/src/client/attribution/CaseView.tsx +3 -1
- package/src/client/attribution/EventView.tsx +14 -4
- package/src/client/attribution/RunView.tsx +2 -2
- package/src/client/attribution/ViewSwitch.tsx +20 -5
- package/src/client/attribution/graph-options.ts +38 -31
- package/src/client/attribution/model.ts +16 -9
- package/src/client/graph/palette.ts +165 -0
- package/src/client/pages.css +224 -28
- package/src/client/pages.tsx +33 -11
- package/src/client/styles.css +61 -6
- package/src/client/wiki-trial.ts +30 -1
- package/src/convert/host.ts +3 -0
- package/src/convert/ingest.ts +73 -9
- package/src/diagnosis/cards.ts +13 -8
- package/src/diagnosis/run.ts +27 -17
- package/src/diagnosis/schema.ts +2 -2
- package/src/diagnosis/seed-source.ts +5 -4
- package/src/diagnosis/seed.ts +6 -8
- package/src/diagnosis/store.ts +99 -69
- package/src/diagnosis/types.ts +10 -16
- package/src/environment/dsh-python.ts +156 -0
- package/src/environment/install.ts +134 -19
- package/src/environment/model-catalog.ts +45 -3
- package/src/environment/service.ts +111 -23
- package/src/environment/types.ts +6 -0
- package/src/events.ts +12 -0
- package/src/extract-contract.ts +6 -3
- package/src/extract-model.ts +2 -0
- package/src/extract-table-contract.ts +29 -0
- package/src/extract-table.ts +126 -0
- package/src/extraction-runs.ts +70 -11
- package/src/fs-atomic.ts +19 -2
- package/src/images.ts +2 -2
- package/src/index.ts +207 -72
- package/src/intent/scope.ts +1 -0
- package/src/intent/tools.ts +38 -1
- package/src/knowledge-bases.ts +4 -12
- package/src/knowledge.ts +24 -4
- package/src/materialize-snapshot.ts +46 -0
- package/src/model-text.ts +5 -3
- package/src/oplog.ts +7 -0
- package/src/pipeline/engine.ts +312 -0
- package/src/pipeline/store.ts +147 -0
- package/src/pipeline/types.ts +92 -0
- package/src/present.ts +10 -4
- package/src/retention.ts +76 -0
- package/src/runtime.ts +853 -131
- package/src/source-evidence.ts +12 -5
- package/src/store/axis-references.ts +3 -3
- package/src/store/derived.ts +40 -7
- package/src/store/knowledge-db.ts +111 -5
- package/src/store/migrate.ts +1 -1
- package/src/trajectory-types.ts +142 -0
- package/src/types.ts +3 -4
- package/src/wiki-finalize.ts +28 -0
- package/src/wiki-materialize.ts +3 -14
- package/src/wiki-trial-types.ts +3 -0
- package/src/wiki-trial.ts +26 -7
- package/src/client/convert/ConvertPanel.tsx +0 -583
- package/src/client/convert/styles.css +0 -138
- package/src/client/convert/styles.ts +0 -49
package/src/runtime.ts
CHANGED
|
@@ -23,13 +23,19 @@ import { LEGACY_ENTITY_TYPES } from './diagnosis/schema.js'
|
|
|
23
23
|
import { DiagnosisStore } from './diagnosis/store.js'
|
|
24
24
|
import { MAX_LOG_ITEMS, OperationLog, operationId } from './oplog.js'
|
|
25
25
|
import { ExtractionRunStore, extractionRunId, type ExtractionRunPage, type ExtractionRunRecord, type ExtractionRunSummary } from './extraction-runs.js'
|
|
26
|
+
import type { MaterializeTrajectoryReport, NormalizeTrajectoryReport, TrajectoryStage } from './trajectory-types.js'
|
|
26
27
|
import { AxisRegistry, normalizeAxisName, type AxisEntity, type AxisGuidance, type AxisGuidanceKind, type AxisKind, type AxisRelation, type AxisResolution, type AxisStatus } from './axis-registry.js'
|
|
27
28
|
import { KNOWLEDGE_STATE_LABEL, KnowledgeStore, type Knowledge, type KnowledgeDocumentImpact, type KnowledgeDocumentRemoval, type KnowledgeFacets, type KnowledgeIngest, type KnowledgeIngestResult, type KnowledgeQuery, type KnowledgeState, type KnowledgeVersion } from './knowledge.js'
|
|
28
|
-
import { compositeEventIdentity, eventEntityFromRecords, type EventEntity, type EventSourceRecord } from './events.js'
|
|
29
|
-
import type { TrialAxis, TrialEvent, TrialKnowledge, TrialSource, TrialTokenUsage } from './wiki-trial-types.js'
|
|
29
|
+
import { compositeEventIdentity, eventEntityFromRecords, stripEventReferenceSection, type EventEntity, type EventSourceRecord } from './events.js'
|
|
30
|
+
import type { TrialAxis, TrialEvent, TrialKnowledge, TrialReport, TrialSource, TrialTokenUsage } from './wiki-trial-types.js'
|
|
30
31
|
import { buildAliasIndex, candidateEntityId, extractCandidates, normalizeCandidates, RELATES_TO, type NormalizedCandidates } from './extract.js'
|
|
31
32
|
import { skeletonBody, type ModelCall } from './extract-model.js'
|
|
32
|
-
import { applyTrialAxisResolution, normalizeKnowledge, runWikiTrial, trialSummary, type AxisPromptBlock, type TrialAxisSet, type TrialAxisSetResolution } from './wiki-trial.js'
|
|
33
|
+
import { applyTrialAxisResolution, cardNameFor, normalizeKnowledge, runWikiTrial, trialSummary, type AxisPromptBlock, type TrialAxisSet, type TrialAxisSetResolution } from './wiki-trial.js'
|
|
34
|
+
import { EXTRACTION_CONTRACT_VERSION } from './extract-table.js'
|
|
35
|
+
import { KnowledgePipeline, type PipelineContext, type PipelineSource } from './pipeline/engine.js'
|
|
36
|
+
import { PipelineStore, type ExtractionCheckpoint } from './pipeline/store.js'
|
|
37
|
+
import type { MaterializationGroup, PipelineStatus } from './pipeline/types.js'
|
|
38
|
+
import type { MaterializationSnapshot } from './materialize-snapshot.js'
|
|
33
39
|
import { candidatesFromReport, coordinateIdsOf, coordinateIsEmpty, type MaterializedCard } from './wiki-materialize.js'
|
|
34
40
|
import {
|
|
35
41
|
buildOrganizePrompt, buildOrganizeSystem, checkOrganize, citationsFrom, layoutHash, layoutOf,
|
|
@@ -40,7 +46,7 @@ import {
|
|
|
40
46
|
import { FTS_FILE, FtsIndex, loadSqlite, segment } from './fts.js'
|
|
41
47
|
import { attachImagesToKnowledge, insertImage, planSourceImages, type SourceImage } from './source-images.js'
|
|
42
48
|
import {
|
|
43
|
-
buildWikiIndex, cardKey, deadLinksOf, planRelated, relatedEntriesOf, stripRelatedSection,
|
|
49
|
+
buildWikiIndex, cardKey, deadLinksOf, planRelated, relatedByGroupKey, relatedEntriesOf, stripRelatedSection,
|
|
44
50
|
RELATED_METADATA_KEY, type DeadLink, type RelatedEntry,
|
|
45
51
|
} from './wiki-finalize.js'
|
|
46
52
|
import { applyRerank, applyThreshold, evidenceExcerpts, graphBoost, matchOf, noteIndexText, reviewWeight, rrfFuse, tokenSet, type RouteRanking } from './recall-rank.js'
|
|
@@ -256,6 +262,15 @@ const RRF_K = 60
|
|
|
256
262
|
const RRF_WEIGHTS = { coordinate: 0.9, relation: 0.45, fts: 0.3, vector: 0.7, graph: 0.2 } as const
|
|
257
263
|
/** 图谱只是 boost,不是路:它的权重只作用在**已经在候选池里**的条目上(见 `graphBoost`)。 */
|
|
258
264
|
const GRAPH_BOOST_WEIGHT = RRF_WEIGHTS.graph
|
|
265
|
+
/**
|
|
266
|
+
* 重排失败后的冷却时间。
|
|
267
|
+
*
|
|
268
|
+
* 失败要先记住,否则每次召回都会重新拉起一个注定失败的 worker(那是「秒级重试」的代价)。
|
|
269
|
+
* 但**不能永久记住**:一次超时是暂态,而写死之后 `rerankNotes` 会在开头直接返回,
|
|
270
|
+
* 于是这个标志自我封闭 —— 只有换模型、重载家当或人工重跑自检才能清掉。带上冷却,
|
|
271
|
+
* 第一次失败仍然如实报(有生产门时仍然明确失败),之后每 30 秒给一次自愈机会。
|
|
272
|
+
*/
|
|
273
|
+
const RERANK_RETRY_COOLDOWN_MS = 30_000
|
|
259
274
|
/**
|
|
260
275
|
* 编码器健康门的判据:两段无关文本的余弦必须低于它。
|
|
261
276
|
*
|
|
@@ -657,6 +672,8 @@ export interface ImportFailure {
|
|
|
657
672
|
* 而这里要回答的是「刚才那一下让库里多了什么」。
|
|
658
673
|
*/
|
|
659
674
|
export interface ImportCompileReport {
|
|
675
|
+
taskId?: string
|
|
676
|
+
state?: import('./pipeline/types.js').PipelineState
|
|
660
677
|
cardsCreated: number
|
|
661
678
|
cardsUpdated: number
|
|
662
679
|
cardsReused?: number
|
|
@@ -1046,38 +1063,6 @@ function appendExtracted(existing: string, additions: string[]): string {
|
|
|
1046
1063
|
return content
|
|
1047
1064
|
}
|
|
1048
1065
|
|
|
1049
|
-
/** 坐标 WikiCard 的事件摘要段由 Event 引用派生;全文与时间线留在独立 Event 实体。 */
|
|
1050
|
-
function withEventReferences(
|
|
1051
|
-
content: string,
|
|
1052
|
-
rawRefs: unknown,
|
|
1053
|
-
eventsById: ReadonlyMap<string, EventEntity>,
|
|
1054
|
-
): string {
|
|
1055
|
-
let refs: string[] = []
|
|
1056
|
-
if (Array.isArray(rawRefs)) refs = rawRefs.filter((item): item is string => typeof item === 'string')
|
|
1057
|
-
else if (typeof rawRefs === 'string') {
|
|
1058
|
-
try {
|
|
1059
|
-
const parsed: unknown = JSON.parse(rawRefs)
|
|
1060
|
-
if (Array.isArray(parsed)) refs = parsed.filter((item): item is string => typeof item === 'string')
|
|
1061
|
-
} catch { /* 无效引用会作为空列表处理 */ }
|
|
1062
|
-
}
|
|
1063
|
-
const lines = content.split('\n')
|
|
1064
|
-
const start = lines.findIndex(line => line.trim() === '## 关联 Event')
|
|
1065
|
-
if (start >= 0) {
|
|
1066
|
-
let end = start + 1
|
|
1067
|
-
while (end < lines.length && !/^#{1,2}\s+/.test(lines[end] ?? '')) end += 1
|
|
1068
|
-
lines.splice(start, end - start)
|
|
1069
|
-
}
|
|
1070
|
-
const base = lines.join('\n').trimEnd()
|
|
1071
|
-
const entries = [...new Set(refs)].map(id => eventsById.get(id)).filter((event): event is EventEntity => event !== undefined)
|
|
1072
|
-
if (entries.length === 0) return base
|
|
1073
|
-
const bullets = entries.map(event => {
|
|
1074
|
-
const occurredAt = event.occurredAt ? ` · ${event.occurredAt}` : ''
|
|
1075
|
-
const summary = event.summary.trim() ? `:${event.summary.trim()}` : ''
|
|
1076
|
-
return `- **${event.title}**${occurredAt}${summary}(${event.id})`
|
|
1077
|
-
})
|
|
1078
|
-
return `${base ? `${base}\n\n` : ''}## 关联 Event\n\n${bullets.join('\n')}`
|
|
1079
|
-
}
|
|
1080
|
-
|
|
1081
1066
|
/** 把 `incoming` 按小节折进 `existing`;结构对不上时整段追加。 */
|
|
1082
1067
|
function foldBody(existing: string, incoming: string): string {
|
|
1083
1068
|
if (!existing) return incoming
|
|
@@ -1355,9 +1340,10 @@ function markdownImageRefs(content: string): string[] {
|
|
|
1355
1340
|
* `tags` / `generated` / `sources`。它们是真 YAML 键,人能读、`grep` 得到、
|
|
1356
1341
|
* 外部工具能按公开规范读走。`generated.at` 就是**采集时间**(OKF 里没有「抓取
|
|
1357
1342
|
* 时间」这个概念,本文件何时被谁写出来就是它)。
|
|
1358
|
-
* 2. **我们自己的运行键**:`
|
|
1359
|
-
* `
|
|
1360
|
-
*
|
|
1343
|
+
* 2. **我们自己的运行键**:`created_at` / `updated_at` / `metadata` / `lineage`。
|
|
1344
|
+
* 来源身份保存在 SQLite 的 `entity_id` / `external_id` 列,不在 front matter 重复一份。
|
|
1345
|
+
* OKF 明确允许自定义键,且要求消费者不得因为未知键拒收,所以运行元数据可以同处
|
|
1346
|
+
* 一个 front matter。
|
|
1361
1347
|
*
|
|
1362
1348
|
* `metadata` 只剩**类型专属的扩展键**(转换任务 id、内容哈希、资产清单、格式…):
|
|
1363
1349
|
* 那些东西 OKF 没有对应概念,但也不该混进上面那组公开字段里。
|
|
@@ -1386,8 +1372,6 @@ function sourceFrontMatter(record: SourceRaw, provenance?: SourceProvenance): Re
|
|
|
1386
1372
|
...(origin.lastModified?.trim() ? { last_modified: origin.lastModified.trim() } : {}),
|
|
1387
1373
|
}))
|
|
1388
1374
|
}
|
|
1389
|
-
frontMatter.source_id = record.sourceId
|
|
1390
|
-
frontMatter.external_id = record.externalId ?? ''
|
|
1391
1375
|
frontMatter.created_at = record.createdAt
|
|
1392
1376
|
frontMatter.updated_at = record.updatedAt
|
|
1393
1377
|
frontMatter.metadata = record.metadata
|
|
@@ -1776,7 +1760,7 @@ function resolveConfiguredPython(configured: string | undefined, resourcePython:
|
|
|
1776
1760
|
|
|
1777
1761
|
export interface KnowledgeRuntimeOptions {
|
|
1778
1762
|
/** Production deployment requires a working Python + embedding + rerank stack. */
|
|
1779
|
-
environmentGuard?: () => Promise<void>
|
|
1763
|
+
environmentGuard?: (kind?: 'embedding' | 'rerank') => Promise<void>
|
|
1780
1764
|
/**
|
|
1781
1765
|
* 这一库的「坐标提取备注」(0068),人写的。
|
|
1782
1766
|
*
|
|
@@ -2000,7 +1984,7 @@ function blankProgress(
|
|
|
2000
1984
|
}
|
|
2001
1985
|
|
|
2002
1986
|
export class KnowledgeRuntime {
|
|
2003
|
-
private readonly environmentGuard?: () => Promise<void>
|
|
1987
|
+
private readonly environmentGuard?: (kind?: 'embedding' | 'rerank') => Promise<void>
|
|
2004
1988
|
/** 当前生效的配置。volatile 改动会就地合并进来(见 `updateConfig`),所以它不是只读的。 */
|
|
2005
1989
|
config: RuntimeKnowledgeConfig
|
|
2006
1990
|
readonly rootDir: string
|
|
@@ -2055,6 +2039,9 @@ export class KnowledgeRuntime {
|
|
|
2055
2039
|
/** 抽取轨迹(`<memoryDir>/extraction-runs.sqlite`)。同一个懒打开的理由。 */
|
|
2056
2040
|
private extractionRunsStore: ExtractionRunStore | null = null
|
|
2057
2041
|
private extractionRunsReady: boolean | null = null
|
|
2042
|
+
private closed = false
|
|
2043
|
+
private pipelineEngine?: KnowledgePipeline
|
|
2044
|
+
private pipelineOpening?: Promise<KnowledgePipeline>
|
|
2058
2045
|
/** 轴注册表(`<memoryDir>/knowledge.sqlite`)。与抽取轨迹同一个懒打开的理由。 */
|
|
2059
2046
|
private axesStore: AxisRegistry | null = null
|
|
2060
2047
|
private axesReady: boolean | null = null
|
|
@@ -2134,8 +2121,8 @@ export class KnowledgeRuntime {
|
|
|
2134
2121
|
private embeddingWorkerFingerprint?: string
|
|
2135
2122
|
/** Separate process so a rerank model failure cannot take embeddings down. */
|
|
2136
2123
|
private rerankWorker: JsonLineWorker | null = null
|
|
2137
|
-
/** Cache the
|
|
2138
|
-
private rerankUnavailable: string | undefined
|
|
2124
|
+
/** Cache the last rerank failure with its time; see `RERANK_RETRY_COOLDOWN_MS`. */
|
|
2125
|
+
private rerankUnavailable: { error: string; at: number } | undefined
|
|
2139
2126
|
/** 解析后的 Python 解释器路径(配置为空时从家当目录解析)。 */
|
|
2140
2127
|
private pythonPath: string
|
|
2141
2128
|
/** 解析后的嵌入模型:本地目录或 HF 模型名。 */
|
|
@@ -2803,12 +2790,20 @@ export class KnowledgeRuntime {
|
|
|
2803
2790
|
}
|
|
2804
2791
|
|
|
2805
2792
|
close(): void {
|
|
2793
|
+
if (this.closed) return
|
|
2794
|
+
this.closed = true
|
|
2795
|
+
this.pipelineEngine?.close()
|
|
2806
2796
|
for (const task of this.compileTasks.values()) {
|
|
2807
2797
|
if (['succeeded', 'failed', 'cancelled'].includes(task.progress.state)) continue
|
|
2808
2798
|
task.cancelRequested = true
|
|
2809
2799
|
task.pauseRequested = false
|
|
2810
2800
|
task.resume?.()
|
|
2811
2801
|
}
|
|
2802
|
+
if (this.compilePending > 0) { void this.compileTail.finally(() => this.closeStores()); return }
|
|
2803
|
+
this.closeStores()
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
private closeStores(): void {
|
|
2812
2807
|
this.embeddingWorker?.close()
|
|
2813
2808
|
this.embeddingWorker = null
|
|
2814
2809
|
this.rerankWorker?.close()
|
|
@@ -3198,34 +3193,23 @@ export class KnowledgeRuntime {
|
|
|
3198
3193
|
failed[0] ? `第一份的原因:${failed[0].message}` : undefined)
|
|
3199
3194
|
}
|
|
3200
3195
|
|
|
3201
|
-
/**
|
|
3202
|
-
* 编译。**只要有一份写成就编**:成功的那几份已经是库里的知识,
|
|
3203
|
-
* 「因为同批里有一份失败就不编」等于把已经进来的知识又挡在门外一次。
|
|
3204
|
-
*/
|
|
3205
3196
|
let compile: ImportCompileReport | null = null
|
|
3206
|
-
if (input.compile
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
warnings.push(`来源已写入,但编译失败:${rebuilt.error?.message ?? '未知原因'}`)
|
|
3213
|
-
} else {
|
|
3214
|
-
const data = rebuilt.data
|
|
3197
|
+
if (input.compile === true) {
|
|
3198
|
+
try {
|
|
3199
|
+
const before = new Map((await this.loadCards()).map(card => [card.entityId, this.cardHashOf(card)]))
|
|
3200
|
+
const pipeline = await this.pipeline()
|
|
3201
|
+
const task = await pipeline.start('all', { sourceIds: written.map(source => source.sourceId) }).completion
|
|
3202
|
+
const after = await this.loadCards()
|
|
3215
3203
|
compile = {
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
sourcesExtracted: data.sourcesExtracted,
|
|
3222
|
-
sourcesSkipped: data.sourcesSkipped,
|
|
3223
|
-
cardsRemoved: data.cardsRemoved,
|
|
3224
|
-
embedding: data.embedding,
|
|
3225
|
-
version: data.version,
|
|
3204
|
+
taskId: task.id, state: task.state,
|
|
3205
|
+
cardsCreated: after.filter(card => !before.has(card.entityId)).length,
|
|
3206
|
+
cardsUpdated: after.filter(card => before.has(card.entityId) && before.get(card.entityId) !== this.cardHashOf(card)).length,
|
|
3207
|
+
sourcesExtracted: Math.max(0, (task.results?.extract?.done ?? 0) - (task.results?.extract?.reused ?? 0) - (task.results?.extract?.failed ?? 0)),
|
|
3208
|
+
sourcesSkipped: task.results?.extract?.reused ?? 0,
|
|
3226
3209
|
}
|
|
3227
|
-
warnings.push(...
|
|
3228
|
-
|
|
3210
|
+
warnings.push(...task.warnings)
|
|
3211
|
+
if (task.state !== 'succeeded') warnings.push(`来源已保存;全链路未完成:${task.summary}。可在知识处理页面继续。`)
|
|
3212
|
+
} catch (cause) { warnings.push(`来源已保存;知识处理未完成:${cause instanceof Error ? cause.message : String(cause)}`) }
|
|
3229
3213
|
}
|
|
3230
3214
|
|
|
3231
3215
|
return success({ written, failed, attempted: items.length, compile }, warnings)
|
|
@@ -3509,6 +3493,16 @@ export class KnowledgeRuntime {
|
|
|
3509
3493
|
* 失败一律返回 `null`,**不抛** —— 「知识库不可用」是一条要显示的结论,不是一个异常。
|
|
3510
3494
|
*/
|
|
3511
3495
|
private storeHandleOpening?: Promise<KnowledgeDb | null>
|
|
3496
|
+
/**
|
|
3497
|
+
* 知识存储现在可用吗(0070)。
|
|
3498
|
+
*
|
|
3499
|
+
* 读路径需要它把「确认是空」与「读不到」分开:`loadCards()` 在存储不可用时返回 `[]`,
|
|
3500
|
+
* 而调用方据此做写决定(例如诊断的旧格式迁移)就会把一次读故障变成一次污染写。
|
|
3501
|
+
*/
|
|
3502
|
+
async knowledgeStoreAvailable(): Promise<boolean> {
|
|
3503
|
+
return (await this.storeHandle()) !== null
|
|
3504
|
+
}
|
|
3505
|
+
|
|
3512
3506
|
private async storeHandle(): Promise<KnowledgeDb | null> {
|
|
3513
3507
|
if (this.store !== undefined) return this.store
|
|
3514
3508
|
this.storeHandleOpening ??= this.openStoreHandle()
|
|
@@ -4329,6 +4323,7 @@ export class KnowledgeRuntime {
|
|
|
4329
4323
|
* 提示词与真编译发出去的那一份必须是**同一份**,否则预览这件事本身没有意义。
|
|
4330
4324
|
*/
|
|
4331
4325
|
private async organizeContext(input: {
|
|
4326
|
+
knowledge?: Knowledge[]
|
|
4332
4327
|
cardTitle: string
|
|
4333
4328
|
coordinate: CardCoordinate
|
|
4334
4329
|
profileId: string
|
|
@@ -4353,10 +4348,11 @@ export class KnowledgeRuntime {
|
|
|
4353
4348
|
const store = await this.knowledgeStore()
|
|
4354
4349
|
if (store === null) return { ok: false, reason: '知识库不可用,这次没有组织卡片(正文退回确定性布局)' }
|
|
4355
4350
|
|
|
4356
|
-
const listed = await store.list({
|
|
4351
|
+
const listed = input.knowledge ? { knowledge: input.knowledge, total: input.knowledge.length } : await store.list({
|
|
4357
4352
|
objectId: input.coordinate.objectId,
|
|
4358
4353
|
actionId: input.coordinate.actionId,
|
|
4359
4354
|
sceneId: input.coordinate.sceneId,
|
|
4355
|
+
excludeWithdrawn: true,
|
|
4360
4356
|
limit: MAX_ORGANIZE_KNOWLEDGE,
|
|
4361
4357
|
})
|
|
4362
4358
|
if (listed.knowledge.length === 0) {
|
|
@@ -4540,7 +4536,8 @@ export class KnowledgeRuntime {
|
|
|
4540
4536
|
if (call === undefined) return failure({ card, warnings: [] }, 'MODEL_UNAVAILABLE', '当前没有可用的模型,无法重新组织这张卡片', true, '检查模型配置后再试。')
|
|
4541
4537
|
|
|
4542
4538
|
const profile = this.organizeProfile()
|
|
4543
|
-
const
|
|
4539
|
+
const organizationCard = { ...card, content: stripEventReferenceSection(card.content) }
|
|
4540
|
+
const related = await this.relatedForCard(organizationCard)
|
|
4544
4541
|
const images = await this.imagesForCard(card)
|
|
4545
4542
|
const organized = await this.organizeCard({
|
|
4546
4543
|
cardTitle: card.title,
|
|
@@ -4551,7 +4548,7 @@ export class KnowledgeRuntime {
|
|
|
4551
4548
|
...(profile === undefined ? {} : { requiredSections: profile.sections.filter(section => section.required).map(section => section.label) }),
|
|
4552
4549
|
...(related.related === undefined ? {} : { related: related.related }),
|
|
4553
4550
|
...(images.images.length === 0 ? {} : { images: images.images }),
|
|
4554
|
-
existing:
|
|
4551
|
+
existing: organizationCard,
|
|
4555
4552
|
call,
|
|
4556
4553
|
})
|
|
4557
4554
|
if (organized.kind !== 'applied') {
|
|
@@ -4563,7 +4560,7 @@ export class KnowledgeRuntime {
|
|
|
4563
4560
|
entityType: card.entityType,
|
|
4564
4561
|
entityId: card.entityId,
|
|
4565
4562
|
title: organized.title,
|
|
4566
|
-
content: organized.content,
|
|
4563
|
+
content: stripEventReferenceSection(organized.content),
|
|
4567
4564
|
sourceRefs: card.sourceRefs,
|
|
4568
4565
|
summary: organized.summary,
|
|
4569
4566
|
status: card.status,
|
|
@@ -4645,6 +4642,7 @@ export class KnowledgeRuntime {
|
|
|
4645
4642
|
}
|
|
4646
4643
|
|
|
4647
4644
|
private async organizeCard(input: {
|
|
4645
|
+
knowledge?: Knowledge[]
|
|
4648
4646
|
cardTitle: string
|
|
4649
4647
|
coordinate: CardCoordinate
|
|
4650
4648
|
profileId: string
|
|
@@ -4670,6 +4668,7 @@ export class KnowledgeRuntime {
|
|
|
4670
4668
|
| { kind: 'skipped'; warnings: string[] }
|
|
4671
4669
|
> {
|
|
4672
4670
|
const context = await this.organizeContext({
|
|
4671
|
+
knowledge: input.knowledge,
|
|
4673
4672
|
cardTitle: input.cardTitle,
|
|
4674
4673
|
coordinate: input.coordinate,
|
|
4675
4674
|
profileId: input.profileId,
|
|
@@ -5019,7 +5018,16 @@ export class KnowledgeRuntime {
|
|
|
5019
5018
|
}
|
|
5020
5019
|
|
|
5021
5020
|
private compileCardNotes(card: KnowledgeCard): WikiNote[] {
|
|
5022
|
-
|
|
5021
|
+
/**
|
|
5022
|
+
* **只排除 `deprecated`,不排除 `problem`。**
|
|
5023
|
+
*
|
|
5024
|
+
* `problem` 是人工标注的「这张卡有问题」,它仍然是知识,而召回契约里的
|
|
5025
|
+
* `include_problems` 是一个**读取时**的开关(见 `recall` 里 eligible 的那一行)。
|
|
5026
|
+
* 在编译期就把它的 note 丢掉,等于把这个开关变成死参数:无论传什么,编译产物
|
|
5027
|
+
* 里根本没有它的 note,FTS / 向量索引里也没有。所以索引层保留,过滤留给读路径。
|
|
5028
|
+
* `deprecated` 不同 —— 它是退役,编译与读取两层都该排除。
|
|
5029
|
+
*/
|
|
5030
|
+
if (card.status === 'deprecated') return []
|
|
5023
5031
|
const notes: WikiNote[] = []
|
|
5024
5032
|
/**
|
|
5025
5033
|
* 卡片文件所在目录:正文里的图片引用是相对它写的(`../../01_source_raw/_assets/…`),
|
|
@@ -5036,7 +5044,7 @@ export class KnowledgeRuntime {
|
|
|
5036
5044
|
* `metadata.related_cards`;这里继续剥离旧版本正文里的标记,保证存量卡片迁移期间
|
|
5037
5045
|
* 也不会把导航送进召回。
|
|
5038
5046
|
*/
|
|
5039
|
-
const recallBody = stripRelatedSection(card.content)
|
|
5047
|
+
const recallBody = stripEventReferenceSection(stripRelatedSection(card.content))
|
|
5040
5048
|
for (const [index, chunk] of splitWikiNotes({ ...card, content: recallBody }).entries()) {
|
|
5041
5049
|
const baseId = `${card.entityType}:${card.entityId}`
|
|
5042
5050
|
const unitId = index === 0 ? baseId : `${baseId}#${index}`
|
|
@@ -5070,10 +5078,17 @@ export class KnowledgeRuntime {
|
|
|
5070
5078
|
/** 1-hop neighbourhood of a set of wiki cards inside the compiled graph. */
|
|
5071
5079
|
private oneHopGraph(keys: Set<string>, includeProblem: boolean, graph: ContextGraph = this.graph): ContextGraph {
|
|
5072
5080
|
const visible = (node: Record<string, unknown>) => includeProblem || node.status !== 'problem'
|
|
5073
|
-
|
|
5081
|
+
/**
|
|
5082
|
+
* 必须读**传进来的** `graph`,不能读 `this.graph`。
|
|
5083
|
+
*
|
|
5084
|
+
* `this.graph` 是编译产物:卡片被删掉之后它不会跟着变小。调用方传的是
|
|
5085
|
+
* `visibleGraph(snapshot.graph, liveGraphKeys(truth))` —— 已按存活实体过滤过的那一份。
|
|
5086
|
+
* 用 `this.graph` 等于把删除掉的卡片留在召回结果的图里。
|
|
5087
|
+
*/
|
|
5088
|
+
const edges = graph.edges.filter(edge => keys.has(String(edge.from)) || keys.has(String(edge.to))).slice(0, 50)
|
|
5074
5089
|
const ids = new Set(keys)
|
|
5075
5090
|
for (const edge of edges) { ids.add(String(edge.from)); ids.add(String(edge.to)) }
|
|
5076
|
-
const nodes =
|
|
5091
|
+
const nodes = graph.nodes
|
|
5077
5092
|
.filter(visible)
|
|
5078
5093
|
.filter(node => ids.has(`${node.type}:${node.id}`))
|
|
5079
5094
|
.slice(0, 50)
|
|
@@ -5100,8 +5115,9 @@ export class KnowledgeRuntime {
|
|
|
5100
5115
|
private async rerankNotes(query: string, hits: RecallHit[], threshold: number): Promise<{ hits: RecallHit[]; result: RerankResult; scored?: Array<{ unitId: string; logit: number; rerankScore: number; kept: boolean }>; warning?: string }> {
|
|
5101
5116
|
const provider = this.config.rerankProvider
|
|
5102
5117
|
if (provider === 'none') return { hits, result: { status: 'skipped', provider, threshold } }
|
|
5103
|
-
if (hits.length
|
|
5104
|
-
|
|
5118
|
+
if (hits.length === 0) return { hits, result: { status: 'skipped', provider, model: this.rerankModelPath, scored: hits.length, threshold } }
|
|
5119
|
+
const cooldownActive = this.rerankUnavailable !== undefined && Date.now() - this.rerankUnavailable.at < RERANK_RETRY_COOLDOWN_MS
|
|
5120
|
+
if (this.rerankUnavailable && cooldownActive) return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error: this.rerankUnavailable.error, threshold }, warning: `重排不可用,已按融合分数排序:${this.rerankUnavailable.error}` }
|
|
5105
5121
|
try {
|
|
5106
5122
|
if (!this.rerankWorker) this.rerankWorker = this.spawnWorker('RERANK')
|
|
5107
5123
|
const response = await this.rerankWorker.request({
|
|
@@ -5114,6 +5130,8 @@ export class KnowledgeRuntime {
|
|
|
5114
5130
|
const raw: number[] = Array.isArray(response?.scores) ? (response.scores as unknown[]).map(Number) : []
|
|
5115
5131
|
const outcome = applyRerank(raw, hits, threshold)
|
|
5116
5132
|
const vocabSize = Number(response?.vocabSize)
|
|
5133
|
+
// 一次成功的推理就是这条缓存的反证:清掉它,别让暂态失败变成会话级残疾。
|
|
5134
|
+
this.rerankUnavailable = undefined
|
|
5117
5135
|
return {
|
|
5118
5136
|
hits: outcome.kept,
|
|
5119
5137
|
scored: outcome.scored,
|
|
@@ -5130,7 +5148,7 @@ export class KnowledgeRuntime {
|
|
|
5130
5148
|
// so a later recall does not respawn a doomed model every time.
|
|
5131
5149
|
this.rerankWorker?.close()
|
|
5132
5150
|
this.rerankWorker = null
|
|
5133
|
-
this.rerankUnavailable = error
|
|
5151
|
+
this.rerankUnavailable = { error, at: Date.now() }
|
|
5134
5152
|
return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error, threshold }, warning: `重排不可用,已按融合分数排序:${error}` }
|
|
5135
5153
|
}
|
|
5136
5154
|
}
|
|
@@ -5345,7 +5363,6 @@ export class KnowledgeRuntime {
|
|
|
5345
5363
|
*/
|
|
5346
5364
|
const profile = this.organizeProfile()
|
|
5347
5365
|
const profileId = profile?.id ?? 'concept'
|
|
5348
|
-
const eventsById = new Map((await this.loadEvents()).map(event => [event.id, event]))
|
|
5349
5366
|
for (const item of pending.values()) {
|
|
5350
5367
|
if (runId) await this.checkpointCompileTask(runId)
|
|
5351
5368
|
const key = `${item.entityType}:${item.entityId}`
|
|
@@ -5394,7 +5411,7 @@ export class KnowledgeRuntime {
|
|
|
5394
5411
|
&& layoutOf(existing.metadata)?.origin === 'model'
|
|
5395
5412
|
const finalTitle = preserveModelLayout ? existing.title : item.title
|
|
5396
5413
|
const contentBeforeEvents = preserveModelLayout ? existing.content : content
|
|
5397
|
-
const finalContent =
|
|
5414
|
+
const finalContent = stripEventReferenceSection(contentBeforeEvents)
|
|
5398
5415
|
const finalSummary = preserveModelLayout ? existing.summary : item.summary
|
|
5399
5416
|
let finalPlacements = preserveModelLayout || item.placements === undefined
|
|
5400
5417
|
? undefined
|
|
@@ -5478,7 +5495,7 @@ export class KnowledgeRuntime {
|
|
|
5478
5495
|
outcome.cardsUpdated += 1
|
|
5479
5496
|
for (const warning of written.warnings ?? []) outcome.warnings.push(`卡片 ${card.entityId}:${warning}`)
|
|
5480
5497
|
}
|
|
5481
|
-
// WikiCard
|
|
5498
|
+
// WikiCard 只由 Knowledge 物化;Event 作为独立事实按坐标挂引用。
|
|
5482
5499
|
const eventSync = await this.syncEventReferences()
|
|
5483
5500
|
outcome.cardsUpdated += eventSync.updated
|
|
5484
5501
|
outcome.warnings.push(...eventSync.warnings)
|
|
@@ -5540,12 +5557,13 @@ export class KnowledgeRuntime {
|
|
|
5540
5557
|
* 用的输入与收尾阶段那个相关块**完全相同**(同一份 `relations` + 同一个坐标投影),
|
|
5541
5558
|
* 所以「正文里能提到谁」与「末尾相关块列了谁」不会是两个答案。
|
|
5542
5559
|
*/
|
|
5560
|
+
const cardsWithoutLegacyEventSections = input.cards.map(card => ({ ...card, content: stripEventReferenceSection(card.content) }))
|
|
5543
5561
|
const relatedPlan = planRelated(
|
|
5544
|
-
|
|
5562
|
+
cardsWithoutLegacyEventSections,
|
|
5545
5563
|
input.relations,
|
|
5546
5564
|
await this.projectedRelated(input.cards),
|
|
5547
5565
|
)
|
|
5548
|
-
const withCoordinate =
|
|
5566
|
+
const withCoordinate = cardsWithoutLegacyEventSections
|
|
5549
5567
|
.filter(card => card.coordinate !== undefined)
|
|
5550
5568
|
.sort((left, right) => left.entityId.localeCompare(right.entityId))
|
|
5551
5569
|
if (withCoordinate.length === 0) return 0
|
|
@@ -5865,7 +5883,8 @@ export class KnowledgeRuntime {
|
|
|
5865
5883
|
outcome.warnings.push(`Event 没有写入知识库:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
5866
5884
|
}
|
|
5867
5885
|
outcome.warnings.push(...eventIngested.warnings)
|
|
5868
|
-
|
|
5886
|
+
outcome.warnings.push(...ingested.warnings)
|
|
5887
|
+
if (ingested.skipped > 0 || ingested.failures > 0 || eventIngested.skipped > 0) extractionComplete = false
|
|
5869
5888
|
// 落位要用库内的 id。这一份报告里每一条知识的映射都并进整轮那一份 ——
|
|
5870
5889
|
// 后面的来源可能命中同一张卡,那时落位指的行也得是活的。
|
|
5871
5890
|
Object.assign(knowledgeIds, ingested.ids)
|
|
@@ -6170,7 +6189,7 @@ export class KnowledgeRuntime {
|
|
|
6170
6189
|
const projected = await this.projectedRelated(cards)
|
|
6171
6190
|
const related = planRelated(cards, relations, projected)
|
|
6172
6191
|
/** 统计清理旧导航块之后的正文断链;相关卡片本身不再伪装成正文链接。 */
|
|
6173
|
-
const nextCards = cards.map(card => ({ ...card, content: stripRelatedSection(card.content) }))
|
|
6192
|
+
const nextCards = cards.map(card => ({ ...card, content: stripEventReferenceSection(stripRelatedSection(card.content)) }))
|
|
6174
6193
|
const deadLinks = deadLinksOf(nextCards)
|
|
6175
6194
|
let rewritten = 0
|
|
6176
6195
|
for (const [index, card] of cards.entries()) {
|
|
@@ -6747,6 +6766,8 @@ export class KnowledgeRuntime {
|
|
|
6747
6766
|
summary: string
|
|
6748
6767
|
report: unknown
|
|
6749
6768
|
actor?: string
|
|
6769
|
+
/** 这条轨迹属于哪一段。省略 = 抽取(旧调用点的行为不变)。 */
|
|
6770
|
+
stage?: TrajectoryStage
|
|
6750
6771
|
/**
|
|
6751
6772
|
* 这次轨迹的 id。**调用方可以先定**,因为注册表里的候选实体要记住「是哪一次发现的」,
|
|
6752
6773
|
* 而那个 id 在报告组装之前就得有。不给就现生成一个。
|
|
@@ -6757,7 +6778,7 @@ export class KnowledgeRuntime {
|
|
|
6757
6778
|
const store = await this.extractionRuns()
|
|
6758
6779
|
if (store === null) return { saved: false, why: '抽取轨迹库不可用(需要 Node 22.5+ 的 node:sqlite)' }
|
|
6759
6780
|
const report = input.report as { model?: { ok?: boolean; why?: string } } | null
|
|
6760
|
-
const id = input.id?.trim() || extractionRunId()
|
|
6781
|
+
const id = input.id?.trim() || extractionRunId(input.stage)
|
|
6761
6782
|
const record: ExtractionRunRecord = {
|
|
6762
6783
|
id,
|
|
6763
6784
|
kb: this.rootDir,
|
|
@@ -6769,6 +6790,7 @@ export class KnowledgeRuntime {
|
|
|
6769
6790
|
...(report?.model?.why ? { modelWhy: report.model.why } : {}),
|
|
6770
6791
|
summary: input.summary,
|
|
6771
6792
|
hasReport: true,
|
|
6793
|
+
stage: input.stage ?? 'extract',
|
|
6772
6794
|
report: input.report,
|
|
6773
6795
|
}
|
|
6774
6796
|
try {
|
|
@@ -6779,6 +6801,154 @@ export class KnowledgeRuntime {
|
|
|
6779
6801
|
}
|
|
6780
6802
|
}
|
|
6781
6803
|
|
|
6804
|
+
/**
|
|
6805
|
+
* 归一段的轨迹(2026-10-18)。
|
|
6806
|
+
*
|
|
6807
|
+
* 这一段的产物以前只留在两处:任务里的流式输出(刷新就没),和事实表里最终写成的行
|
|
6808
|
+
* (只剩结果)。中间那步 —— 「模型给的这 283 个名字里,哪些命中了已有实体、哪些新建了
|
|
6809
|
+
* 候选、哪些因为不唯一没猜」—— 跑完就查不到了,而它恰好是坐标错了之后唯一能复盘的东西。
|
|
6810
|
+
*
|
|
6811
|
+
* **复用的那一次不落轨迹**:那一次什么都没做(指纹没变、一条事实都没重写),落一条
|
|
6812
|
+
* 「复用」只会把真正的运行挤出第一页。
|
|
6813
|
+
*/
|
|
6814
|
+
private async recordNormalizeTrajectory(input: {
|
|
6815
|
+
source: PipelineSource
|
|
6816
|
+
checkpoint: ExtractionCheckpoint
|
|
6817
|
+
report: TrialReport
|
|
6818
|
+
knowledge: { saved: number; merged: number; skipped: number; failures?: number; warnings: string[] }
|
|
6819
|
+
events: { saved: number; merged: number; skipped: number; warnings: string[] }
|
|
6820
|
+
unresolvedKnowledgeIds: string[]
|
|
6821
|
+
fingerprint: string
|
|
6822
|
+
warnings: string[]
|
|
6823
|
+
context: PipelineContext
|
|
6824
|
+
}): Promise<void> {
|
|
6825
|
+
const report = input.report
|
|
6826
|
+
const axes: NormalizeTrajectoryReport['axes'] = ([['object', report.objects], ['action', report.actions], ['scene', report.scenes]] as const).flatMap(([kind, rows]) => rows.map(axis => ({
|
|
6827
|
+
kind,
|
|
6828
|
+
name: axis.name,
|
|
6829
|
+
outcome: axis.outcome,
|
|
6830
|
+
...(axis.id ? { id: axis.id } : {}),
|
|
6831
|
+
...(axis.how ? { how: axis.how } : {}),
|
|
6832
|
+
aliases: axis.aliases,
|
|
6833
|
+
...(axis.parentMention ? { parentMention: axis.parentMention } : {}),
|
|
6834
|
+
})))
|
|
6835
|
+
const count = (outcome: string): number => axes.filter(row => row.outcome === outcome).length
|
|
6836
|
+
const unresolved = new Set(input.unresolvedKnowledgeIds)
|
|
6837
|
+
const payload: NormalizeTrajectoryReport = {
|
|
6838
|
+
stage: 'normalize',
|
|
6839
|
+
at: new Date().toISOString(),
|
|
6840
|
+
source: { id: input.source.sourceId, title: input.source.title, version: input.source.version },
|
|
6841
|
+
extractionId: input.checkpoint.id,
|
|
6842
|
+
fingerprint: input.fingerprint,
|
|
6843
|
+
mentions: report.objects.length + report.actions.length + report.scenes.length,
|
|
6844
|
+
counts: { matched: count('matched'), created: count('new'), ambiguous: count('ambiguous'), mechanical: count('no-registry') },
|
|
6845
|
+
axes: axes.slice(0, 400),
|
|
6846
|
+
knowledge: {
|
|
6847
|
+
saved: input.knowledge.saved,
|
|
6848
|
+
merged: input.knowledge.merged,
|
|
6849
|
+
skipped: input.knowledge.skipped,
|
|
6850
|
+
failures: input.knowledge.failures ?? 0,
|
|
6851
|
+
unresolved: unresolved.size,
|
|
6852
|
+
},
|
|
6853
|
+
events: { saved: input.events.saved, merged: input.events.merged, skipped: input.events.skipped },
|
|
6854
|
+
rows: report.knowledge.slice(0, 300).map(item => ({
|
|
6855
|
+
text: item.text,
|
|
6856
|
+
coordinate: [item.object, item.action, item.scene].filter(Boolean).join(' · ') || '无坐标',
|
|
6857
|
+
sources: item.sources.length,
|
|
6858
|
+
unresolved: unresolved.has(item.id),
|
|
6859
|
+
})),
|
|
6860
|
+
model: {
|
|
6861
|
+
ok: report.model.parsed === true,
|
|
6862
|
+
...(report.model.why ? { why: report.model.why } : {}),
|
|
6863
|
+
provider: report.model.request?.provider ?? input.context.model.provider,
|
|
6864
|
+
model: report.model.request?.model ?? input.context.model.model,
|
|
6865
|
+
reasoningEffort: report.model.request?.reasoningEffort ?? input.context.model.reasoningEffort,
|
|
6866
|
+
},
|
|
6867
|
+
...(report.axesContext ? { axesContext: report.axesContext } : {}),
|
|
6868
|
+
warnings: input.warnings,
|
|
6869
|
+
}
|
|
6870
|
+
const saved = await this.recordExtractionRun({
|
|
6871
|
+
stage: 'normalize',
|
|
6872
|
+
docId: input.source.sourceId,
|
|
6873
|
+
title: `${input.source.title} · 归一`,
|
|
6874
|
+
summary: `命中 ${payload.counts.matched} · 新建候选 ${payload.counts.created} · 歧义 ${payload.counts.ambiguous};写入 ${payload.knowledge.saved} 条 Knowledge / ${payload.events.saved} 个 Event`,
|
|
6875
|
+
report: payload,
|
|
6876
|
+
})
|
|
6877
|
+
if (!saved.saved) throw new Error(saved.why ?? '归一轨迹没有存下来')
|
|
6878
|
+
}
|
|
6879
|
+
|
|
6880
|
+
/**
|
|
6881
|
+
* 物化段的轨迹(2026-10-18):一个卡片分组 → 一张卡。
|
|
6882
|
+
*
|
|
6883
|
+
* 这一段的产物是卡片本身,所以「卡里有什么」已经查得到;查不到的是**这一次为什么这么排**:
|
|
6884
|
+
* 分组指纹、卡片组织的结论(模型排的还是机械拼的)、进了哪几条知识、正文里为什么链到
|
|
6885
|
+
* 那一张卡。这些是改完提示词、想知道「这次和上次差在哪」时唯一能对着看的东西。
|
|
6886
|
+
*
|
|
6887
|
+
* 粒度是**一组**而不是一个任务:一张卡组织失败,不该让同批其它卡的轨迹一起消失。
|
|
6888
|
+
*/
|
|
6889
|
+
private async recordMaterializeTrajectory(input: {
|
|
6890
|
+
group: MaterializationGroup
|
|
6891
|
+
action: 'created' | 'updated' | 'withdrawn'
|
|
6892
|
+
card: { id: string; title: string; status: string; characters: number; events: number; images: number }
|
|
6893
|
+
knowledge: Knowledge[]
|
|
6894
|
+
placements?: CardKnowledgeRow[]
|
|
6895
|
+
organizeUsed: boolean
|
|
6896
|
+
layout?: CardLayoutRecord
|
|
6897
|
+
related: OrganizeRelatedCandidate[]
|
|
6898
|
+
context: PipelineContext
|
|
6899
|
+
warnings?: string[]
|
|
6900
|
+
}): Promise<void> {
|
|
6901
|
+
const sections: MaterializeTrajectoryReport['card']['sections'] = []
|
|
6902
|
+
for (const row of input.placements ?? []) {
|
|
6903
|
+
const hit = sections.find(section => section.label === row.section)
|
|
6904
|
+
if (hit) hit.count += 1
|
|
6905
|
+
else sections.push({ label: row.section, count: 1 })
|
|
6906
|
+
}
|
|
6907
|
+
const payload: MaterializeTrajectoryReport = {
|
|
6908
|
+
stage: 'materialize',
|
|
6909
|
+
at: new Date().toISOString(),
|
|
6910
|
+
group: {
|
|
6911
|
+
key: input.group.key,
|
|
6912
|
+
title: input.group.title,
|
|
6913
|
+
coordinate: [input.group.coordinate.objectId, input.group.coordinate.actionId, input.group.coordinate.sceneId].filter(Boolean).join(' · ') || '无坐标',
|
|
6914
|
+
state: input.group.state,
|
|
6915
|
+
fingerprint: input.group.fingerprint,
|
|
6916
|
+
},
|
|
6917
|
+
action: input.action,
|
|
6918
|
+
card: {
|
|
6919
|
+
id: input.card.id,
|
|
6920
|
+
title: input.card.title,
|
|
6921
|
+
status: input.card.status,
|
|
6922
|
+
characters: input.card.characters,
|
|
6923
|
+
knowledge: input.knowledge.length,
|
|
6924
|
+
events: input.card.events,
|
|
6925
|
+
images: input.card.images,
|
|
6926
|
+
sections,
|
|
6927
|
+
},
|
|
6928
|
+
organize: input.organizeUsed
|
|
6929
|
+
? { used: true, ...(input.layout?.origin ? { origin: input.layout.origin } : {}), ...(input.layout?.hash ? { layoutHash: input.layout.hash } : {}) }
|
|
6930
|
+
: { used: false },
|
|
6931
|
+
rows: input.knowledge.slice(0, 300).map(row => ({ id: row.id, text: row.text, sources: row.sources.length })),
|
|
6932
|
+
sources: [...input.group.sourceIds],
|
|
6933
|
+
related: input.related.map(candidate => ({ title: candidate.title, reason: candidate.reason })),
|
|
6934
|
+
model: {
|
|
6935
|
+
ok: true,
|
|
6936
|
+
provider: input.context.model.provider,
|
|
6937
|
+
model: input.context.model.model,
|
|
6938
|
+
reasoningEffort: input.context.model.reasoningEffort,
|
|
6939
|
+
},
|
|
6940
|
+
warnings: input.warnings ?? [],
|
|
6941
|
+
}
|
|
6942
|
+
const label = payload.action === 'created' ? '新建' : payload.action === 'updated' ? '更新' : '停用'
|
|
6943
|
+
const saved = await this.recordExtractionRun({
|
|
6944
|
+
stage: 'materialize',
|
|
6945
|
+
title: `${payload.card.title || payload.group.title} · ${label}`,
|
|
6946
|
+
summary: `${label}「${payload.card.title || payload.group.title}」:${payload.card.knowledge} 条知识 · ${payload.card.events} 个事件 · ${payload.card.images} 张图 · ${payload.card.sections.length} 节${payload.organize.used ? '' : '(机械拼装,未用卡片组织)'}`,
|
|
6947
|
+
report: payload,
|
|
6948
|
+
})
|
|
6949
|
+
if (!saved.saved) throw new Error(saved.why ?? '物化轨迹没有存下来')
|
|
6950
|
+
}
|
|
6951
|
+
|
|
6782
6952
|
/**
|
|
6783
6953
|
* 轴注册表(0063 §3):Object / Action / Scene 的 canonical 身份。
|
|
6784
6954
|
*
|
|
@@ -6853,6 +7023,7 @@ export class KnowledgeRuntime {
|
|
|
6853
7023
|
* 不能当成「没命中」。
|
|
6854
7024
|
*/
|
|
6855
7025
|
async resolveTrialAxisSet(input: {
|
|
7026
|
+
registry?: AxisRegistry
|
|
6856
7027
|
axes: TrialAxisSet
|
|
6857
7028
|
axesBlock?: AxisPromptBlock
|
|
6858
7029
|
runId: string
|
|
@@ -6875,6 +7046,7 @@ export class KnowledgeRuntime {
|
|
|
6875
7046
|
candidates: Candidate[]
|
|
6876
7047
|
}
|
|
6877
7048
|
type Judgment = { decision: 'match' | 'new' | 'uncertain'; candidateId: string }
|
|
7049
|
+
const resolve = (value: Parameters<KnowledgeRuntime['resolveAxisMention']>[0]) => this.resolveAxisMention({ ...value, registry: input.registry })
|
|
6878
7050
|
const groups: Array<{ kind: AxisKind; key: GroupKey }> = [
|
|
6879
7051
|
{ kind: 'object', key: 'objects' },
|
|
6880
7052
|
{ kind: 'action', key: 'actions' },
|
|
@@ -6885,7 +7057,7 @@ export class KnowledgeRuntime {
|
|
|
6885
7057
|
const axisKey = (kind: AxisKind, mention: string): string => `${kind}\u0000${normalizeAxisName(mention)}`
|
|
6886
7058
|
const supplementalEntries: Record<AxisKind, Candidate[]> = { object: [], action: [], scene: [] }
|
|
6887
7059
|
try {
|
|
6888
|
-
const registry = await this.axisRegistry()
|
|
7060
|
+
const registry = input.registry ?? await this.axisRegistry()
|
|
6889
7061
|
if (registry !== null) {
|
|
6890
7062
|
// 初次抽取的候选清单可能被截断。再按本批轴 mention 做一次本地召回,补进未下发的已有坐标。
|
|
6891
7063
|
const query = groups.flatMap(group => input.axes[group.key].flatMap(axis => [
|
|
@@ -6935,7 +7107,7 @@ export class KnowledgeRuntime {
|
|
|
6935
7107
|
const key = axisKey(group.kind, axis.name)
|
|
6936
7108
|
let lookup: AxisResolution | null
|
|
6937
7109
|
try {
|
|
6938
|
-
lookup = await
|
|
7110
|
+
lookup = await resolve({
|
|
6939
7111
|
kind: group.kind,
|
|
6940
7112
|
mention: axis.name,
|
|
6941
7113
|
parentMention: axis.parentMention,
|
|
@@ -6998,7 +7170,7 @@ export class KnowledgeRuntime {
|
|
|
6998
7170
|
}
|
|
6999
7171
|
|
|
7000
7172
|
// 没有可供模型比较的现有候选时,沿用注册表的新建候选路径。
|
|
7001
|
-
const created = await
|
|
7173
|
+
const created = await resolve({
|
|
7002
7174
|
kind: group.kind,
|
|
7003
7175
|
mention: axis.name,
|
|
7004
7176
|
parentMention: axis.parentMention,
|
|
@@ -7101,7 +7273,7 @@ export class KnowledgeRuntime {
|
|
|
7101
7273
|
const key = axisKey(item.kind, item.axis.name)
|
|
7102
7274
|
const judgment = decisions.get(key)
|
|
7103
7275
|
if (judgment?.decision === 'new') {
|
|
7104
|
-
const resolution = await
|
|
7276
|
+
const resolution = await resolve({
|
|
7105
7277
|
kind: item.kind,
|
|
7106
7278
|
mention: item.axis.name,
|
|
7107
7279
|
parentMention: item.axis.parentMention,
|
|
@@ -7115,7 +7287,7 @@ export class KnowledgeRuntime {
|
|
|
7115
7287
|
if (judgment?.decision === 'match') {
|
|
7116
7288
|
const candidate = item.candidates.find(value => value.id === judgment.candidateId)
|
|
7117
7289
|
if (candidate) {
|
|
7118
|
-
const resolution = await
|
|
7290
|
+
const resolution = await resolve({
|
|
7119
7291
|
kind: item.kind,
|
|
7120
7292
|
mention: item.axis.name,
|
|
7121
7293
|
parentMention: item.axis.parentMention,
|
|
@@ -7146,6 +7318,7 @@ export class KnowledgeRuntime {
|
|
|
7146
7318
|
}
|
|
7147
7319
|
|
|
7148
7320
|
async resolveAxisMention(input: {
|
|
7321
|
+
registry?: AxisRegistry
|
|
7149
7322
|
kind: AxisKind
|
|
7150
7323
|
mention: string
|
|
7151
7324
|
parentMention?: string
|
|
@@ -7163,7 +7336,7 @@ export class KnowledgeRuntime {
|
|
|
7163
7336
|
/** 模型给这一轮的定义(0068)。只在新建、或给一个还没有定义的实体补一次。 */
|
|
7164
7337
|
description?: string
|
|
7165
7338
|
}): Promise<AxisResolution | null> {
|
|
7166
|
-
const registry = await this.axisRegistry()
|
|
7339
|
+
const registry = input.registry ?? await this.axisRegistry()
|
|
7167
7340
|
if (registry === null) return null
|
|
7168
7341
|
const runId = input.runId ?? ''
|
|
7169
7342
|
let parentId = ''
|
|
@@ -7177,7 +7350,7 @@ export class KnowledgeRuntime {
|
|
|
7177
7350
|
})
|
|
7178
7351
|
if (parent.entity) parentId = parent.entity.id
|
|
7179
7352
|
}
|
|
7180
|
-
const preferId = await this.axisIdByName(input.kind, input.preferMention)
|
|
7353
|
+
const preferId = await this.axisIdByName(input.kind, input.preferMention, registry)
|
|
7181
7354
|
return registry.resolve({
|
|
7182
7355
|
kind: input.kind, mention: input.mention,
|
|
7183
7356
|
...(parentId ? { parentId } : {}), ...(runId ? { runId } : {}),
|
|
@@ -7194,10 +7367,10 @@ export class KnowledgeRuntime {
|
|
|
7194
7367
|
* 「是已有的某个东西」而那个名字在库里指两个实体时,把它当成指认就等于替人做了
|
|
7195
7368
|
* 「哪一个」的决定 —— 而注册表的规矩是「不唯一就不猜」。
|
|
7196
7369
|
*/
|
|
7197
|
-
private async axisIdByName(kind: AxisKind, mention: string | undefined): Promise<string | undefined> {
|
|
7370
|
+
private async axisIdByName(kind: AxisKind, mention: string | undefined, supplied?: AxisRegistry): Promise<string | undefined> {
|
|
7198
7371
|
const name = (mention ?? '').trim()
|
|
7199
7372
|
if (!name) return undefined
|
|
7200
|
-
const registry = await this.axisRegistry()
|
|
7373
|
+
const registry = supplied ?? await this.axisRegistry()
|
|
7201
7374
|
if (registry === null) return undefined
|
|
7202
7375
|
try {
|
|
7203
7376
|
const probe = await registry.resolve({ kind, mention: name, create: false })
|
|
@@ -7474,7 +7647,7 @@ export class KnowledgeRuntime {
|
|
|
7474
7647
|
const cardOrphans = cards.filter(card => card.sourceRefs.every(ref => !live.has(ref))).length
|
|
7475
7648
|
const events = (await this.loadEvents()).filter(event => event.records.some(record => !live.has(record.sourceId))).length
|
|
7476
7649
|
/**
|
|
7477
|
-
* 未被引用的来源图片也进这张账(2026-10-05)。`
|
|
7650
|
+
* 未被引用的来源图片也进这张账(2026-10-05)。`image_publish` 写入的独立图片会被识别并保留。
|
|
7478
7651
|
*
|
|
7479
7652
|
* 少了它,「来源与卡片都清完了、只剩一堆没人引用的图」这种状态在界面上**一个字都没有**
|
|
7480
7653
|
* —— 清理孤儿的入口因此不出现,而用户唯一能看到的读数(「收录 N 个文件」)又把它算成
|
|
@@ -7511,7 +7684,7 @@ export class KnowledgeRuntime {
|
|
|
7511
7684
|
|
|
7512
7685
|
/**
|
|
7513
7686
|
* 还有没有人在引用这个桶 —— 判据是来源正文、来源 front matter 的 `metadata.assets`、
|
|
7514
|
-
* 卡片正文出现过 `_assets/<bucket>/`,或者它是 `
|
|
7687
|
+
* 卡片正文出现过 `_assets/<bucket>/`,或者它是 `image_publish` 写入的独立图床图片。
|
|
7515
7688
|
* 独立图片没有来源/卡片引用是正常状态,必须保留。
|
|
7516
7689
|
*
|
|
7517
7690
|
* 为什么三处都要看:桶名是**内容哈希的前 12 位**(`convert/ingest.ts`),不是来源 id。
|
|
@@ -7549,7 +7722,7 @@ export class KnowledgeRuntime {
|
|
|
7549
7722
|
}
|
|
7550
7723
|
|
|
7551
7724
|
/**
|
|
7552
|
-
* `
|
|
7725
|
+
* `image_publish` 的图片不需要被来源或卡片引用。新写入的桶有明确标记;同时兼容
|
|
7553
7726
|
* 标记加入前已发布的图片:它们以图片自身的 SHA-256 前 12 位作为桶名。
|
|
7554
7727
|
*/
|
|
7555
7728
|
private async isPublishedImageBucket(name: string): Promise<boolean> {
|
|
@@ -7557,7 +7730,7 @@ export class KnowledgeRuntime {
|
|
|
7557
7730
|
if (existsSync(join(root, name, PUBLISHED_IMAGE_MARKER))) return true
|
|
7558
7731
|
}
|
|
7559
7732
|
|
|
7560
|
-
//
|
|
7733
|
+
// 历史版本发布的图片只会写入知识库根目录的 assets/,因此只在这里做哈希识别。
|
|
7561
7734
|
if (!/^[a-f0-9]{12}$/i.test(name)) return false
|
|
7562
7735
|
const dir = join(this.rootDir, ASSETS_DIR, name)
|
|
7563
7736
|
try {
|
|
@@ -7786,6 +7959,15 @@ export class KnowledgeRuntime {
|
|
|
7786
7959
|
saved: number
|
|
7787
7960
|
merged: number
|
|
7788
7961
|
skipped: number
|
|
7962
|
+
/**
|
|
7963
|
+
* **写入失败**的条数,与 `skipped` 分开(0070)。
|
|
7964
|
+
*
|
|
7965
|
+
* `skipped` = 引文没通过核验(数据问题,重试无用);`failures` = 落库时抛错
|
|
7966
|
+
* (磁盘、锁、约束,基础设施问题,重试有意义)。合成一个数会让下游把两者
|
|
7967
|
+
* 说成同一句话,而它们的处置完全不同。
|
|
7968
|
+
*/
|
|
7969
|
+
failures: number
|
|
7970
|
+
warnings: string[]
|
|
7789
7971
|
/**
|
|
7790
7972
|
* **报告里的知识 id → 库里的知识 id**(0069)。
|
|
7791
7973
|
*
|
|
@@ -7798,11 +7980,13 @@ export class KnowledgeRuntime {
|
|
|
7798
7980
|
ids: Record<string, string>
|
|
7799
7981
|
}> {
|
|
7800
7982
|
const store = await this.knowledgeStore()
|
|
7801
|
-
if (store === null) return { saved: 0, merged: 0, skipped: input.knowledge.length, ids: {} }
|
|
7983
|
+
if (store === null) return { saved: 0, merged: 0, skipped: input.knowledge.length, failures: 0, warnings: [], ids: {} }
|
|
7802
7984
|
const axes = { objects: input.objects, actions: input.actions, scenes: input.scenes }
|
|
7803
7985
|
let saved = 0
|
|
7804
7986
|
let merged = 0
|
|
7805
7987
|
let skipped = 0
|
|
7988
|
+
let failures = 0
|
|
7989
|
+
const warnings: string[] = []
|
|
7806
7990
|
const ids: Record<string, string> = {}
|
|
7807
7991
|
for (const item of input.knowledge) {
|
|
7808
7992
|
const usable = item.sources.filter(source => source.found)
|
|
@@ -7832,9 +8016,20 @@ export class KnowledgeRuntime {
|
|
|
7832
8016
|
* 直接用它,不需要再自己判一次归并方向。
|
|
7833
8017
|
*/
|
|
7834
8018
|
ids[item.id] = result.knowledge.id
|
|
7835
|
-
} catch {
|
|
8019
|
+
} catch (cause) {
|
|
8020
|
+
/*
|
|
8021
|
+
* 写入失败与「引文没通过核验」必须分开报。
|
|
8022
|
+
*
|
|
8023
|
+
* 原来这里是一个 `catch { skipped += 1 }`:磁盘满、SQLITE_BUSY、约束冲突
|
|
8024
|
+
* 全都被算成「引文没核验过」,原因被丢掉,而下游只看到同一句话 ——
|
|
8025
|
+
* 运维读到的是一份数据问题,实际是基础设施问题,于是无限重试。
|
|
8026
|
+
* Event 那条路径本来就把消息收进 warnings(见下面),这里对齐它。
|
|
8027
|
+
*/
|
|
8028
|
+
failures += 1
|
|
8029
|
+
warnings.push(`Knowledge「${item.text.slice(0, 40)}」写入失败:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
8030
|
+
}
|
|
7836
8031
|
}
|
|
7837
|
-
return { saved, merged, skipped, ids }
|
|
8032
|
+
return { saved, merged, skipped, failures, warnings, ids }
|
|
7838
8033
|
}
|
|
7839
8034
|
|
|
7840
8035
|
/**
|
|
@@ -7884,14 +8079,12 @@ export class KnowledgeRuntime {
|
|
|
7884
8079
|
refs.push(event.id)
|
|
7885
8080
|
eventsByCoordinate.set(key, refs)
|
|
7886
8081
|
}
|
|
7887
|
-
const eventIndex = new Map(events.map(event => [event.id, event]))
|
|
7888
8082
|
const warnings: string[] = []
|
|
7889
8083
|
let updated = 0
|
|
7890
8084
|
for (const card of await this.loadCards()) {
|
|
7891
|
-
const refs = card.coordinate === undefined
|
|
8085
|
+
const refs = card.status === 'deprecated' || card.coordinate === undefined || card.entityType === 'diagnosis/case'
|
|
7892
8086
|
? []
|
|
7893
8087
|
: (eventsByCoordinate.get(coordinateKeyOf(card.coordinate)) ?? []).sort()
|
|
7894
|
-
db.setCardEvents(card.entityId, refs)
|
|
7895
8088
|
const oldRaw = card.metadata.event_refs
|
|
7896
8089
|
let oldRefs: string[] = []
|
|
7897
8090
|
let oldRefsValid = false
|
|
@@ -7907,11 +8100,17 @@ export class KnowledgeRuntime {
|
|
|
7907
8100
|
oldRefs = oldRaw.filter((item): item is string => typeof item === 'string').sort()
|
|
7908
8101
|
oldRefsValid = true
|
|
7909
8102
|
}
|
|
7910
|
-
const content =
|
|
8103
|
+
const content = stripEventReferenceSection(card.content)
|
|
7911
8104
|
const contentChanged = content !== card.content
|
|
7912
8105
|
const refsChanged = !oldRefsValid || JSON.stringify(refs) !== JSON.stringify(oldRefs)
|
|
7913
|
-
if (!refsChanged && !contentChanged)
|
|
7914
|
-
|
|
8106
|
+
if (!refsChanged && !contentChanged) {
|
|
8107
|
+
db.setCardEvents(card.entityId, refs)
|
|
8108
|
+
continue
|
|
8109
|
+
}
|
|
8110
|
+
if (!oldRefsValid && refs.length === 0 && oldRaw === undefined && !contentChanged) {
|
|
8111
|
+
db.setCardEvents(card.entityId, refs)
|
|
8112
|
+
continue
|
|
8113
|
+
}
|
|
7915
8114
|
const written = await this.writeCard({
|
|
7916
8115
|
...card,
|
|
7917
8116
|
sourceRefs: card.sourceRefs,
|
|
@@ -7919,8 +8118,11 @@ export class KnowledgeRuntime {
|
|
|
7919
8118
|
metadata: { ...card.metadata, event_refs: JSON.stringify(refs) },
|
|
7920
8119
|
origin: 'internal',
|
|
7921
8120
|
})
|
|
7922
|
-
if (written.ok === false) warnings.push(
|
|
7923
|
-
else
|
|
8121
|
+
if (written.ok === false) warnings.push(`WikiCard ${card.title} 的 Event 引用未更新:${written.error.message}`)
|
|
8122
|
+
else {
|
|
8123
|
+
db.setCardEvents(card.entityId, refs)
|
|
8124
|
+
updated += 1
|
|
8125
|
+
}
|
|
7924
8126
|
}
|
|
7925
8127
|
return { updated, warnings }
|
|
7926
8128
|
}
|
|
@@ -7959,10 +8161,6 @@ export class KnowledgeRuntime {
|
|
|
7959
8161
|
|
|
7960
8162
|
for (const event of db.events()) {
|
|
7961
8163
|
const remaining = event.records.filter(record => record.sourceId !== input.sourceId)
|
|
7962
|
-
if (remaining.length === 0) {
|
|
7963
|
-
db.removeEvent(event.id)
|
|
7964
|
-
continue
|
|
7965
|
-
}
|
|
7966
8164
|
const state: EventState = {
|
|
7967
8165
|
id: event.id,
|
|
7968
8166
|
identityKey: event.identityKey,
|
|
@@ -8063,7 +8261,8 @@ export class KnowledgeRuntime {
|
|
|
8063
8261
|
}
|
|
8064
8262
|
|
|
8065
8263
|
for (const state of states.values()) {
|
|
8066
|
-
if (!state.dirty
|
|
8264
|
+
if (!state.dirty) continue
|
|
8265
|
+
if (state.records.length === 0) { db.removeEvent(state.id); continue }
|
|
8067
8266
|
const records = state.records
|
|
8068
8267
|
const entity = eventEntityFromRecords({
|
|
8069
8268
|
id: state.id,
|
|
@@ -8084,8 +8283,518 @@ export class KnowledgeRuntime {
|
|
|
8084
8283
|
return { saved, merged, skipped, ids, warnings }
|
|
8085
8284
|
}
|
|
8086
8285
|
|
|
8087
|
-
/**
|
|
8088
|
-
async
|
|
8286
|
+
/** The stage runner shares this base's existing writer queue and audit database. */
|
|
8287
|
+
async pipeline(): Promise<KnowledgePipeline> {
|
|
8288
|
+
if (this.closed) throw new Error('知识库已关闭')
|
|
8289
|
+
this.pipelineOpening ??= (async () => {
|
|
8290
|
+
const db = await this.storeHandle()
|
|
8291
|
+
const audit = await this.extractionRuns()
|
|
8292
|
+
if (this.closed) throw new Error('知识库已关闭')
|
|
8293
|
+
if (!db || !audit || !await this.knowledgeStore()) throw new Error('知识库或任务历史不可用,无法启动处理任务')
|
|
8294
|
+
const store = new PipelineStore(db.connection(), audit.connection())
|
|
8295
|
+
const engine = new KnowledgePipeline(store, {
|
|
8296
|
+
capture: () => ({ model: this.modelSeamNow(), note: this.extractionNote(), profile: structuredClone(this.organizeProfile()), organize: this.config.cardOrganize, format: 'table' }),
|
|
8297
|
+
serialize: work => {
|
|
8298
|
+
this.compilePending += 1
|
|
8299
|
+
const run = this.compileTail.then(work, work)
|
|
8300
|
+
this.compileTail = run.then(() => undefined, () => undefined).finally(() => { this.compilePending -= 1 })
|
|
8301
|
+
return run
|
|
8302
|
+
},
|
|
8303
|
+
sources: () => this.pipelineSources(),
|
|
8304
|
+
source: sourceId => this.pipelineSource(sourceId),
|
|
8305
|
+
/**
|
|
8306
|
+
* 抽取指纹必须覆盖**真正喂给模型的那段实体清单**(0070),但要按本文提到的条目收窄。
|
|
8307
|
+
*
|
|
8308
|
+
* `extract` 把 `axisPromptBlock(source.content)` 拼进提示词(注册表里的实体名、
|
|
8309
|
+
* 别名、定义与人工指引都在里面),而这些会随人的确认、合并与模型的补定义变化。
|
|
8310
|
+
* 指纹漏掉它 → 注册表变了仍复用旧报告,模型当时的 `existing` / `dropped` 判断
|
|
8311
|
+
* 被冻住。旧路径的指纹是含注册表成分的(`axesRevision`),这里原来不对称。
|
|
8312
|
+
*
|
|
8313
|
+
* 见 `pipelineExtractionFingerprint` —— 那里还有一条必需的配套:归一成功后要**刷新**这份
|
|
8314
|
+
* 指纹,否则归一自己学到的实体(它们本来就出现在这份文档里)会让它永远无法复用。
|
|
8315
|
+
*/
|
|
8316
|
+
fingerprint: (source, context) => this.pipelineExtractionFingerprint(source, context),
|
|
8317
|
+
extract: async (source, context, onDelta) => {
|
|
8318
|
+
if (!context.model.call) throw new Error(`抽取模型不可用:${context.model.why ?? '请在编译设置中选择模型'}`)
|
|
8319
|
+
const axes = await this.axisPromptBlock(source.content)
|
|
8320
|
+
return runWikiTrial({ title: source.title, text: source.content, docId: source.sourceId, axes, note: context.note, call: context.model.call, modelRequest: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort }, outputFormat: context.format, sessionId: `knowledge:${this.baseId || 'default'}`, onDelta })
|
|
8321
|
+
},
|
|
8322
|
+
normalize: (source, checkpoint, context, force, output) => this.normalizePipelineSource(store, source, checkpoint, context, force, output),
|
|
8323
|
+
/**
|
|
8324
|
+
* 引擎只把 `snapshot` 原样传回来,所以那边看到的是 `unknown`;这里把它取回成
|
|
8325
|
+
* 真实类型(见 `materialize-snapshot.ts` 里为什么不放在 `pipeline/types.ts`)。
|
|
8326
|
+
*/
|
|
8327
|
+
groups: async context => {
|
|
8328
|
+
const snapshot = await this.materializationSnapshot(store, context)
|
|
8329
|
+
return { groups: snapshot.groups, blocked: snapshot.blocked, blockedAmbiguous: snapshot.blockedAmbiguous, blockedAwaitingNormalize: snapshot.blockedAwaitingNormalize, snapshot }
|
|
8330
|
+
},
|
|
8331
|
+
materialize: (group, context, output, snapshot) => this.materializePipelineGroup(store, group, context, output, snapshot as MaterializationSnapshot),
|
|
8332
|
+
build: async (taskId, checkpoint) => {
|
|
8333
|
+
let result: Envelope<RebuildReport>
|
|
8334
|
+
try { result = await this.compileOnce(taskId, 'all', undefined, { extract: false, checkpoint }) }
|
|
8335
|
+
catch (cause) { await this.settleProgressFailure(taskId, cause); throw cause }
|
|
8336
|
+
this.settleProgressReport(taskId, result)
|
|
8337
|
+
if (!result.ok) throw new Error(result.error.message)
|
|
8338
|
+
return result.data.skipped
|
|
8339
|
+
? { summary: `索引无变化:复用 ${result.data.notesReused ?? 0} 条召回片段,仍为 v${result.data.version}。`, skipped: true }
|
|
8340
|
+
: { summary: `检索索引已更新至 v${result.data.version}。`, skipped: false }
|
|
8341
|
+
},
|
|
8342
|
+
record: async checkpoint => {
|
|
8343
|
+
/**
|
|
8344
|
+
* `scope: 'extract-only'` —— 这份报告的 ②③④ 段**不是这个库里真的做过的事**。
|
|
8345
|
+
*
|
|
8346
|
+
* 报告的形状来自 `runWikiTrial`(试跑:一次调用里把归一、物化、建索引的机械结果
|
|
8347
|
+
* 也顺带算出来)。但在这条流水线里,归一与物化是**独立阶段**,各有自己的任务与轨迹。
|
|
8348
|
+
* 不标这一笔,用户点开抽取轨迹会看到「物化 · 0 张 Wiki Card」并以为物化跑过了。
|
|
8349
|
+
*
|
|
8350
|
+
* 旧的全链路编译(`compileOnce`)不标:那一条路上报告里的四段是真跑过的。
|
|
8351
|
+
*/
|
|
8352
|
+
const audit = await this.recordExtractionRun({ id: checkpoint.id, docId: checkpoint.sourceId, title: checkpoint.report.title, summary: trialSummary(checkpoint.report), report: { ...checkpoint.report, scope: 'extract-only' } })
|
|
8353
|
+
if (!audit.saved) throw new Error(`抽取报告保存失败:${audit.why}`)
|
|
8354
|
+
},
|
|
8355
|
+
})
|
|
8356
|
+
this.pipelineEngine = engine
|
|
8357
|
+
if (this.closed) engine.close()
|
|
8358
|
+
return engine
|
|
8359
|
+
})()
|
|
8360
|
+
return this.pipelineOpening
|
|
8361
|
+
}
|
|
8362
|
+
|
|
8363
|
+
private async pipelineSources(): Promise<PipelineSource[]> {
|
|
8364
|
+
const db = await this.storeHandle()
|
|
8365
|
+
if (!db) throw new Error('来源存储不可用')
|
|
8366
|
+
return db.sources().map(row => ({ ...row.record, version: row.version, sourceHash: contentHashOf(row.record.content) })).sort((a, b) => a.sourceId.localeCompare(b.sourceId))
|
|
8367
|
+
}
|
|
8368
|
+
|
|
8369
|
+
private async pipelineSource(sourceId: string): Promise<PipelineSource | undefined> {
|
|
8370
|
+
const db = await this.storeHandle()
|
|
8371
|
+
if (!db) throw new Error('来源存储不可用')
|
|
8372
|
+
const row = db.source(sourceId)
|
|
8373
|
+
return row ? { ...row.record, version: row.version, sourceHash: contentHashOf(row.record.content) } : undefined
|
|
8374
|
+
}
|
|
8375
|
+
|
|
8376
|
+
/**
|
|
8377
|
+
* 抽取指纹(0070)。
|
|
8378
|
+
*
|
|
8379
|
+
* 判据是「这段提示词会不会变」,所以实体清单要按**本文真的提到**的条目收窄:小库发的是
|
|
8380
|
+
* 全集(`selection: 'full'`),直接哈希整块清单会让「新增一个与本文无关的实体」触发全库
|
|
8381
|
+
* 重抽 —— 而那正是既有测试钉住不许发生的退化。
|
|
8382
|
+
*
|
|
8383
|
+
* **已知的取舍**(不是疏漏):提示词里那份清单是 `promptEntries` 按相关度排/截出来的,
|
|
8384
|
+
* 而这里按字面包含关系重筛一遍。两者不完全等价 —— 「在提示词里但本文没有字面提到」的
|
|
8385
|
+
* 实体被改名时,这里不会失效。选择这条判据是因为另一条(原样哈希发出去的清单)与
|
|
8386
|
+
* 上面那条被测试钉住的产品规则直接冲突。残留风险由归一那一侧兜住:归一总是对着**实时**
|
|
8387
|
+
* 注册表重新解析坐标,所以模型当时的 `existing` 判断不会原样变成最终卡名。
|
|
8388
|
+
*
|
|
8389
|
+
* 它有两个调用点:抽取前判断能不能复用,以及归一成功后**刷新**已存的那一份。后者是必需的:
|
|
8390
|
+
* 归一学到的实体本来就出现在这份文档里,不刷新的话这份文档下次抽取永远复用不上。这与
|
|
8391
|
+
* `normalizationFingerprint` 存「补判之后」的快照是同一条规则,不是两套。
|
|
8392
|
+
*/
|
|
8393
|
+
private async pipelineExtractionFingerprint(source: PipelineSource, context: PipelineContext): Promise<string> {
|
|
8394
|
+
const axes = await this.axisPromptBlock(source.content)
|
|
8395
|
+
const text = normalizeAxisName(source.content)
|
|
8396
|
+
const relevant = (['object', 'action', 'scene'] as const).map(kind => (axes?.entries[kind] ?? [])
|
|
8397
|
+
.filter(entry => [entry.name, ...entry.aliases].map(normalizeAxisName).some(value => value.length > 0 && text.includes(value)))
|
|
8398
|
+
.map(entry => ({ id: entry.id, name: entry.name, aliases: entry.aliases, description: entry.description, guidance: entry.guidance }))
|
|
8399
|
+
.sort((left, right) => left.id.localeCompare(right.id)))
|
|
8400
|
+
return contentHashOf(JSON.stringify({ source: source.sourceHash, title: source.title, contract: EXTRACTION_CONTRACT_VERSION, note: context.note, format: context.format, axes: relevant, model: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort } }))
|
|
8401
|
+
}
|
|
8402
|
+
|
|
8403
|
+
private async normalizationFingerprint(checkpoint: ExtractionCheckpoint, context: PipelineContext): Promise<string> {
|
|
8404
|
+
const registry = await this.axisRegistry()
|
|
8405
|
+
const report = checkpoint.normalized?.report ?? checkpoint.report
|
|
8406
|
+
const query = [...report.objects, ...report.actions, ...report.scenes].flatMap(axis => [axis.name, axis.existing ?? '', axis.definition ?? '', axis.parentMention ?? '']).join('\n')
|
|
8407
|
+
const candidates = await registry?.promptEntries(24, query)
|
|
8408
|
+
const mentions = [...checkpoint.report.objects, ...checkpoint.report.actions, ...checkpoint.report.scenes].flatMap(axis => [axis.name, axis.existing ?? '', axis.parentMention ?? '']).map(normalizeAxisName).filter(Boolean)
|
|
8409
|
+
const resolvedIds = new Set([...report.objects, ...report.actions, ...report.scenes].map(axis => axis.id).filter(Boolean))
|
|
8410
|
+
// Ignore unrelated filler rows and registry recency; retain candidates that can affect these mentions.
|
|
8411
|
+
const relevant = Object.fromEntries(Object.entries(candidates?.entries ?? {}).map(([kind, entries]) => [kind, entries.filter(entry => resolvedIds.has(entry.id) || mentions.some(mention => [entry.name, ...entry.aliases, entry.description].map(normalizeAxisName).some(value => value.includes(mention) || (value.length > 1 && mention.includes(value))))).sort((a, b) => a.id.localeCompare(b.id))]))
|
|
8412
|
+
return contentHashOf(JSON.stringify({ extraction: checkpoint.id, model: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort }, candidates: relevant }))
|
|
8413
|
+
}
|
|
8414
|
+
|
|
8415
|
+
private async normalizePipelineSource(store: PipelineStore, source: PipelineSource, checkpoint: ExtractionCheckpoint, context: PipelineContext, force: boolean, output: (text: string) => void): Promise<{ reused: boolean; warnings: string[] }> {
|
|
8416
|
+
const fingerprint = await this.normalizationFingerprint(checkpoint, context)
|
|
8417
|
+
if (!force && checkpoint.normalized?.fingerprint === fingerprint && store.head(source.sourceId).normalized?.id === checkpoint.id) return { reused: true, warnings: [] }
|
|
8418
|
+
if (!checkpoint.report.model.response) throw new Error('抽取记录没有原始返回,无法归一;请重新抽取该文档')
|
|
8419
|
+
const registry = await this.axisRegistry()
|
|
8420
|
+
if (!registry) throw new Error('实体注册表不可用')
|
|
8421
|
+
const draft = await registry.draft()
|
|
8422
|
+
try {
|
|
8423
|
+
const axesBlock = await this.axisPromptBlock(source.content)
|
|
8424
|
+
const call: ModelCall | undefined = context.model.call ? request => context.model.call!({ ...request, onDelta: output }) : undefined
|
|
8425
|
+
const report = await runWikiTrial({
|
|
8426
|
+
title: source.title, text: source.content, docId: source.sourceId, axes: axesBlock,
|
|
8427
|
+
// Reparse the captured response; this callback never requests another extraction.
|
|
8428
|
+
call: async () => checkpoint.report.model.response!, modelRequest: checkpoint.report.model.request,
|
|
8429
|
+
resolveAxisSet: axes => this.resolveTrialAxisSet({ registry: draft.registry, axes, axesBlock, runId: checkpoint.id, call, modelRequest: context.model }),
|
|
8430
|
+
})
|
|
8431
|
+
if (!report.model.parsed) throw new Error(report.model.why ?? '抽取结果无法归一')
|
|
8432
|
+
const db = await this.storeHandle()
|
|
8433
|
+
const current = db?.source(source.sourceId)
|
|
8434
|
+
if (this.closed) throw new Error('知识库已关闭,本次归一未提交')
|
|
8435
|
+
if (!db || !current || current.version !== source.version || contentHashOf(current.record.content) !== source.sourceHash) throw new Error('来源在归一期间已变化,请重新抽取最新版本')
|
|
8436
|
+
const sql = db.connection()
|
|
8437
|
+
sql.exec('SAVEPOINT pipeline_normalize')
|
|
8438
|
+
try {
|
|
8439
|
+
draft.commit()
|
|
8440
|
+
// Preserve evidence history while replacing this source's active contribution.
|
|
8441
|
+
sql.prepare('UPDATE knowledge_sources SET withdrawn=1 WHERE document_id=?').run(source.sourceId)
|
|
8442
|
+
const knowledge = await this.ingestTrialKnowledge({ runId: checkpoint.id, knowledge: report.knowledge, objects: report.objects, actions: report.actions, scenes: report.scenes, sourceVersions: { [source.sourceId]: source.version } })
|
|
8443
|
+
const events = await this.ingestTrialEvents({ sourceId: source.sourceId, runId: checkpoint.id, events: report.events, objects: report.objects, actions: report.actions, scenes: report.scenes })
|
|
8444
|
+
if (knowledge.skipped || knowledge.failures || events.skipped) throw new Error(`出处核验或写入失败:${knowledge.skipped} 条 Knowledge 引文未通过核验、${knowledge.failures} 条写入失败、${events.skipped} 个 Event;本次归一没有提交`)
|
|
8445
|
+
const unresolvedKnowledgeIds = report.knowledge.filter(item => ([['object', report.objects], ['action', report.actions], ['scene', report.scenes]] as const).some(([kind, axes]) => {
|
|
8446
|
+
const name = item[kind]
|
|
8447
|
+
const axis = name ? axes.find(axis => axis.name === name) : undefined
|
|
8448
|
+
return name !== null && (!axis || axis.outcome === 'ambiguous' || axis.outcome === 'no-registry')
|
|
8449
|
+
})).map(item => knowledge.ids[item.id]!).filter(Boolean)
|
|
8450
|
+
const normalized = { report, knowledgeIds: knowledge.ids, eventIds: events.ids, unresolvedKnowledgeIds, fingerprint: '' }
|
|
8451
|
+
// Store the post-resolution candidate snapshot so learning its own entities does not invalidate this run.
|
|
8452
|
+
normalized.fingerprint = await this.normalizationFingerprint({ ...checkpoint, normalized }, context)
|
|
8453
|
+
/**
|
|
8454
|
+
* 归一学到的实体本来就在这份文档的提示词里,所以它会把「抽取指纹」改掉。
|
|
8455
|
+
* 把已存的那一份刷新成**学完之后**的值,否则这份文档下次抽取永远复用不上
|
|
8456
|
+
* (与上面那行存「补判之后」的快照同一条规则)。
|
|
8457
|
+
*/
|
|
8458
|
+
store.refreshFingerprint(checkpoint.id, await this.pipelineExtractionFingerprint(source, context))
|
|
8459
|
+
store.saveNormalization(checkpoint, normalized)
|
|
8460
|
+
sql.exec('RELEASE pipeline_normalize')
|
|
8461
|
+
// Updating Event references is a cheap relation refresh. It does not reorganize or index card content.
|
|
8462
|
+
const eventReferences = await this.syncEventReferences()
|
|
8463
|
+
const warnings = [...knowledge.warnings, ...events.warnings, ...eventReferences.warnings, ...(unresolvedKnowledgeIds.length ? [`${unresolvedKnowledgeIds.length} 条 Knowledge 的坐标仍有歧义,保留事实与出处,暂不物化。`] : [])]
|
|
8464
|
+
/**
|
|
8465
|
+
* 轨迹在**提交之后**才落(`RELEASE` 之后):写在 savepoint 里面的话,回滚了还会留下
|
|
8466
|
+
* 一条「归一成功了」的记录。
|
|
8467
|
+
*
|
|
8468
|
+
* 落不下来只记一条警告,不把这一段判失败 —— 事实已经写进知识库了,这时候抛错会让人
|
|
8469
|
+
* 以为没生效。抽取那一段是反过来的(它的产物只有报告,存不下就等于什么都没做)。
|
|
8470
|
+
*/
|
|
8471
|
+
try {
|
|
8472
|
+
await this.recordNormalizeTrajectory({ source, checkpoint, report, knowledge, events, unresolvedKnowledgeIds, fingerprint: normalized.fingerprint, warnings, context })
|
|
8473
|
+
} catch (cause) {
|
|
8474
|
+
warnings.push(`归一轨迹没有存下来:${cause instanceof Error ? cause.message : String(cause)}`)
|
|
8475
|
+
}
|
|
8476
|
+
return { reused: false, warnings }
|
|
8477
|
+
} catch (cause) { sql.exec('ROLLBACK TO pipeline_normalize; RELEASE pipeline_normalize'); throw cause }
|
|
8478
|
+
} finally { draft.close() }
|
|
8479
|
+
}
|
|
8480
|
+
|
|
8481
|
+
private async activePipelineKnowledge(): Promise<Knowledge[]> {
|
|
8482
|
+
const knowledge = await this.knowledgeStore()
|
|
8483
|
+
const db = await this.storeHandle()
|
|
8484
|
+
if (!knowledge || !db) return []
|
|
8485
|
+
/**
|
|
8486
|
+
* **来源正文预读一次**,别在每一行上重读整份文档。
|
|
8487
|
+
*
|
|
8488
|
+
* `assessSource` 要对每条 knowledge-source 判定「引文是否还在当前版本里」,而那需要
|
|
8489
|
+
* 该文档的正文。原来的写法是 `db.source(id)`:每行一次全表 join + 整份正文 + 全文
|
|
8490
|
+
* flatten + 子串匹配 —— 5000 条知识 × 每份 50 KB 就是几百 MB 的重复工作,而这只是
|
|
8491
|
+
* 一次调用。正文在这里读一次(一条查询),下面按 documentId 查表。
|
|
8492
|
+
*/
|
|
8493
|
+
const bodies = new Map<string, { version: number; content: string }>()
|
|
8494
|
+
for (const row of db.sources()) bodies.set(row.record.sourceId, { version: row.version, content: row.record.content })
|
|
8495
|
+
const rows: Knowledge[] = []
|
|
8496
|
+
for (let offset = 0;; offset += 500) {
|
|
8497
|
+
const page = await knowledge.list({ excludeWithdrawn: true, limit: 500, offset })
|
|
8498
|
+
rows.push(...page.knowledge.filter(row => row.status !== 'deprecated').map(row => assessKnowledgeEvidence(row, db, bodies)).map(row => ({ ...row, sources: row.sources.filter(source => !source.withdrawn && source.evidenceStatus !== 'missing') })).filter(row => row.sources.length > 0))
|
|
8499
|
+
if (offset + page.knowledge.length >= page.total) break
|
|
8500
|
+
}
|
|
8501
|
+
return rows
|
|
8502
|
+
}
|
|
8503
|
+
|
|
8504
|
+
/**
|
|
8505
|
+
* 这一组现在能不能物化(有没有来源变了、新抽取待归一、或坐标还有歧义)。
|
|
8506
|
+
*
|
|
8507
|
+
* 抽成一处是因为它有**两个**调用点:阶段开始时的全量快照,与单组写回之前的复查。
|
|
8508
|
+
* 各写一份的话,「什么算变化」迟早会有两套说法。
|
|
8509
|
+
*/
|
|
8510
|
+
private needsNormalizationForGroup(store: PipelineStore, knowledge: Knowledge[], unresolved: { ids: Set<string>; sources: Set<string> }, db: KnowledgeDb | null): boolean {
|
|
8511
|
+
return knowledge.some(row => unresolved.ids.has(row.id)) || [...new Set(knowledge.flatMap(row => row.sources.map(source => source.documentId)))].filter(Boolean).some(id => {
|
|
8512
|
+
const source = db?.source(id)
|
|
8513
|
+
const head = store.head(id)
|
|
8514
|
+
return head.normalized ? head.normalized.sourceVersion !== source?.version || head.extracted?.id !== head.normalized.id : knowledge.some(row => row.sources.some(evidence => evidence.documentId === id && evidence.toVersion !== undefined && evidence.toVersion !== source?.version))
|
|
8515
|
+
})
|
|
8516
|
+
}
|
|
8517
|
+
|
|
8518
|
+
/** 相关推荐的整库计划。它是增强,所以算不出来时给空表而不是把编译卡住。 */
|
|
8519
|
+
private async relatedPlan(): Promise<Map<string, OrganizeRelatedCandidate[]>> {
|
|
8520
|
+
const plan = new Map<string, OrganizeRelatedCandidate[]>()
|
|
8521
|
+
try {
|
|
8522
|
+
const cards = await this.loadCards()
|
|
8523
|
+
const relations = Object.values(await this.readExtractedRelations()).flat()
|
|
8524
|
+
for (const [key, entries] of planRelated(cards, relations, await this.projectedRelated(cards))) {
|
|
8525
|
+
if (entries.length > 0) plan.set(key, entries.map(entry => ({ title: entry.title, entityType: entry.entityType, reason: entry.reason })))
|
|
8526
|
+
}
|
|
8527
|
+
} catch { /* 增强项:算不出来就不给清单 */ }
|
|
8528
|
+
return plan
|
|
8529
|
+
}
|
|
8530
|
+
|
|
8531
|
+
/**
|
|
8532
|
+
* 一个分组的**事实指纹**:只有它变了才需要重新成卡。
|
|
8533
|
+
*
|
|
8534
|
+
* 抽成一处是因为它有**两个**调用点:阶段开始时的全库快照,与单组写回之前的复查。
|
|
8535
|
+
* 两处必须用同一个表达式,否则「什么算变化」就有两套说法。
|
|
8536
|
+
*/
|
|
8537
|
+
private pipelineGroupFingerprint(knowledge: Knowledge[], context: PipelineContext): string {
|
|
8538
|
+
return contentHashOf(JSON.stringify({ knowledge: knowledge.map(row => ({ id: row.id, text: row.text, status: row.status, sources: row.sources.filter(source => !source.withdrawn) })).sort((a, b) => a.id.localeCompare(b.id)), profile: context.profile, organize: context.organize, model: context.organize ? { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort } : undefined }))
|
|
8539
|
+
}
|
|
8540
|
+
|
|
8541
|
+
/**
|
|
8542
|
+
* **只读这一组**的事实指纹,写回之前复查用。
|
|
8543
|
+
*
|
|
8544
|
+
* 快照在阶段内是稳定的,但那有个前提:写入者都走同一条串行队列。别的进程、或同一个库的
|
|
8545
|
+
* 另一条路径(强制重归一复用同一个检查点 id、诊断载入示例、删除流程)不受它约束 ——
|
|
8546
|
+
* 那时「分组的知识行变了,而 head 指针没变」是可能的,只看 head 会漏掉,于是拿陈旧的
|
|
8547
|
+
* 行去写卡片(丢更新)。这里按坐标只读这一组,代价与该组的知识条数成正比,不扫全库。
|
|
8548
|
+
*/
|
|
8549
|
+
private async freshGroupFingerprint(coordinate: CardCoordinate, context: PipelineContext, db: KnowledgeDb): Promise<string> {
|
|
8550
|
+
const store = await this.knowledgeStore()
|
|
8551
|
+
if (!store) return ''
|
|
8552
|
+
const raw: Knowledge[] = []
|
|
8553
|
+
for (let offset = 0;; offset += 500) {
|
|
8554
|
+
const page = await store.list({ objectId: coordinate.objectId, actionId: coordinate.actionId, sceneId: coordinate.sceneId, excludeWithdrawn: true, limit: 500, offset })
|
|
8555
|
+
raw.push(...page.knowledge)
|
|
8556
|
+
if (offset + page.knowledge.length >= page.total) break
|
|
8557
|
+
}
|
|
8558
|
+
// 只读这一组**用到**的那些来源正文,而不是全库的。
|
|
8559
|
+
const documentIds = [...new Set(raw.flatMap(row => row.sources.map(source => source.documentId)).filter(Boolean))]
|
|
8560
|
+
const bodies = new Map<string, { version: number; content: string }>()
|
|
8561
|
+
for (const id of documentIds) {
|
|
8562
|
+
const source = db.source(id)
|
|
8563
|
+
if (source) bodies.set(id, { version: source.version, content: source.record.content })
|
|
8564
|
+
}
|
|
8565
|
+
const rows = raw
|
|
8566
|
+
.filter(row => row.status !== 'deprecated')
|
|
8567
|
+
.map(row => assessKnowledgeEvidence(row, db, bodies))
|
|
8568
|
+
.map(row => ({ ...row, sources: row.sources.filter(source => !source.withdrawn && source.evidenceStatus !== 'missing') }))
|
|
8569
|
+
.filter(row => row.sources.length > 0)
|
|
8570
|
+
return this.pipelineGroupFingerprint(rows, context)
|
|
8571
|
+
}
|
|
8572
|
+
|
|
8573
|
+
/**
|
|
8574
|
+
* 物化阶段的**一次**全库快照。
|
|
8575
|
+
*
|
|
8576
|
+
* 逐组现算会变成 O(组数 × 语料):每个分组要扫 4 遍全库(三次分组、一次知识),
|
|
8577
|
+
* 再加上每组一次图片分发与一次相关推荐 —— 几百张卡时单线程跑不完。这里全部算一次。
|
|
8578
|
+
*/
|
|
8579
|
+
private async materializationSnapshot(store: PipelineStore, context: PipelineContext, options: { plans?: boolean } = {}): Promise<MaterializationSnapshot> {
|
|
8580
|
+
/**
|
|
8581
|
+
* `plans: false` 给**只读状态**的调用方(`pipelineStatus`):它只要 groups / blocked /
|
|
8582
|
+
* 各类计数,而图片分发与相关推荐是整库计算(全部来源正文 + 全部卡片 + 关系投影),
|
|
8583
|
+
* 让它跟着 2.5 秒一次的状态轮询跑,等于把整个语料每 2.5 秒重算一遍。
|
|
8584
|
+
*/
|
|
8585
|
+
const plans = options.plans !== false
|
|
8586
|
+
const unresolved = store.unresolvedKnowledge()
|
|
8587
|
+
const grouped = new Map<string, { coordinate: CardCoordinate; knowledge: Knowledge[] }>()
|
|
8588
|
+
let blockedAmbiguous = 0
|
|
8589
|
+
let blockedAwaitingNormalize = 0
|
|
8590
|
+
const groupFor = (coordinate: CardCoordinate) => {
|
|
8591
|
+
const key = coordinateKeyOf(coordinate)
|
|
8592
|
+
if (!grouped.has(key)) grouped.set(key, { coordinate, knowledge: [] })
|
|
8593
|
+
return grouped.get(key)!
|
|
8594
|
+
}
|
|
8595
|
+
for (const row of await this.activePipelineKnowledge()) {
|
|
8596
|
+
const coordinate = { objectId: row.objectId, actionId: row.actionId, sceneId: row.sceneId }
|
|
8597
|
+
if (coordinateIsEmpty(coordinate)) { blockedAmbiguous += 1; continue }
|
|
8598
|
+
groupFor(coordinate).knowledge.push(row)
|
|
8599
|
+
}
|
|
8600
|
+
const db = await this.storeHandle()
|
|
8601
|
+
const events = new Map<string, EventEntity[]>()
|
|
8602
|
+
for (const event of db?.events() ?? []) {
|
|
8603
|
+
const coordinate = event.coordinateIds
|
|
8604
|
+
if (!coordinate.objectId || !coordinate.actionId || !coordinate.sceneId) continue
|
|
8605
|
+
const key = coordinateKeyOf(coordinate)
|
|
8606
|
+
const bucket = events.get(key) ?? []
|
|
8607
|
+
bucket.push(event)
|
|
8608
|
+
events.set(key, bucket)
|
|
8609
|
+
}
|
|
8610
|
+
const cards = await this.loadCards()
|
|
8611
|
+
const byCoordinate = new Map(cards.filter(card => card.coordinate).map(card => [coordinateKeyOf(card.coordinate!), card]))
|
|
8612
|
+
const groups: MaterializationGroup[] = []
|
|
8613
|
+
const knowledge = new Map<string, Knowledge[]>()
|
|
8614
|
+
for (const [key, value] of grouped) {
|
|
8615
|
+
const names = await this.coordinateNames(value.coordinate)
|
|
8616
|
+
const existing = byCoordinate.get(key)
|
|
8617
|
+
// Event 只作为卡片引用;它的增删和正文变化不得触发 Knowledge 重新组织。
|
|
8618
|
+
const fingerprint = this.pipelineGroupFingerprint(value.knowledge, context)
|
|
8619
|
+
const receipt = store.materialization(key)
|
|
8620
|
+
const current = existing && receipt?.fingerprint === fingerprint && receipt.cardId === existing.entityId
|
|
8621
|
+
const contributingSources = [...new Set(value.knowledge.flatMap(row => row.sources.map(source => source.documentId)))].filter(Boolean).sort()
|
|
8622
|
+
const needsNormalization = this.needsNormalizationForGroup(store, value.knowledge, unresolved, db)
|
|
8623
|
+
if (needsNormalization) blockedAwaitingNormalize += value.knowledge.length
|
|
8624
|
+
knowledge.set(key, value.knowledge)
|
|
8625
|
+
groups.push({ key, title: existing?.title ?? cardNameFor(names.object || null, names.action || null, names.scene || null), coordinate: value.coordinate, knowledgeCount: value.knowledge.length, eventCount: events.get(key)?.length ?? 0, sourceIds: contributingSources, ...(existing ? { cardId: existing.entityId } : {}), state: needsNormalization ? 'blocked' : current ? 'current' : existing ? 'changed' : 'new', ...(needsNormalization ? { reason: '分组中的来源已变化、新抽取待归一或坐标有歧义,请先完成归一' } : {}), fingerprint })
|
|
8626
|
+
}
|
|
8627
|
+
for (const card of cards) {
|
|
8628
|
+
if (!card.coordinate || grouped.has(coordinateKeyOf(card.coordinate)) || !store.materialization(coordinateKeyOf(card.coordinate)) || card.status === 'deprecated') continue
|
|
8629
|
+
groups.push({ key: coordinateKeyOf(card.coordinate), title: card.title, coordinate: card.coordinate, knowledgeCount: 0, eventCount: 0, sourceIds: card.sourceRefs, cardId: card.entityId, state: card.sourceRefs.some(id => unresolved.sources.has(id)) ? 'blocked' : 'withdrawn', ...(card.sourceRefs.some(id => unresolved.sources.has(id)) ? { reason: '来源中的坐标仍有歧义,请先完成归一' } : {}), fingerprint: 'withdrawn' })
|
|
8630
|
+
}
|
|
8631
|
+
const sources = plans ? (await this.loadSources()).map(source => ({ sourceId: source.sourceId, content: source.content })) : []
|
|
8632
|
+
const imagePlan = plans
|
|
8633
|
+
? planSourceImages({
|
|
8634
|
+
cards: groups.filter(item => item.state !== 'withdrawn').map(item => ({ key: item.key, content: (knowledge.get(item.key) ?? []).map(row => row.text).join('\n\n'), sourceRefs: item.sourceIds })),
|
|
8635
|
+
sources,
|
|
8636
|
+
})
|
|
8637
|
+
: { byCard: new Map<string, SourceImage[]>() }
|
|
8638
|
+
/**
|
|
8639
|
+
* 相关推荐是按**卡片身份**(`type:entityId`)算的,而分组键是坐标 —— 两套键永远对不上,
|
|
8640
|
+
* 所以必须在这里把身份映射到分组键。少了这一步,`organizeCard` 拿不到「可引用的邻居」,
|
|
8641
|
+
* 而模型一旦照样写出 `[[名字]]`,`checkOrganize` 会判 `ORGANIZE_UNKNOWN_REFERENCE`、
|
|
8642
|
+
* 整组物化失败(卡片永远写不出来)。
|
|
8643
|
+
*
|
|
8644
|
+
* 只对**已经有卡片**的分组建映射:新建的卡片在相关计划里本来就不存在(与旧行为一致)。
|
|
8645
|
+
* 键用的是卡片自己的类型,与 `relatedForCard(existing)` 一致。
|
|
8646
|
+
*/
|
|
8647
|
+
const relatedByKey = new Map<string, OrganizeRelatedCandidate[]>()
|
|
8648
|
+
if (plans) {
|
|
8649
|
+
const plan = await this.relatedPlan()
|
|
8650
|
+
for (const [key, entries] of relatedByGroupKey(groups, groupKey => {
|
|
8651
|
+
const card = byCoordinate.get(groupKey)
|
|
8652
|
+
return card === undefined ? undefined : cardKey(card)
|
|
8653
|
+
}, plan)) {
|
|
8654
|
+
relatedByKey.set(key, entries)
|
|
8655
|
+
}
|
|
8656
|
+
}
|
|
8657
|
+
return {
|
|
8658
|
+
groups: groups.sort((a, b) => a.title.localeCompare(b.title)),
|
|
8659
|
+
blocked: blockedAmbiguous + blockedAwaitingNormalize,
|
|
8660
|
+
blockedAmbiguous, blockedAwaitingNormalize,
|
|
8661
|
+
knowledge, events, sources, images: imagePlan.byCard, related: relatedByKey,
|
|
8662
|
+
}
|
|
8663
|
+
}
|
|
8664
|
+
|
|
8665
|
+
private async materializePipelineGroup(store: PipelineStore, group: MaterializationGroup, context: PipelineContext, output: (text: string) => void, snapshot: MaterializationSnapshot): Promise<void> {
|
|
8666
|
+
const db = await this.storeHandle()
|
|
8667
|
+
if (!db) throw new Error('知识库不可用')
|
|
8668
|
+
/**
|
|
8669
|
+
* 复查读**同一份快照**:语料在物化阶段内不会变(归一排在编译队列的后面),
|
|
8670
|
+
* 所以分组指纹是稳定的;这里只对目标组重算「是否等待归一」。
|
|
8671
|
+
* 会变的是人工裁决 —— 那一条在写回之前单独重读(见下面的 `statusAtWrite`)。
|
|
8672
|
+
*/
|
|
8673
|
+
const known = snapshot.groups.find(item => item.key === group.key)
|
|
8674
|
+
if (!known || known.fingerprint !== group.fingerprint) throw new Error('卡片分组在处理期间已变化,请刷新并重试')
|
|
8675
|
+
const knowledge = snapshot.knowledge.get(group.key) ?? []
|
|
8676
|
+
if (this.needsNormalizationForGroup(store, knowledge, store.unresolvedKnowledge(), db)) throw new Error(known.reason ?? '分组中的来源已变化、新抽取待归一或坐标有歧义,请先完成归一')
|
|
8677
|
+
const storedExisting = group.cardId ? await this.readCard(group.cardId) : undefined
|
|
8678
|
+
const existing = storedExisting
|
|
8679
|
+
? { ...storedExisting, content: stripEventReferenceSection(storedExisting.content) }
|
|
8680
|
+
: undefined
|
|
8681
|
+
if (group.state === 'withdrawn') {
|
|
8682
|
+
if (!existing) return
|
|
8683
|
+
/**
|
|
8684
|
+
* 撤回也要过同一道人工裁决的闸门(0070):这一条原来在它**后面**才判,
|
|
8685
|
+
* 于是「有人在处理期间给这张卡做了 review / 停用」会被 `{...existing, status:'deprecated'}`
|
|
8686
|
+
* 静默覆盖。
|
|
8687
|
+
*/
|
|
8688
|
+
const statusBeforeDeprecate = (await this.readCard(existing.entityId))?.status
|
|
8689
|
+
if (statusBeforeDeprecate === undefined) throw new Error('卡片在处理期间已被删除,本次物化未提交,请刷新后重试')
|
|
8690
|
+
if (statusBeforeDeprecate !== existing.status) throw new Error(`卡片在处理期间被人工改动(状态 ${existing.status} → ${statusBeforeDeprecate}),本次物化未提交,请重试`)
|
|
8691
|
+
const written = await this.writeCard({ ...existing, status: 'deprecated', origin: 'compile' })
|
|
8692
|
+
if (!written.ok) throw new Error(written.error.message)
|
|
8693
|
+
store.materialized(group.key, group.fingerprint, existing.entityId)
|
|
8694
|
+
// 停用也是一次物化:卡片从可召回里退出,这一步发生过,就该留一条轨迹。
|
|
8695
|
+
await this.recordMaterializeTrajectory({
|
|
8696
|
+
group, action: 'withdrawn', knowledge: [],
|
|
8697
|
+
card: { id: existing.entityId, title: existing.title, status: 'deprecated', characters: existing.content.length, events: 0, images: 0 },
|
|
8698
|
+
organizeUsed: false, related: [], context,
|
|
8699
|
+
}).catch(() => undefined)
|
|
8700
|
+
return
|
|
8701
|
+
}
|
|
8702
|
+
const events = snapshot.events.get(group.key) ?? []
|
|
8703
|
+
const profile = context.profile
|
|
8704
|
+
const entityType = (profile?.id ?? 'concept') as SchemaType
|
|
8705
|
+
let title = group.title
|
|
8706
|
+
let content = `## 知识\n\n${knowledge.map(row => `- ${row.text}`).join('\n')}`
|
|
8707
|
+
let summary = knowledge[0]?.text ?? ''
|
|
8708
|
+
let placements: CardKnowledgeRow[] = knowledge.map((row, index) => ({ knowledgeId: row.id, section: '知识', ordinal: index + 1, origin: 'compiler' }))
|
|
8709
|
+
let metadata: Record<string, unknown> = { ...(existing?.metadata ?? {}), pipeline_materialization: true }
|
|
8710
|
+
const entityId = existing?.entityId ?? `coord-${hash(group.key)}`
|
|
8711
|
+
const images = snapshot.images.get(group.key) ?? []
|
|
8712
|
+
const related = existing && snapshot.related.has(group.key) ? { related: snapshot.related.get(group.key)! } : {}
|
|
8713
|
+
metadata.citations = citationsFrom({ placements, knowledge: new Map(knowledge.map(row => [row.id, { source: row.sources[0]?.documentId, heading: row.sources[0]?.headingPath.at(-1) }])) })
|
|
8714
|
+
if (context.organize && knowledge.length > 0) {
|
|
8715
|
+
if (!context.model.call) throw new Error(`卡片组织模型不可用:${context.model.why ?? '请选择模型'}`)
|
|
8716
|
+
const result = await this.organizeCard({ knowledge, images, ...related, cardTitle: title, coordinate: group.coordinate, profileId: entityType, force: true, suggestedSections: profile?.sections.map(section => section.label), requiredSections: profile?.sections.filter(section => section.required).map(section => section.label), existing, call: request => context.model.call!({ ...request, onDelta: output }) })
|
|
8717
|
+
if (result.kind !== 'applied') throw new Error(result.kind === 'reused' ? '组织结果已变化,请刷新后重试' : result.warnings.join(';'))
|
|
8718
|
+
title = result.title; content = result.content; summary = result.summary; placements = result.placements
|
|
8719
|
+
metadata = { ...metadata, layout: result.layout, citations: result.citations }
|
|
8720
|
+
}
|
|
8721
|
+
for (const image of images) content = insertImage(content, image)
|
|
8722
|
+
const eventIds = events.map(event => event.id).sort()
|
|
8723
|
+
content = stripEventReferenceSection(content)
|
|
8724
|
+
metadata.event_refs = JSON.stringify(eventIds)
|
|
8725
|
+
/**
|
|
8726
|
+
* 冻结的是这一组的事实,而且要在**写之前**再核一次(0070)。
|
|
8727
|
+
*
|
|
8728
|
+
* 两条都要:斜杠前那条查的是「来源是不是等归一」(head 指针级别),它看不见
|
|
8729
|
+
* 「只有知识行变了」;这条按坐标重算整组指纹,能把那种变化也挡住。
|
|
8730
|
+
*/
|
|
8731
|
+
if (this.needsNormalizationForGroup(store, knowledge, store.unresolvedKnowledge(), db)) throw new Error('卡片组织期间事实发生变化,结果未提交,请重试')
|
|
8732
|
+
if (await this.freshGroupFingerprint(group.coordinate, context, db) !== group.fingerprint) throw new Error('卡片分组在处理期间事实发生了变化,结果未提交,请重试')
|
|
8733
|
+
if (this.closed) throw new Error('知识库已关闭,本次物化未提交')
|
|
8734
|
+
/**
|
|
8735
|
+
* 状态必须**在写之前**重读,并且和读到的旧值比对。
|
|
8736
|
+
*
|
|
8737
|
+
* `existing` 是在模型调用(`organizeCard`,秒到几十秒)**之前**读的,而 `status` 是
|
|
8738
|
+
* 显式传下去的 —— `writeCard` 的「不传 = 保留库里已有的」保护因此不生效。用户在
|
|
8739
|
+
* 这段时间里点了「停用」,旧值会把人工裁决静默覆盖掉,卡片又回到索引里,
|
|
8740
|
+
* 而且没有任何地方报错。所以:值变了就**拒绝提交**,让人重试,而不是猜哪个对。
|
|
8741
|
+
*/
|
|
8742
|
+
const statusAtWrite = entityId ? (await this.readCard(entityId))?.status : undefined
|
|
8743
|
+
// 「卡片没了」与「状态被改了」是两件事:前者要重建成新卡,后者要人重试。分开说。
|
|
8744
|
+
if (existing !== undefined && statusAtWrite === undefined) throw new Error('卡片在处理期间已被删除,本次物化未提交,请刷新后重试')
|
|
8745
|
+
if (statusAtWrite !== existing?.status) {
|
|
8746
|
+
throw new Error(`卡片在处理期间被人工改动(状态 ${existing?.status ?? '未知'} → ${statusAtWrite ?? '未知'}),本次物化未提交,请重试`)
|
|
8747
|
+
}
|
|
8748
|
+
const written = await this.writeCard({ entityType, entityId, title, content, summary, sourceRefs: group.sourceIds, status: statusAtWrite === 'deprecated' ? 'unreviewed' : statusAtWrite ?? 'unreviewed', coordinate: group.coordinate, placements, metadata, origin: 'compile' })
|
|
8749
|
+
if (!written.ok) throw new Error(written.error.message)
|
|
8750
|
+
db.setCardEvents(entityId, eventIds)
|
|
8751
|
+
store.materialized(group.key, group.fingerprint, entityId)
|
|
8752
|
+
/**
|
|
8753
|
+
* 卡片已经写好了,轨迹落不下来不该把它判失败(同归一那条理由)。所以这里吞掉异常 ——
|
|
8754
|
+
* 但**不吞在别处**:写卡失败仍然抛。
|
|
8755
|
+
*/
|
|
8756
|
+
await this.recordMaterializeTrajectory({
|
|
8757
|
+
group,
|
|
8758
|
+
action: existing === undefined ? 'created' : 'updated',
|
|
8759
|
+
knowledge,
|
|
8760
|
+
placements,
|
|
8761
|
+
organizeUsed: Boolean(context.organize && knowledge.length > 0),
|
|
8762
|
+
...(layoutOf(metadata) ? { layout: layoutOf(metadata)! } : {}),
|
|
8763
|
+
related: snapshot.related.get(group.key) ?? [],
|
|
8764
|
+
card: {
|
|
8765
|
+
id: entityId,
|
|
8766
|
+
title,
|
|
8767
|
+
status: statusAtWrite === 'deprecated' ? 'unreviewed' : statusAtWrite ?? 'unreviewed',
|
|
8768
|
+
characters: content.length,
|
|
8769
|
+
events: eventIds.length,
|
|
8770
|
+
images: images.length,
|
|
8771
|
+
},
|
|
8772
|
+
context,
|
|
8773
|
+
}).catch(() => undefined)
|
|
8774
|
+
}
|
|
8775
|
+
|
|
8776
|
+
async pipelineStatus(): Promise<PipelineStatus> {
|
|
8777
|
+
const pipeline = await this.pipeline()
|
|
8778
|
+
const sources = await this.pipelineSources()
|
|
8779
|
+
const context: PipelineContext = { model: this.modelSeamNow(), note: this.extractionNote(), profile: this.organizeProfile(), organize: this.config.cardOrganize, format: 'table' }
|
|
8780
|
+
const rows: PipelineStatus['sources'] = await Promise.all(sources.map(async source => {
|
|
8781
|
+
const head = pipeline.store.head(source.sourceId)
|
|
8782
|
+
const extracted = head.extracted?.sourceVersion === source.version && head.extracted.sourceHash === source.sourceHash
|
|
8783
|
+
const normalized = extracted && head.normalized?.id === head.extracted?.id && head.normalized?.normalized?.fingerprint === await this.normalizationFingerprint(head.normalized!, context)
|
|
8784
|
+
return { id: source.sourceId, title: source.title, version: source.version, extractionId: head.extracted?.id, normalizedId: head.normalized?.id, state: normalized ? 'normalized' as const : extracted ? 'extracted' as const : head.extracted ? 'outdated' as const : 'pending' as const }
|
|
8785
|
+
}))
|
|
8786
|
+
// 状态查询不要图片分发与相关推荐:它每 2.5 秒被轮询一次,而那两项是整库计算。
|
|
8787
|
+
const { groups, blocked } = await this.materializationSnapshot(pipeline.store, context, { plans: false })
|
|
8788
|
+
const db = await this.derivedHandle()
|
|
8789
|
+
const manifest = db?.artifact('build-manifest.json')
|
|
8790
|
+
const hashes = manifest ? (JSON.parse(manifest) as { cardHashes?: Record<string, string> }).cardHashes ?? {} : {}
|
|
8791
|
+
const cards = await this.loadCards()
|
|
8792
|
+
const indexPending = cards.some(card => hashes[`${card.entityType}:${card.entityId}`] !== this.cardHashOf(card)) || Object.keys(hashes).length !== cards.length
|
|
8793
|
+
return { sources: rows, groups, blocked, pendingNew: groups.filter(group => group.state === 'new').length, pendingUpdated: groups.filter(group => group.state === 'changed').length, pendingWithdrawn: groups.filter(group => group.state === 'withdrawn').length, indexPending }
|
|
8794
|
+
}
|
|
8795
|
+
|
|
8796
|
+
/** 轨迹列表(新→旧,不含报告本体)。`stage` 省略 = 三段一起。 */
|
|
8797
|
+
async listExtractionRuns(query: { docId?: string; stage?: TrajectoryStage; limit?: number; offset?: number } = {}): Promise<ExtractionRunPage | null> {
|
|
8089
8798
|
const store = await this.extractionRuns()
|
|
8090
8799
|
if (store === null) return null
|
|
8091
8800
|
try { return await store.list(query) } catch { return null }
|
|
@@ -8271,11 +8980,12 @@ export class KnowledgeRuntime {
|
|
|
8271
8980
|
* 跑一趟编译的**实体**(0066)。它不做排队,也不对外承诺「你 await 它就等于你发起了
|
|
8272
8981
|
* 它」—— 那两件事分别在 `enqueueCompile()` 与 `rebuild()` / `startCompile()` 里。
|
|
8273
8982
|
*/
|
|
8274
|
-
private async compileOnce(runId: string, only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
8275
|
-
await this.environmentGuard?.()
|
|
8983
|
+
private async compileOnce(runId: string, only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean; checkpoint?: () => Promise<void> } = {}): Promise<Envelope<RebuildReport>> {
|
|
8984
|
+
if (only === 'all' || only === 'embeddings') await this.environmentGuard?.('embedding')
|
|
8276
8985
|
this.publication = { artifacts: new Map() }
|
|
8277
8986
|
const configAtStart = JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() })
|
|
8278
8987
|
try {
|
|
8988
|
+
await options.checkpoint?.()
|
|
8279
8989
|
await this.loadCompiledState()
|
|
8280
8990
|
const result = await this.prepareCompile(runId, only, lineage, options)
|
|
8281
8991
|
await this.checkpointCompileTask(runId)
|
|
@@ -8305,6 +9015,7 @@ export class KnowledgeRuntime {
|
|
|
8305
9015
|
this.publication.artifacts.set(COMPILE_SNAPSHOT, JSON.stringify(snapshot))
|
|
8306
9016
|
await this.checkpointCompileTask(runId)
|
|
8307
9017
|
if (JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() }) !== configAtStart) throw new Error('COMPILE_CONFIG_CHANGED: 编译期间设置已变化,请重新编译')
|
|
9018
|
+
await options.checkpoint?.()
|
|
8308
9019
|
db.publish(this.publication)
|
|
8309
9020
|
this.vectorIndex = null
|
|
8310
9021
|
}
|
|
@@ -8452,7 +9163,7 @@ export class KnowledgeRuntime {
|
|
|
8452
9163
|
tally.knowledge > 0
|
|
8453
9164
|
? `抽出知识 ${tally.knowledge} 条`
|
|
8454
9165
|
: '未用模型:正文按 ## 小节切',
|
|
8455
|
-
tally.skipped > 0 ?
|
|
9166
|
+
tally.skipped > 0 ? `复用未变来源 ${tally.skipped} 份` : options.forceExtract ? '本次为手动强制重抽' : '正在检查新增或变化的来源',
|
|
8456
9167
|
]
|
|
8457
9168
|
if (done < total) {
|
|
8458
9169
|
setStep('knowledge', 'running', knowledgeFacts)
|
|
@@ -9132,7 +9843,7 @@ export class KnowledgeRuntime {
|
|
|
9132
9843
|
if (derived === null) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '当前 Node 没有 node:sqlite(需要 22.5+),关键词路已降级' } }
|
|
9133
9844
|
if (!(captured?.ready ?? derived.hasFts())) return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '关键词索引还没建(在 derived.db 里),先编译一次' } }
|
|
9134
9845
|
try {
|
|
9135
|
-
const found = (captured?.hits ?? derived.searchFts(query, pool)).filter(hit => eligibleIds.has(hit.unitId))
|
|
9846
|
+
const found = (captured?.hits ?? derived.searchFts(query, pool, eligibleIds)).filter(hit => eligibleIds.has(hit.unitId))
|
|
9136
9847
|
/**
|
|
9137
9848
|
* 覆盖率门槛(见 `MIN_QUERY_COVERAGE`):只蹭到一两个高频单字的命中在这里淘汰,
|
|
9138
9849
|
* 于是「整批都弱」也能弃答 —— 这是相对阈值做不到的那一半。
|
|
@@ -9426,7 +10137,7 @@ export class KnowledgeRuntime {
|
|
|
9426
10137
|
* 只取**命中这几张卡**的落位。用 `placements()`(全表)在这里是错的量级:
|
|
9427
10138
|
* 一次召回十来张卡,而库里可能有上千张。
|
|
9428
10139
|
*/
|
|
9429
|
-
try { placements = snapshot
|
|
10140
|
+
try { placements = snapshot ? new Map(snapshot.placements) : store.placementsOf([...new Set(hits.map(hit => hit.entityId))]) } catch { return }
|
|
9430
10141
|
/**
|
|
9431
10142
|
* 每条命中要哪几条 Knowledge:以 Card 正文切出的 heading 为准。
|
|
9432
10143
|
*
|
|
@@ -9442,7 +10153,7 @@ export class KnowledgeRuntime {
|
|
|
9442
10153
|
})
|
|
9443
10154
|
const wanted = new Set(perHit.flat().map(row => row.knowledgeId))
|
|
9444
10155
|
if (wanted.size === 0) return
|
|
9445
|
-
const rows = snapshot
|
|
10156
|
+
const rows = snapshot
|
|
9446
10157
|
? new Map(snapshot.knowledge.filter(row => wanted.has(row.id) && knowledge !== null)
|
|
9447
10158
|
.flatMap(row => {
|
|
9448
10159
|
// Deleted facts never reappear through a historical snapshot.
|
|
@@ -9536,11 +10247,14 @@ export class KnowledgeRuntime {
|
|
|
9536
10247
|
if (!query?.trim()) return failure(empty(), 'QUERY_REQUIRED', 'query 不能为空', true)
|
|
9537
10248
|
if (!Number.isInteger(recallTopK) || recallTopK < 1 || recallTopK > 50) return failure(empty(), 'LIMIT_INVALID', '召回 top_k 必须是 1 到 50 的整数', false)
|
|
9538
10249
|
if (!Number.isInteger(requestedTopN) || requestedTopN < 1 || (options.contextTopN !== undefined && requestedTopN > recallTopK)) return failure(empty(), 'CONTEXT_TOP_N_INVALID', `上下文 top_n 必须是 1 到 ${recallTopK} 的整数`, false)
|
|
9539
|
-
try {
|
|
10250
|
+
try {
|
|
10251
|
+
if (recallMode !== 'fts') await this.environmentGuard?.('embedding')
|
|
10252
|
+
if (options.rerank !== false && this.config.rerankProvider !== 'none') await this.environmentGuard?.('rerank')
|
|
10253
|
+
}
|
|
9540
10254
|
catch (cause) { return failure(empty(), 'ENVIRONMENT_NOT_READY', cause instanceof Error ? cause.message : String(cause), true, '在 Tita 设置中完成环境准备并重新自检。') }
|
|
9541
10255
|
const derived = await this.derivedHandle()
|
|
9542
10256
|
const liveCards = await this.loadCards()
|
|
9543
|
-
const liveKeys = new Set(liveCards.map(card => cardKey(card)))
|
|
10257
|
+
const liveKeys = new Set(liveCards.filter(card => card.status !== 'deprecated').map(card => cardKey(card)))
|
|
9544
10258
|
const snapshotHash = derived?.artifactHash(COMPILE_SNAPSHOT)
|
|
9545
10259
|
const cached = snapshotHash && this.recallSnapshotCache?.hash === snapshotHash ? this.recallSnapshotCache : undefined
|
|
9546
10260
|
const raw = cached ? undefined : derived?.artifact(COMPILE_SNAPSHOT)
|
|
@@ -9559,7 +10273,6 @@ export class KnowledgeRuntime {
|
|
|
9559
10273
|
}
|
|
9560
10274
|
}
|
|
9561
10275
|
if (snapshot.notes.length === 0) return failure(empty(), 'INDEX_NOT_READY', '知识索引尚未构建,请先编译知识库', true, '先执行 rebuild_knowledge,再进行召回。')
|
|
9562
|
-
const capturedFts = { ready: derived?.hasFts() ?? false, hits: derived?.searchFts(query, candidatePool) ?? [] }
|
|
9563
10276
|
const capturedVector = cached ? cached.vector : decodeVectorIndex(derived?.vectorIndex(), ENCODER_FINGERPRINT)
|
|
9564
10277
|
if (this.environmentGuard && recallMode !== 'fts' && (!capturedVector
|
|
9565
10278
|
|| capturedVector.manifest.provider !== this.config.embeddingProvider
|
|
@@ -9589,6 +10302,15 @@ export class KnowledgeRuntime {
|
|
|
9589
10302
|
const eligible = snapshot.notes.filter(unit => liveKeys.has(`${unit.entityType}:${unit.entityId}`)
|
|
9590
10303
|
&& (includeProblems || currentByKey.get(`${unit.entityType}:${unit.entityId}`)?.status !== 'problem'))
|
|
9591
10304
|
const eligibleIds = new Set(eligible.map(unit => unit.unitId))
|
|
10305
|
+
/**
|
|
10306
|
+
* 关键词路的可见性必须在 **SQL 里、在 LIMIT 之前**收窄。
|
|
10307
|
+
*
|
|
10308
|
+
* 原来这里是先 `LIMIT candidatePool` 再在 JS 里 `filter(eligibleIds)`:删除或停用
|
|
10309
|
+
* 卡片之后索引不会立刻重建,那些死 unitId 仍占着候选位,把能用的候选挤出窗口 ——
|
|
10310
|
+
* 关键词路于是报「0 命中」,整次召回按弃答处理(面板上也看不出原因)。
|
|
10311
|
+
* 向量路一直是先过滤再取 top-k(见下面的扫描循环),这里是它漏掉的那一半。
|
|
10312
|
+
*/
|
|
10313
|
+
const capturedFts = { ready: derived?.hasFts() ?? false, hits: derived?.searchFts(query, candidatePool, eligibleIds) ?? [] }
|
|
9592
10314
|
const queryTerms = [...new Set(segment(query))]
|
|
9593
10315
|
const warnings: string[] = []
|
|
9594
10316
|
// 一份 SourceRaw 语料,两处要用:关键词路的**查询词覆盖率**必须对着「建索引时那份
|
|
@@ -9700,7 +10422,7 @@ export class KnowledgeRuntime {
|
|
|
9700
10422
|
].join('\n')
|
|
9701
10423
|
const reranked = await this.rerankNotes(rerankQuery, hits, this.config.rerankThreshold)
|
|
9702
10424
|
rerankScored = reranked.scored ?? []
|
|
9703
|
-
hits = reranked.hits
|
|
10425
|
+
hits = reranked.hits
|
|
9704
10426
|
rerank = reranked.result
|
|
9705
10427
|
if (reranked.warning) warnings.push(reranked.warning)
|
|
9706
10428
|
}
|
|
@@ -9709,7 +10431,7 @@ export class KnowledgeRuntime {
|
|
|
9709
10431
|
}
|
|
9710
10432
|
// 重排换的是相关性顺序,评分加权仍然要压在最上面 —— 否则「review 过的排前面一点」
|
|
9711
10433
|
// 只在关掉重排时成立,而重排默认是开着的。
|
|
9712
|
-
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId))
|
|
10434
|
+
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId)).slice(0, recallTopK)
|
|
9713
10435
|
|
|
9714
10436
|
// --- R4 · 证据片段 -----------------------------------------------------
|
|
9715
10437
|
const truth = await this.storeHandle()
|