dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -52
- package/cordis.patch.yml +21 -17
- package/eval/benchmark-extraction-format.mjs +20 -0
- package/lib/client.js +2846 -3872
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +1299 -1660
- package/lib/types/axis-registry.d.ts +6 -0
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +45 -0
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/diagnosis/cards.d.ts +7 -3
- package/lib/types/diagnosis/cards.d.ts.map +1 -1
- package/lib/types/diagnosis/cards.js +13 -8
- package/lib/types/diagnosis/cards.js.map +1 -1
- package/lib/types/diagnosis/run.d.ts +4 -13
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +26 -16
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/schema.js +2 -2
- package/lib/types/diagnosis/schema.js.map +1 -1
- package/lib/types/diagnosis/seed-source.d.ts +1 -1
- package/lib/types/diagnosis/seed-source.d.ts.map +1 -1
- package/lib/types/diagnosis/seed-source.js +5 -4
- package/lib/types/diagnosis/seed-source.js.map +1 -1
- package/lib/types/diagnosis/seed.d.ts +4 -4
- package/lib/types/diagnosis/seed.d.ts.map +1 -1
- package/lib/types/diagnosis/seed.js +6 -8
- package/lib/types/diagnosis/seed.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +8 -16
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +41 -66
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +10 -16
- package/lib/types/diagnosis/types.d.ts.map +1 -1
- package/lib/types/diagnosis/types.js.map +1 -1
- package/lib/types/environment/service.d.ts +4 -3
- package/lib/types/environment/service.d.ts.map +1 -1
- package/lib/types/environment/service.js +91 -23
- package/lib/types/environment/service.js.map +1 -1
- package/lib/types/environment/types.d.ts +9 -0
- package/lib/types/environment/types.d.ts.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js +13 -0
- package/lib/types/events.js.map +1 -1
- package/lib/types/extract-contract.d.ts +1 -1
- package/lib/types/extract-contract.d.ts.map +1 -1
- package/lib/types/extract-contract.js +34 -31
- package/lib/types/extract-contract.js.map +1 -1
- package/lib/types/extract-model.d.ts +2 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract-table-contract.d.ts +3 -0
- package/lib/types/extract-table-contract.d.ts.map +1 -0
- package/lib/types/extract-table-contract.js +30 -0
- package/lib/types/extract-table-contract.js.map +1 -0
- package/lib/types/extract-table.d.ts +8 -0
- package/lib/types/extract-table.d.ts.map +1 -0
- package/lib/types/extract-table.js +151 -0
- package/lib/types/extract-table.js.map +1 -0
- package/lib/types/extraction-runs.d.ts +3 -0
- package/lib/types/extraction-runs.d.ts.map +1 -1
- package/lib/types/extraction-runs.js +6 -0
- package/lib/types/extraction-runs.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +196 -44
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/scope.d.ts.map +1 -1
- package/lib/types/intent/scope.js +1 -0
- package/lib/types/intent/scope.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +3 -2
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge.d.ts +3 -0
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +12 -3
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/model-text.d.ts +1 -1
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +5 -4
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/pipeline/engine.d.ts +57 -0
- package/lib/types/pipeline/engine.d.ts.map +1 -0
- package/lib/types/pipeline/engine.js +268 -0
- package/lib/types/pipeline/engine.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +56 -0
- package/lib/types/pipeline/store.d.ts.map +1 -0
- package/lib/types/pipeline/store.js +115 -0
- package/lib/types/pipeline/store.js.map +1 -0
- package/lib/types/pipeline/types.d.ts +82 -0
- package/lib/types/pipeline/types.d.ts.map +1 -0
- package/lib/types/pipeline/types.js +5 -0
- package/lib/types/pipeline/types.js.map +1 -0
- package/lib/types/present.d.ts +2 -0
- package/lib/types/present.d.ts.map +1 -1
- package/lib/types/present.js +9 -4
- package/lib/types/present.js.map +1 -1
- package/lib/types/runtime.d.ts +20 -1
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +389 -112
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/source-evidence.d.ts.map +1 -1
- package/lib/types/source-evidence.js +2 -0
- package/lib/types/source-evidence.js.map +1 -1
- package/lib/types/store/axis-references.js +3 -3
- package/lib/types/store/axis-references.js.map +1 -1
- package/lib/types/store/migrate.js +1 -1
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/types.d.ts +3 -4
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-materialize.d.ts.map +1 -1
- package/lib/types/wiki-materialize.js +3 -15
- package/lib/types/wiki-materialize.js.map +1 -1
- package/lib/types/wiki-trial-types.d.ts +3 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial.d.ts +2 -0
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +31 -7
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +6 -3
- package/preset/tita-data-agent/agent.cordis.yml +21 -17
- package/src/axis-registry.ts +38 -0
- package/src/client/AttributionEventGuide.tsx +10 -9
- package/src/client/EnvironmentSetup.tsx +35 -15
- package/src/client/ExtractionTasksPage.tsx +129 -176
- package/src/client/KnowledgeBaseTab.tsx +31 -76
- package/src/client/NodeSettings.tsx +2 -3
- package/src/client/PipelineNotice.tsx +26 -0
- package/src/client/TabGuide.tsx +4 -4
- package/src/client/TypeDefinitionDrawer.tsx +23 -8
- package/src/client/WikiCardPreview.tsx +43 -10
- package/src/client/WikiFlowPanel.tsx +4 -4
- package/src/client/architecture.tsx +1 -1
- package/src/client/attribution/AttributionTab.tsx +6 -6
- package/src/client/attribution/CaseView.tsx +3 -1
- package/src/client/attribution/EventView.tsx +14 -4
- package/src/client/attribution/RunView.tsx +2 -2
- package/src/client/attribution/graph-options.ts +2 -2
- package/src/client/pages.tsx +2 -2
- package/src/diagnosis/cards.ts +13 -8
- package/src/diagnosis/run.ts +27 -17
- package/src/diagnosis/schema.ts +2 -2
- package/src/diagnosis/seed-source.ts +5 -4
- package/src/diagnosis/seed.ts +6 -8
- package/src/diagnosis/store.ts +47 -68
- package/src/diagnosis/types.ts +10 -16
- package/src/environment/service.ts +79 -23
- package/src/environment/types.ts +6 -0
- package/src/events.ts +12 -0
- package/src/extract-contract.ts +6 -3
- package/src/extract-model.ts +2 -0
- package/src/extract-table-contract.ts +29 -0
- package/src/extract-table.ts +126 -0
- package/src/extraction-runs.ts +6 -0
- package/src/index.ts +156 -45
- package/src/intent/scope.ts +1 -0
- package/src/knowledge-bases.ts +3 -2
- package/src/knowledge.ts +13 -3
- package/src/model-text.ts +5 -3
- package/src/pipeline/engine.ts +238 -0
- package/src/pipeline/store.ts +119 -0
- package/src/pipeline/types.ts +74 -0
- package/src/present.ts +10 -4
- package/src/runtime.ts +350 -110
- package/src/source-evidence.ts +1 -0
- package/src/store/axis-references.ts +3 -3
- package/src/store/migrate.ts +1 -1
- package/src/types.ts +3 -4
- package/src/wiki-materialize.ts +3 -14
- package/src/wiki-trial-types.ts +3 -0
- package/src/wiki-trial.ts +26 -7
package/src/runtime.ts
CHANGED
|
@@ -25,11 +25,15 @@ import { MAX_LOG_ITEMS, OperationLog, operationId } from './oplog.js'
|
|
|
25
25
|
import { ExtractionRunStore, extractionRunId, type ExtractionRunPage, type ExtractionRunRecord, type ExtractionRunSummary } from './extraction-runs.js'
|
|
26
26
|
import { AxisRegistry, normalizeAxisName, type AxisEntity, type AxisGuidance, type AxisGuidanceKind, type AxisKind, type AxisRelation, type AxisResolution, type AxisStatus } from './axis-registry.js'
|
|
27
27
|
import { KNOWLEDGE_STATE_LABEL, KnowledgeStore, type Knowledge, type KnowledgeDocumentImpact, type KnowledgeDocumentRemoval, type KnowledgeFacets, type KnowledgeIngest, type KnowledgeIngestResult, type KnowledgeQuery, type KnowledgeState, type KnowledgeVersion } from './knowledge.js'
|
|
28
|
-
import { compositeEventIdentity, eventEntityFromRecords, type EventEntity, type EventSourceRecord } from './events.js'
|
|
28
|
+
import { compositeEventIdentity, eventEntityFromRecords, stripEventReferenceSection, type EventEntity, type EventSourceRecord } from './events.js'
|
|
29
29
|
import type { TrialAxis, TrialEvent, TrialKnowledge, TrialSource, TrialTokenUsage } from './wiki-trial-types.js'
|
|
30
30
|
import { buildAliasIndex, candidateEntityId, extractCandidates, normalizeCandidates, RELATES_TO, type NormalizedCandidates } from './extract.js'
|
|
31
31
|
import { skeletonBody, type ModelCall } from './extract-model.js'
|
|
32
|
-
import { applyTrialAxisResolution, normalizeKnowledge, runWikiTrial, trialSummary, type AxisPromptBlock, type TrialAxisSet, type TrialAxisSetResolution } from './wiki-trial.js'
|
|
32
|
+
import { applyTrialAxisResolution, cardNameFor, normalizeKnowledge, runWikiTrial, trialSummary, type AxisPromptBlock, type TrialAxisSet, type TrialAxisSetResolution } from './wiki-trial.js'
|
|
33
|
+
import { EXTRACTION_CONTRACT_VERSION } from './extract-table.js'
|
|
34
|
+
import { KnowledgePipeline, type PipelineContext, type PipelineSource } from './pipeline/engine.js'
|
|
35
|
+
import { PipelineStore, type ExtractionCheckpoint } from './pipeline/store.js'
|
|
36
|
+
import type { MaterializationGroup, PipelineStatus } from './pipeline/types.js'
|
|
33
37
|
import { candidatesFromReport, coordinateIdsOf, coordinateIsEmpty, type MaterializedCard } from './wiki-materialize.js'
|
|
34
38
|
import {
|
|
35
39
|
buildOrganizePrompt, buildOrganizeSystem, checkOrganize, citationsFrom, layoutHash, layoutOf,
|
|
@@ -657,6 +661,8 @@ export interface ImportFailure {
|
|
|
657
661
|
* 而这里要回答的是「刚才那一下让库里多了什么」。
|
|
658
662
|
*/
|
|
659
663
|
export interface ImportCompileReport {
|
|
664
|
+
taskId?: string
|
|
665
|
+
state?: import('./pipeline/types.js').PipelineState
|
|
660
666
|
cardsCreated: number
|
|
661
667
|
cardsUpdated: number
|
|
662
668
|
cardsReused?: number
|
|
@@ -1046,38 +1052,6 @@ function appendExtracted(existing: string, additions: string[]): string {
|
|
|
1046
1052
|
return content
|
|
1047
1053
|
}
|
|
1048
1054
|
|
|
1049
|
-
/** 坐标 WikiCard 的事件摘要段由 Event 引用派生;全文与时间线留在独立 Event 实体。 */
|
|
1050
|
-
function withEventReferences(
|
|
1051
|
-
content: string,
|
|
1052
|
-
rawRefs: unknown,
|
|
1053
|
-
eventsById: ReadonlyMap<string, EventEntity>,
|
|
1054
|
-
): string {
|
|
1055
|
-
let refs: string[] = []
|
|
1056
|
-
if (Array.isArray(rawRefs)) refs = rawRefs.filter((item): item is string => typeof item === 'string')
|
|
1057
|
-
else if (typeof rawRefs === 'string') {
|
|
1058
|
-
try {
|
|
1059
|
-
const parsed: unknown = JSON.parse(rawRefs)
|
|
1060
|
-
if (Array.isArray(parsed)) refs = parsed.filter((item): item is string => typeof item === 'string')
|
|
1061
|
-
} catch { /* 无效引用会作为空列表处理 */ }
|
|
1062
|
-
}
|
|
1063
|
-
const lines = content.split('\n')
|
|
1064
|
-
const start = lines.findIndex(line => line.trim() === '## 关联 Event')
|
|
1065
|
-
if (start >= 0) {
|
|
1066
|
-
let end = start + 1
|
|
1067
|
-
while (end < lines.length && !/^#{1,2}\s+/.test(lines[end] ?? '')) end += 1
|
|
1068
|
-
lines.splice(start, end - start)
|
|
1069
|
-
}
|
|
1070
|
-
const base = lines.join('\n').trimEnd()
|
|
1071
|
-
const entries = [...new Set(refs)].map(id => eventsById.get(id)).filter((event): event is EventEntity => event !== undefined)
|
|
1072
|
-
if (entries.length === 0) return base
|
|
1073
|
-
const bullets = entries.map(event => {
|
|
1074
|
-
const occurredAt = event.occurredAt ? ` · ${event.occurredAt}` : ''
|
|
1075
|
-
const summary = event.summary.trim() ? `:${event.summary.trim()}` : ''
|
|
1076
|
-
return `- **${event.title}**${occurredAt}${summary}(${event.id})`
|
|
1077
|
-
})
|
|
1078
|
-
return `${base ? `${base}\n\n` : ''}## 关联 Event\n\n${bullets.join('\n')}`
|
|
1079
|
-
}
|
|
1080
|
-
|
|
1081
1055
|
/** 把 `incoming` 按小节折进 `existing`;结构对不上时整段追加。 */
|
|
1082
1056
|
function foldBody(existing: string, incoming: string): string {
|
|
1083
1057
|
if (!existing) return incoming
|
|
@@ -1355,9 +1329,10 @@ function markdownImageRefs(content: string): string[] {
|
|
|
1355
1329
|
* `tags` / `generated` / `sources`。它们是真 YAML 键,人能读、`grep` 得到、
|
|
1356
1330
|
* 外部工具能按公开规范读走。`generated.at` 就是**采集时间**(OKF 里没有「抓取
|
|
1357
1331
|
* 时间」这个概念,本文件何时被谁写出来就是它)。
|
|
1358
|
-
* 2. **我们自己的运行键**:`
|
|
1359
|
-
* `
|
|
1360
|
-
*
|
|
1332
|
+
* 2. **我们自己的运行键**:`created_at` / `updated_at` / `metadata` / `lineage`。
|
|
1333
|
+
* 来源身份保存在 SQLite 的 `entity_id` / `external_id` 列,不在 front matter 重复一份。
|
|
1334
|
+
* OKF 明确允许自定义键,且要求消费者不得因为未知键拒收,所以运行元数据可以同处
|
|
1335
|
+
* 一个 front matter。
|
|
1361
1336
|
*
|
|
1362
1337
|
* `metadata` 只剩**类型专属的扩展键**(转换任务 id、内容哈希、资产清单、格式…):
|
|
1363
1338
|
* 那些东西 OKF 没有对应概念,但也不该混进上面那组公开字段里。
|
|
@@ -1386,8 +1361,6 @@ function sourceFrontMatter(record: SourceRaw, provenance?: SourceProvenance): Re
|
|
|
1386
1361
|
...(origin.lastModified?.trim() ? { last_modified: origin.lastModified.trim() } : {}),
|
|
1387
1362
|
}))
|
|
1388
1363
|
}
|
|
1389
|
-
frontMatter.source_id = record.sourceId
|
|
1390
|
-
frontMatter.external_id = record.externalId ?? ''
|
|
1391
1364
|
frontMatter.created_at = record.createdAt
|
|
1392
1365
|
frontMatter.updated_at = record.updatedAt
|
|
1393
1366
|
frontMatter.metadata = record.metadata
|
|
@@ -1776,7 +1749,7 @@ function resolveConfiguredPython(configured: string | undefined, resourcePython:
|
|
|
1776
1749
|
|
|
1777
1750
|
export interface KnowledgeRuntimeOptions {
|
|
1778
1751
|
/** Production deployment requires a working Python + embedding + rerank stack. */
|
|
1779
|
-
environmentGuard?: () => Promise<void>
|
|
1752
|
+
environmentGuard?: (kind?: 'embedding' | 'rerank') => Promise<void>
|
|
1780
1753
|
/**
|
|
1781
1754
|
* 这一库的「坐标提取备注」(0068),人写的。
|
|
1782
1755
|
*
|
|
@@ -2000,7 +1973,7 @@ function blankProgress(
|
|
|
2000
1973
|
}
|
|
2001
1974
|
|
|
2002
1975
|
export class KnowledgeRuntime {
|
|
2003
|
-
private readonly environmentGuard?: () => Promise<void>
|
|
1976
|
+
private readonly environmentGuard?: (kind?: 'embedding' | 'rerank') => Promise<void>
|
|
2004
1977
|
/** 当前生效的配置。volatile 改动会就地合并进来(见 `updateConfig`),所以它不是只读的。 */
|
|
2005
1978
|
config: RuntimeKnowledgeConfig
|
|
2006
1979
|
readonly rootDir: string
|
|
@@ -2055,6 +2028,9 @@ export class KnowledgeRuntime {
|
|
|
2055
2028
|
/** 抽取轨迹(`<memoryDir>/extraction-runs.sqlite`)。同一个懒打开的理由。 */
|
|
2056
2029
|
private extractionRunsStore: ExtractionRunStore | null = null
|
|
2057
2030
|
private extractionRunsReady: boolean | null = null
|
|
2031
|
+
private closed = false
|
|
2032
|
+
private pipelineEngine?: KnowledgePipeline
|
|
2033
|
+
private pipelineOpening?: Promise<KnowledgePipeline>
|
|
2058
2034
|
/** 轴注册表(`<memoryDir>/knowledge.sqlite`)。与抽取轨迹同一个懒打开的理由。 */
|
|
2059
2035
|
private axesStore: AxisRegistry | null = null
|
|
2060
2036
|
private axesReady: boolean | null = null
|
|
@@ -2803,12 +2779,20 @@ export class KnowledgeRuntime {
|
|
|
2803
2779
|
}
|
|
2804
2780
|
|
|
2805
2781
|
close(): void {
|
|
2782
|
+
if (this.closed) return
|
|
2783
|
+
this.closed = true
|
|
2784
|
+
this.pipelineEngine?.close()
|
|
2806
2785
|
for (const task of this.compileTasks.values()) {
|
|
2807
2786
|
if (['succeeded', 'failed', 'cancelled'].includes(task.progress.state)) continue
|
|
2808
2787
|
task.cancelRequested = true
|
|
2809
2788
|
task.pauseRequested = false
|
|
2810
2789
|
task.resume?.()
|
|
2811
2790
|
}
|
|
2791
|
+
if (this.compilePending > 0) { void this.compileTail.finally(() => this.closeStores()); return }
|
|
2792
|
+
this.closeStores()
|
|
2793
|
+
}
|
|
2794
|
+
|
|
2795
|
+
private closeStores(): void {
|
|
2812
2796
|
this.embeddingWorker?.close()
|
|
2813
2797
|
this.embeddingWorker = null
|
|
2814
2798
|
this.rerankWorker?.close()
|
|
@@ -3198,34 +3182,23 @@ export class KnowledgeRuntime {
|
|
|
3198
3182
|
failed[0] ? `第一份的原因:${failed[0].message}` : undefined)
|
|
3199
3183
|
}
|
|
3200
3184
|
|
|
3201
|
-
/**
|
|
3202
|
-
* 编译。**只要有一份写成就编**:成功的那几份已经是库里的知识,
|
|
3203
|
-
* 「因为同批里有一份失败就不编」等于把已经进来的知识又挡在门外一次。
|
|
3204
|
-
*/
|
|
3205
3185
|
let compile: ImportCompileReport | null = null
|
|
3206
|
-
if (input.compile
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
warnings.push(`来源已写入,但编译失败:${rebuilt.error?.message ?? '未知原因'}`)
|
|
3213
|
-
} else {
|
|
3214
|
-
const data = rebuilt.data
|
|
3186
|
+
if (input.compile === true) {
|
|
3187
|
+
try {
|
|
3188
|
+
const before = new Map((await this.loadCards()).map(card => [card.entityId, this.cardHashOf(card)]))
|
|
3189
|
+
const pipeline = await this.pipeline()
|
|
3190
|
+
const task = await pipeline.start('all', { sourceIds: written.map(source => source.sourceId) }).completion
|
|
3191
|
+
const after = await this.loadCards()
|
|
3215
3192
|
compile = {
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
sourcesExtracted: data.sourcesExtracted,
|
|
3222
|
-
sourcesSkipped: data.sourcesSkipped,
|
|
3223
|
-
cardsRemoved: data.cardsRemoved,
|
|
3224
|
-
embedding: data.embedding,
|
|
3225
|
-
version: data.version,
|
|
3193
|
+
taskId: task.id, state: task.state,
|
|
3194
|
+
cardsCreated: after.filter(card => !before.has(card.entityId)).length,
|
|
3195
|
+
cardsUpdated: after.filter(card => before.has(card.entityId) && before.get(card.entityId) !== this.cardHashOf(card)).length,
|
|
3196
|
+
sourcesExtracted: Math.max(0, (task.results?.extract?.done ?? 0) - (task.results?.extract?.reused ?? 0) - (task.results?.extract?.failed ?? 0)),
|
|
3197
|
+
sourcesSkipped: task.results?.extract?.reused ?? 0,
|
|
3226
3198
|
}
|
|
3227
|
-
warnings.push(...
|
|
3228
|
-
|
|
3199
|
+
warnings.push(...task.warnings)
|
|
3200
|
+
if (task.state !== 'succeeded') warnings.push(`来源已保存;全链路未完成:${task.summary}。可在知识处理页面继续。`)
|
|
3201
|
+
} catch (cause) { warnings.push(`来源已保存;知识处理未完成:${cause instanceof Error ? cause.message : String(cause)}`) }
|
|
3229
3202
|
}
|
|
3230
3203
|
|
|
3231
3204
|
return success({ written, failed, attempted: items.length, compile }, warnings)
|
|
@@ -4329,6 +4302,7 @@ export class KnowledgeRuntime {
|
|
|
4329
4302
|
* 提示词与真编译发出去的那一份必须是**同一份**,否则预览这件事本身没有意义。
|
|
4330
4303
|
*/
|
|
4331
4304
|
private async organizeContext(input: {
|
|
4305
|
+
knowledge?: Knowledge[]
|
|
4332
4306
|
cardTitle: string
|
|
4333
4307
|
coordinate: CardCoordinate
|
|
4334
4308
|
profileId: string
|
|
@@ -4353,10 +4327,11 @@ export class KnowledgeRuntime {
|
|
|
4353
4327
|
const store = await this.knowledgeStore()
|
|
4354
4328
|
if (store === null) return { ok: false, reason: '知识库不可用,这次没有组织卡片(正文退回确定性布局)' }
|
|
4355
4329
|
|
|
4356
|
-
const listed = await store.list({
|
|
4330
|
+
const listed = input.knowledge ? { knowledge: input.knowledge, total: input.knowledge.length } : await store.list({
|
|
4357
4331
|
objectId: input.coordinate.objectId,
|
|
4358
4332
|
actionId: input.coordinate.actionId,
|
|
4359
4333
|
sceneId: input.coordinate.sceneId,
|
|
4334
|
+
excludeWithdrawn: true,
|
|
4360
4335
|
limit: MAX_ORGANIZE_KNOWLEDGE,
|
|
4361
4336
|
})
|
|
4362
4337
|
if (listed.knowledge.length === 0) {
|
|
@@ -4540,7 +4515,8 @@ export class KnowledgeRuntime {
|
|
|
4540
4515
|
if (call === undefined) return failure({ card, warnings: [] }, 'MODEL_UNAVAILABLE', '当前没有可用的模型,无法重新组织这张卡片', true, '检查模型配置后再试。')
|
|
4541
4516
|
|
|
4542
4517
|
const profile = this.organizeProfile()
|
|
4543
|
-
const
|
|
4518
|
+
const organizationCard = { ...card, content: stripEventReferenceSection(card.content) }
|
|
4519
|
+
const related = await this.relatedForCard(organizationCard)
|
|
4544
4520
|
const images = await this.imagesForCard(card)
|
|
4545
4521
|
const organized = await this.organizeCard({
|
|
4546
4522
|
cardTitle: card.title,
|
|
@@ -4551,7 +4527,7 @@ export class KnowledgeRuntime {
|
|
|
4551
4527
|
...(profile === undefined ? {} : { requiredSections: profile.sections.filter(section => section.required).map(section => section.label) }),
|
|
4552
4528
|
...(related.related === undefined ? {} : { related: related.related }),
|
|
4553
4529
|
...(images.images.length === 0 ? {} : { images: images.images }),
|
|
4554
|
-
existing:
|
|
4530
|
+
existing: organizationCard,
|
|
4555
4531
|
call,
|
|
4556
4532
|
})
|
|
4557
4533
|
if (organized.kind !== 'applied') {
|
|
@@ -4563,7 +4539,7 @@ export class KnowledgeRuntime {
|
|
|
4563
4539
|
entityType: card.entityType,
|
|
4564
4540
|
entityId: card.entityId,
|
|
4565
4541
|
title: organized.title,
|
|
4566
|
-
content: organized.content,
|
|
4542
|
+
content: stripEventReferenceSection(organized.content),
|
|
4567
4543
|
sourceRefs: card.sourceRefs,
|
|
4568
4544
|
summary: organized.summary,
|
|
4569
4545
|
status: card.status,
|
|
@@ -4645,6 +4621,7 @@ export class KnowledgeRuntime {
|
|
|
4645
4621
|
}
|
|
4646
4622
|
|
|
4647
4623
|
private async organizeCard(input: {
|
|
4624
|
+
knowledge?: Knowledge[]
|
|
4648
4625
|
cardTitle: string
|
|
4649
4626
|
coordinate: CardCoordinate
|
|
4650
4627
|
profileId: string
|
|
@@ -4670,6 +4647,7 @@ export class KnowledgeRuntime {
|
|
|
4670
4647
|
| { kind: 'skipped'; warnings: string[] }
|
|
4671
4648
|
> {
|
|
4672
4649
|
const context = await this.organizeContext({
|
|
4650
|
+
knowledge: input.knowledge,
|
|
4673
4651
|
cardTitle: input.cardTitle,
|
|
4674
4652
|
coordinate: input.coordinate,
|
|
4675
4653
|
profileId: input.profileId,
|
|
@@ -5019,7 +4997,7 @@ export class KnowledgeRuntime {
|
|
|
5019
4997
|
}
|
|
5020
4998
|
|
|
5021
4999
|
private compileCardNotes(card: KnowledgeCard): WikiNote[] {
|
|
5022
|
-
if (card.status === 'problem') return []
|
|
5000
|
+
if (card.status === 'problem' || card.status === 'deprecated') return []
|
|
5023
5001
|
const notes: WikiNote[] = []
|
|
5024
5002
|
/**
|
|
5025
5003
|
* 卡片文件所在目录:正文里的图片引用是相对它写的(`../../01_source_raw/_assets/…`),
|
|
@@ -5036,7 +5014,7 @@ export class KnowledgeRuntime {
|
|
|
5036
5014
|
* `metadata.related_cards`;这里继续剥离旧版本正文里的标记,保证存量卡片迁移期间
|
|
5037
5015
|
* 也不会把导航送进召回。
|
|
5038
5016
|
*/
|
|
5039
|
-
const recallBody = stripRelatedSection(card.content)
|
|
5017
|
+
const recallBody = stripEventReferenceSection(stripRelatedSection(card.content))
|
|
5040
5018
|
for (const [index, chunk] of splitWikiNotes({ ...card, content: recallBody }).entries()) {
|
|
5041
5019
|
const baseId = `${card.entityType}:${card.entityId}`
|
|
5042
5020
|
const unitId = index === 0 ? baseId : `${baseId}#${index}`
|
|
@@ -5100,7 +5078,7 @@ export class KnowledgeRuntime {
|
|
|
5100
5078
|
private async rerankNotes(query: string, hits: RecallHit[], threshold: number): Promise<{ hits: RecallHit[]; result: RerankResult; scored?: Array<{ unitId: string; logit: number; rerankScore: number; kept: boolean }>; warning?: string }> {
|
|
5101
5079
|
const provider = this.config.rerankProvider
|
|
5102
5080
|
if (provider === 'none') return { hits, result: { status: 'skipped', provider, threshold } }
|
|
5103
|
-
if (hits.length
|
|
5081
|
+
if (hits.length === 0) return { hits, result: { status: 'skipped', provider, model: this.rerankModelPath, scored: hits.length, threshold } }
|
|
5104
5082
|
if (this.rerankUnavailable) return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error: this.rerankUnavailable, threshold }, warning: `重排不可用,已按融合分数排序:${this.rerankUnavailable}` }
|
|
5105
5083
|
try {
|
|
5106
5084
|
if (!this.rerankWorker) this.rerankWorker = this.spawnWorker('RERANK')
|
|
@@ -5345,7 +5323,6 @@ export class KnowledgeRuntime {
|
|
|
5345
5323
|
*/
|
|
5346
5324
|
const profile = this.organizeProfile()
|
|
5347
5325
|
const profileId = profile?.id ?? 'concept'
|
|
5348
|
-
const eventsById = new Map((await this.loadEvents()).map(event => [event.id, event]))
|
|
5349
5326
|
for (const item of pending.values()) {
|
|
5350
5327
|
if (runId) await this.checkpointCompileTask(runId)
|
|
5351
5328
|
const key = `${item.entityType}:${item.entityId}`
|
|
@@ -5394,7 +5371,7 @@ export class KnowledgeRuntime {
|
|
|
5394
5371
|
&& layoutOf(existing.metadata)?.origin === 'model'
|
|
5395
5372
|
const finalTitle = preserveModelLayout ? existing.title : item.title
|
|
5396
5373
|
const contentBeforeEvents = preserveModelLayout ? existing.content : content
|
|
5397
|
-
const finalContent =
|
|
5374
|
+
const finalContent = stripEventReferenceSection(contentBeforeEvents)
|
|
5398
5375
|
const finalSummary = preserveModelLayout ? existing.summary : item.summary
|
|
5399
5376
|
let finalPlacements = preserveModelLayout || item.placements === undefined
|
|
5400
5377
|
? undefined
|
|
@@ -5478,7 +5455,7 @@ export class KnowledgeRuntime {
|
|
|
5478
5455
|
outcome.cardsUpdated += 1
|
|
5479
5456
|
for (const warning of written.warnings ?? []) outcome.warnings.push(`卡片 ${card.entityId}:${warning}`)
|
|
5480
5457
|
}
|
|
5481
|
-
// WikiCard
|
|
5458
|
+
// WikiCard 只由 Knowledge 物化;Event 作为独立事实按坐标挂引用。
|
|
5482
5459
|
const eventSync = await this.syncEventReferences()
|
|
5483
5460
|
outcome.cardsUpdated += eventSync.updated
|
|
5484
5461
|
outcome.warnings.push(...eventSync.warnings)
|
|
@@ -5540,12 +5517,13 @@ export class KnowledgeRuntime {
|
|
|
5540
5517
|
* 用的输入与收尾阶段那个相关块**完全相同**(同一份 `relations` + 同一个坐标投影),
|
|
5541
5518
|
* 所以「正文里能提到谁」与「末尾相关块列了谁」不会是两个答案。
|
|
5542
5519
|
*/
|
|
5520
|
+
const cardsWithoutLegacyEventSections = input.cards.map(card => ({ ...card, content: stripEventReferenceSection(card.content) }))
|
|
5543
5521
|
const relatedPlan = planRelated(
|
|
5544
|
-
|
|
5522
|
+
cardsWithoutLegacyEventSections,
|
|
5545
5523
|
input.relations,
|
|
5546
5524
|
await this.projectedRelated(input.cards),
|
|
5547
5525
|
)
|
|
5548
|
-
const withCoordinate =
|
|
5526
|
+
const withCoordinate = cardsWithoutLegacyEventSections
|
|
5549
5527
|
.filter(card => card.coordinate !== undefined)
|
|
5550
5528
|
.sort((left, right) => left.entityId.localeCompare(right.entityId))
|
|
5551
5529
|
if (withCoordinate.length === 0) return 0
|
|
@@ -6170,7 +6148,7 @@ export class KnowledgeRuntime {
|
|
|
6170
6148
|
const projected = await this.projectedRelated(cards)
|
|
6171
6149
|
const related = planRelated(cards, relations, projected)
|
|
6172
6150
|
/** 统计清理旧导航块之后的正文断链;相关卡片本身不再伪装成正文链接。 */
|
|
6173
|
-
const nextCards = cards.map(card => ({ ...card, content: stripRelatedSection(card.content) }))
|
|
6151
|
+
const nextCards = cards.map(card => ({ ...card, content: stripEventReferenceSection(stripRelatedSection(card.content)) }))
|
|
6174
6152
|
const deadLinks = deadLinksOf(nextCards)
|
|
6175
6153
|
let rewritten = 0
|
|
6176
6154
|
for (const [index, card] of cards.entries()) {
|
|
@@ -6853,6 +6831,7 @@ export class KnowledgeRuntime {
|
|
|
6853
6831
|
* 不能当成「没命中」。
|
|
6854
6832
|
*/
|
|
6855
6833
|
async resolveTrialAxisSet(input: {
|
|
6834
|
+
registry?: AxisRegistry
|
|
6856
6835
|
axes: TrialAxisSet
|
|
6857
6836
|
axesBlock?: AxisPromptBlock
|
|
6858
6837
|
runId: string
|
|
@@ -6875,6 +6854,7 @@ export class KnowledgeRuntime {
|
|
|
6875
6854
|
candidates: Candidate[]
|
|
6876
6855
|
}
|
|
6877
6856
|
type Judgment = { decision: 'match' | 'new' | 'uncertain'; candidateId: string }
|
|
6857
|
+
const resolve = (value: Parameters<KnowledgeRuntime['resolveAxisMention']>[0]) => this.resolveAxisMention({ ...value, registry: input.registry })
|
|
6878
6858
|
const groups: Array<{ kind: AxisKind; key: GroupKey }> = [
|
|
6879
6859
|
{ kind: 'object', key: 'objects' },
|
|
6880
6860
|
{ kind: 'action', key: 'actions' },
|
|
@@ -6885,7 +6865,7 @@ export class KnowledgeRuntime {
|
|
|
6885
6865
|
const axisKey = (kind: AxisKind, mention: string): string => `${kind}\u0000${normalizeAxisName(mention)}`
|
|
6886
6866
|
const supplementalEntries: Record<AxisKind, Candidate[]> = { object: [], action: [], scene: [] }
|
|
6887
6867
|
try {
|
|
6888
|
-
const registry = await this.axisRegistry()
|
|
6868
|
+
const registry = input.registry ?? await this.axisRegistry()
|
|
6889
6869
|
if (registry !== null) {
|
|
6890
6870
|
// 初次抽取的候选清单可能被截断。再按本批轴 mention 做一次本地召回,补进未下发的已有坐标。
|
|
6891
6871
|
const query = groups.flatMap(group => input.axes[group.key].flatMap(axis => [
|
|
@@ -6935,7 +6915,7 @@ export class KnowledgeRuntime {
|
|
|
6935
6915
|
const key = axisKey(group.kind, axis.name)
|
|
6936
6916
|
let lookup: AxisResolution | null
|
|
6937
6917
|
try {
|
|
6938
|
-
lookup = await
|
|
6918
|
+
lookup = await resolve({
|
|
6939
6919
|
kind: group.kind,
|
|
6940
6920
|
mention: axis.name,
|
|
6941
6921
|
parentMention: axis.parentMention,
|
|
@@ -6998,7 +6978,7 @@ export class KnowledgeRuntime {
|
|
|
6998
6978
|
}
|
|
6999
6979
|
|
|
7000
6980
|
// 没有可供模型比较的现有候选时,沿用注册表的新建候选路径。
|
|
7001
|
-
const created = await
|
|
6981
|
+
const created = await resolve({
|
|
7002
6982
|
kind: group.kind,
|
|
7003
6983
|
mention: axis.name,
|
|
7004
6984
|
parentMention: axis.parentMention,
|
|
@@ -7101,7 +7081,7 @@ export class KnowledgeRuntime {
|
|
|
7101
7081
|
const key = axisKey(item.kind, item.axis.name)
|
|
7102
7082
|
const judgment = decisions.get(key)
|
|
7103
7083
|
if (judgment?.decision === 'new') {
|
|
7104
|
-
const resolution = await
|
|
7084
|
+
const resolution = await resolve({
|
|
7105
7085
|
kind: item.kind,
|
|
7106
7086
|
mention: item.axis.name,
|
|
7107
7087
|
parentMention: item.axis.parentMention,
|
|
@@ -7115,7 +7095,7 @@ export class KnowledgeRuntime {
|
|
|
7115
7095
|
if (judgment?.decision === 'match') {
|
|
7116
7096
|
const candidate = item.candidates.find(value => value.id === judgment.candidateId)
|
|
7117
7097
|
if (candidate) {
|
|
7118
|
-
const resolution = await
|
|
7098
|
+
const resolution = await resolve({
|
|
7119
7099
|
kind: item.kind,
|
|
7120
7100
|
mention: item.axis.name,
|
|
7121
7101
|
parentMention: item.axis.parentMention,
|
|
@@ -7146,6 +7126,7 @@ export class KnowledgeRuntime {
|
|
|
7146
7126
|
}
|
|
7147
7127
|
|
|
7148
7128
|
async resolveAxisMention(input: {
|
|
7129
|
+
registry?: AxisRegistry
|
|
7149
7130
|
kind: AxisKind
|
|
7150
7131
|
mention: string
|
|
7151
7132
|
parentMention?: string
|
|
@@ -7163,7 +7144,7 @@ export class KnowledgeRuntime {
|
|
|
7163
7144
|
/** 模型给这一轮的定义(0068)。只在新建、或给一个还没有定义的实体补一次。 */
|
|
7164
7145
|
description?: string
|
|
7165
7146
|
}): Promise<AxisResolution | null> {
|
|
7166
|
-
const registry = await this.axisRegistry()
|
|
7147
|
+
const registry = input.registry ?? await this.axisRegistry()
|
|
7167
7148
|
if (registry === null) return null
|
|
7168
7149
|
const runId = input.runId ?? ''
|
|
7169
7150
|
let parentId = ''
|
|
@@ -7177,7 +7158,7 @@ export class KnowledgeRuntime {
|
|
|
7177
7158
|
})
|
|
7178
7159
|
if (parent.entity) parentId = parent.entity.id
|
|
7179
7160
|
}
|
|
7180
|
-
const preferId = await this.axisIdByName(input.kind, input.preferMention)
|
|
7161
|
+
const preferId = await this.axisIdByName(input.kind, input.preferMention, registry)
|
|
7181
7162
|
return registry.resolve({
|
|
7182
7163
|
kind: input.kind, mention: input.mention,
|
|
7183
7164
|
...(parentId ? { parentId } : {}), ...(runId ? { runId } : {}),
|
|
@@ -7194,10 +7175,10 @@ export class KnowledgeRuntime {
|
|
|
7194
7175
|
* 「是已有的某个东西」而那个名字在库里指两个实体时,把它当成指认就等于替人做了
|
|
7195
7176
|
* 「哪一个」的决定 —— 而注册表的规矩是「不唯一就不猜」。
|
|
7196
7177
|
*/
|
|
7197
|
-
private async axisIdByName(kind: AxisKind, mention: string | undefined): Promise<string | undefined> {
|
|
7178
|
+
private async axisIdByName(kind: AxisKind, mention: string | undefined, supplied?: AxisRegistry): Promise<string | undefined> {
|
|
7198
7179
|
const name = (mention ?? '').trim()
|
|
7199
7180
|
if (!name) return undefined
|
|
7200
|
-
const registry = await this.axisRegistry()
|
|
7181
|
+
const registry = supplied ?? await this.axisRegistry()
|
|
7201
7182
|
if (registry === null) return undefined
|
|
7202
7183
|
try {
|
|
7203
7184
|
const probe = await registry.resolve({ kind, mention: name, create: false })
|
|
@@ -7884,14 +7865,12 @@ export class KnowledgeRuntime {
|
|
|
7884
7865
|
refs.push(event.id)
|
|
7885
7866
|
eventsByCoordinate.set(key, refs)
|
|
7886
7867
|
}
|
|
7887
|
-
const eventIndex = new Map(events.map(event => [event.id, event]))
|
|
7888
7868
|
const warnings: string[] = []
|
|
7889
7869
|
let updated = 0
|
|
7890
7870
|
for (const card of await this.loadCards()) {
|
|
7891
|
-
const refs = card.coordinate === undefined
|
|
7871
|
+
const refs = card.status === 'deprecated' || card.coordinate === undefined || card.entityType === 'diagnosis/case'
|
|
7892
7872
|
? []
|
|
7893
7873
|
: (eventsByCoordinate.get(coordinateKeyOf(card.coordinate)) ?? []).sort()
|
|
7894
|
-
db.setCardEvents(card.entityId, refs)
|
|
7895
7874
|
const oldRaw = card.metadata.event_refs
|
|
7896
7875
|
let oldRefs: string[] = []
|
|
7897
7876
|
let oldRefsValid = false
|
|
@@ -7907,11 +7886,17 @@ export class KnowledgeRuntime {
|
|
|
7907
7886
|
oldRefs = oldRaw.filter((item): item is string => typeof item === 'string').sort()
|
|
7908
7887
|
oldRefsValid = true
|
|
7909
7888
|
}
|
|
7910
|
-
const content =
|
|
7889
|
+
const content = stripEventReferenceSection(card.content)
|
|
7911
7890
|
const contentChanged = content !== card.content
|
|
7912
7891
|
const refsChanged = !oldRefsValid || JSON.stringify(refs) !== JSON.stringify(oldRefs)
|
|
7913
|
-
if (!refsChanged && !contentChanged)
|
|
7914
|
-
|
|
7892
|
+
if (!refsChanged && !contentChanged) {
|
|
7893
|
+
db.setCardEvents(card.entityId, refs)
|
|
7894
|
+
continue
|
|
7895
|
+
}
|
|
7896
|
+
if (!oldRefsValid && refs.length === 0 && oldRaw === undefined && !contentChanged) {
|
|
7897
|
+
db.setCardEvents(card.entityId, refs)
|
|
7898
|
+
continue
|
|
7899
|
+
}
|
|
7915
7900
|
const written = await this.writeCard({
|
|
7916
7901
|
...card,
|
|
7917
7902
|
sourceRefs: card.sourceRefs,
|
|
@@ -7919,8 +7904,11 @@ export class KnowledgeRuntime {
|
|
|
7919
7904
|
metadata: { ...card.metadata, event_refs: JSON.stringify(refs) },
|
|
7920
7905
|
origin: 'internal',
|
|
7921
7906
|
})
|
|
7922
|
-
if (written.ok === false) warnings.push(
|
|
7923
|
-
else
|
|
7907
|
+
if (written.ok === false) warnings.push(`WikiCard ${card.title} 的 Event 引用未更新:${written.error.message}`)
|
|
7908
|
+
else {
|
|
7909
|
+
db.setCardEvents(card.entityId, refs)
|
|
7910
|
+
updated += 1
|
|
7911
|
+
}
|
|
7924
7912
|
}
|
|
7925
7913
|
return { updated, warnings }
|
|
7926
7914
|
}
|
|
@@ -7959,10 +7947,6 @@ export class KnowledgeRuntime {
|
|
|
7959
7947
|
|
|
7960
7948
|
for (const event of db.events()) {
|
|
7961
7949
|
const remaining = event.records.filter(record => record.sourceId !== input.sourceId)
|
|
7962
|
-
if (remaining.length === 0) {
|
|
7963
|
-
db.removeEvent(event.id)
|
|
7964
|
-
continue
|
|
7965
|
-
}
|
|
7966
7950
|
const state: EventState = {
|
|
7967
7951
|
id: event.id,
|
|
7968
7952
|
identityKey: event.identityKey,
|
|
@@ -8063,7 +8047,8 @@ export class KnowledgeRuntime {
|
|
|
8063
8047
|
}
|
|
8064
8048
|
|
|
8065
8049
|
for (const state of states.values()) {
|
|
8066
|
-
if (!state.dirty
|
|
8050
|
+
if (!state.dirty) continue
|
|
8051
|
+
if (state.records.length === 0) { db.removeEvent(state.id); continue }
|
|
8067
8052
|
const records = state.records
|
|
8068
8053
|
const entity = eventEntityFromRecords({
|
|
8069
8054
|
id: state.id,
|
|
@@ -8084,6 +8069,256 @@ export class KnowledgeRuntime {
|
|
|
8084
8069
|
return { saved, merged, skipped, ids, warnings }
|
|
8085
8070
|
}
|
|
8086
8071
|
|
|
8072
|
+
/** The stage runner shares this base's existing writer queue and audit database. */
|
|
8073
|
+
async pipeline(): Promise<KnowledgePipeline> {
|
|
8074
|
+
if (this.closed) throw new Error('知识库已关闭')
|
|
8075
|
+
this.pipelineOpening ??= (async () => {
|
|
8076
|
+
const db = await this.storeHandle()
|
|
8077
|
+
const audit = await this.extractionRuns()
|
|
8078
|
+
if (this.closed) throw new Error('知识库已关闭')
|
|
8079
|
+
if (!db || !audit || !await this.knowledgeStore()) throw new Error('知识库或任务历史不可用,无法启动处理任务')
|
|
8080
|
+
const store = new PipelineStore(db.connection(), audit.connection())
|
|
8081
|
+
const engine = new KnowledgePipeline(store, {
|
|
8082
|
+
capture: () => ({ model: this.modelSeamNow(), note: this.extractionNote(), profile: structuredClone(this.organizeProfile()), organize: this.config.cardOrganize, format: 'table' }),
|
|
8083
|
+
serialize: work => {
|
|
8084
|
+
this.compilePending += 1
|
|
8085
|
+
const run = this.compileTail.then(work, work)
|
|
8086
|
+
this.compileTail = run.then(() => undefined, () => undefined).finally(() => { this.compilePending -= 1 })
|
|
8087
|
+
return run
|
|
8088
|
+
},
|
|
8089
|
+
sources: () => this.pipelineSources(),
|
|
8090
|
+
fingerprint: (source, context) => contentHashOf(JSON.stringify({ source: source.sourceHash, title: source.title, contract: EXTRACTION_CONTRACT_VERSION, note: context.note, format: context.format, model: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort } })),
|
|
8091
|
+
extract: async (source, context, onDelta) => {
|
|
8092
|
+
if (!context.model.call) throw new Error(`抽取模型不可用:${context.model.why ?? '请在编译设置中选择模型'}`)
|
|
8093
|
+
const axes = await this.axisPromptBlock(source.content)
|
|
8094
|
+
return runWikiTrial({ title: source.title, text: source.content, docId: source.sourceId, axes, note: context.note, call: context.model.call, modelRequest: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort }, outputFormat: context.format, sessionId: `knowledge:${this.baseId || 'default'}`, onDelta })
|
|
8095
|
+
},
|
|
8096
|
+
normalize: (source, checkpoint, context, force, output) => this.normalizePipelineSource(store, source, checkpoint, context, force, output),
|
|
8097
|
+
groups: context => this.materializationGroups(store, context),
|
|
8098
|
+
materialize: (group, context, output) => this.materializePipelineGroup(store, group, context, output),
|
|
8099
|
+
build: async (taskId, checkpoint) => {
|
|
8100
|
+
let result: Envelope<RebuildReport>
|
|
8101
|
+
try { result = await this.compileOnce(taskId, 'all', undefined, { extract: false, checkpoint }) }
|
|
8102
|
+
catch (cause) { await this.settleProgressFailure(taskId, cause); throw cause }
|
|
8103
|
+
this.settleProgressReport(taskId, result)
|
|
8104
|
+
if (!result.ok) throw new Error(result.error.message)
|
|
8105
|
+
return `检索索引已发布:v${result.data.version}`
|
|
8106
|
+
},
|
|
8107
|
+
record: async checkpoint => {
|
|
8108
|
+
const audit = await this.recordExtractionRun({ id: checkpoint.id, docId: checkpoint.sourceId, title: checkpoint.report.title, summary: trialSummary(checkpoint.report), report: checkpoint.report })
|
|
8109
|
+
if (!audit.saved) throw new Error(`抽取报告保存失败:${audit.why}`)
|
|
8110
|
+
},
|
|
8111
|
+
})
|
|
8112
|
+
this.pipelineEngine = engine
|
|
8113
|
+
if (this.closed) engine.close()
|
|
8114
|
+
return engine
|
|
8115
|
+
})()
|
|
8116
|
+
return this.pipelineOpening
|
|
8117
|
+
}
|
|
8118
|
+
|
|
8119
|
+
private async pipelineSources(): Promise<PipelineSource[]> {
|
|
8120
|
+
const db = await this.storeHandle()
|
|
8121
|
+
if (!db) throw new Error('来源存储不可用')
|
|
8122
|
+
return db.sources().map(row => ({ ...row.record, version: row.version, sourceHash: contentHashOf(row.record.content) })).sort((a, b) => a.sourceId.localeCompare(b.sourceId))
|
|
8123
|
+
}
|
|
8124
|
+
|
|
8125
|
+
private async normalizationFingerprint(checkpoint: ExtractionCheckpoint, context: PipelineContext): Promise<string> {
|
|
8126
|
+
const registry = await this.axisRegistry()
|
|
8127
|
+
const report = checkpoint.normalized?.report ?? checkpoint.report
|
|
8128
|
+
const query = [...report.objects, ...report.actions, ...report.scenes].flatMap(axis => [axis.name, axis.existing ?? '', axis.definition ?? '', axis.parentMention ?? '']).join('\n')
|
|
8129
|
+
const candidates = await registry?.promptEntries(24, query)
|
|
8130
|
+
const mentions = [...checkpoint.report.objects, ...checkpoint.report.actions, ...checkpoint.report.scenes].flatMap(axis => [axis.name, axis.existing ?? '', axis.parentMention ?? '']).map(normalizeAxisName).filter(Boolean)
|
|
8131
|
+
const resolvedIds = new Set([...report.objects, ...report.actions, ...report.scenes].map(axis => axis.id).filter(Boolean))
|
|
8132
|
+
// Ignore unrelated filler rows and registry recency; retain candidates that can affect these mentions.
|
|
8133
|
+
const relevant = Object.fromEntries(Object.entries(candidates?.entries ?? {}).map(([kind, entries]) => [kind, entries.filter(entry => resolvedIds.has(entry.id) || mentions.some(mention => [entry.name, ...entry.aliases, entry.description].map(normalizeAxisName).some(value => value.includes(mention) || (value.length > 1 && mention.includes(value))))).sort((a, b) => a.id.localeCompare(b.id))]))
|
|
8134
|
+
return contentHashOf(JSON.stringify({ extraction: checkpoint.id, model: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort }, candidates: relevant }))
|
|
8135
|
+
}
|
|
8136
|
+
|
|
8137
|
+
private async normalizePipelineSource(store: PipelineStore, source: PipelineSource, checkpoint: ExtractionCheckpoint, context: PipelineContext, force: boolean, output: (text: string) => void): Promise<{ reused: boolean; warnings: string[] }> {
|
|
8138
|
+
const fingerprint = await this.normalizationFingerprint(checkpoint, context)
|
|
8139
|
+
if (!force && checkpoint.normalized?.fingerprint === fingerprint && store.head(source.sourceId).normalized?.id === checkpoint.id) return { reused: true, warnings: [] }
|
|
8140
|
+
if (!checkpoint.report.model.response) throw new Error('抽取记录没有原始返回,无法归一;请重新抽取该文档')
|
|
8141
|
+
const registry = await this.axisRegistry()
|
|
8142
|
+
if (!registry) throw new Error('实体注册表不可用')
|
|
8143
|
+
const draft = await registry.draft()
|
|
8144
|
+
try {
|
|
8145
|
+
const axesBlock = await this.axisPromptBlock(source.content)
|
|
8146
|
+
const call: ModelCall | undefined = context.model.call ? request => context.model.call!({ ...request, onDelta: output }) : undefined
|
|
8147
|
+
const report = await runWikiTrial({
|
|
8148
|
+
title: source.title, text: source.content, docId: source.sourceId, axes: axesBlock,
|
|
8149
|
+
// Reparse the captured response; this callback never requests another extraction.
|
|
8150
|
+
call: async () => checkpoint.report.model.response!, modelRequest: checkpoint.report.model.request,
|
|
8151
|
+
resolveAxisSet: axes => this.resolveTrialAxisSet({ registry: draft.registry, axes, axesBlock, runId: checkpoint.id, call, modelRequest: context.model }),
|
|
8152
|
+
})
|
|
8153
|
+
if (!report.model.parsed) throw new Error(report.model.why ?? '抽取结果无法归一')
|
|
8154
|
+
const db = await this.storeHandle()
|
|
8155
|
+
const current = db?.source(source.sourceId)
|
|
8156
|
+
if (this.closed) throw new Error('知识库已关闭,本次归一未提交')
|
|
8157
|
+
if (!db || !current || current.version !== source.version || contentHashOf(current.record.content) !== source.sourceHash) throw new Error('来源在归一期间已变化,请重新抽取最新版本')
|
|
8158
|
+
const sql = db.connection()
|
|
8159
|
+
sql.exec('SAVEPOINT pipeline_normalize')
|
|
8160
|
+
try {
|
|
8161
|
+
draft.commit()
|
|
8162
|
+
// Preserve evidence history while replacing this source's active contribution.
|
|
8163
|
+
sql.prepare('UPDATE knowledge_sources SET withdrawn=1 WHERE document_id=?').run(source.sourceId)
|
|
8164
|
+
const knowledge = await this.ingestTrialKnowledge({ runId: checkpoint.id, knowledge: report.knowledge, objects: report.objects, actions: report.actions, scenes: report.scenes, sourceVersions: { [source.sourceId]: source.version } })
|
|
8165
|
+
const events = await this.ingestTrialEvents({ sourceId: source.sourceId, runId: checkpoint.id, events: report.events, objects: report.objects, actions: report.actions, scenes: report.scenes })
|
|
8166
|
+
if (knowledge.skipped || events.skipped) throw new Error(`出处核验或写入失败:${knowledge.skipped} 条 Knowledge、${events.skipped} 个 Event;本次归一没有提交`)
|
|
8167
|
+
const unresolvedKnowledgeIds = report.knowledge.filter(item => ([['object', report.objects], ['action', report.actions], ['scene', report.scenes]] as const).some(([kind, axes]) => {
|
|
8168
|
+
const name = item[kind]
|
|
8169
|
+
const axis = name ? axes.find(axis => axis.name === name) : undefined
|
|
8170
|
+
return name !== null && (!axis || axis.outcome === 'ambiguous' || axis.outcome === 'no-registry')
|
|
8171
|
+
})).map(item => knowledge.ids[item.id]!).filter(Boolean)
|
|
8172
|
+
const normalized = { report, knowledgeIds: knowledge.ids, eventIds: events.ids, unresolvedKnowledgeIds, fingerprint: '' }
|
|
8173
|
+
// Store the post-resolution candidate snapshot so learning its own entities does not invalidate this run.
|
|
8174
|
+
normalized.fingerprint = await this.normalizationFingerprint({ ...checkpoint, normalized }, context)
|
|
8175
|
+
store.saveNormalization(checkpoint, normalized)
|
|
8176
|
+
sql.exec('RELEASE pipeline_normalize')
|
|
8177
|
+
// Updating Event references is a cheap relation refresh. It does not reorganize or index card content.
|
|
8178
|
+
const eventReferences = await this.syncEventReferences()
|
|
8179
|
+
return { reused: false, warnings: [...events.warnings, ...eventReferences.warnings, ...(unresolvedKnowledgeIds.length ? [`${unresolvedKnowledgeIds.length} 条 Knowledge 的坐标仍有歧义,保留事实与出处,暂不物化。`] : [])] }
|
|
8180
|
+
} catch (cause) { sql.exec('ROLLBACK TO pipeline_normalize; RELEASE pipeline_normalize'); throw cause }
|
|
8181
|
+
} finally { draft.close() }
|
|
8182
|
+
}
|
|
8183
|
+
|
|
8184
|
+
private async activePipelineKnowledge(): Promise<Knowledge[]> {
|
|
8185
|
+
const knowledge = await this.knowledgeStore()
|
|
8186
|
+
const db = await this.storeHandle()
|
|
8187
|
+
if (!knowledge || !db) return []
|
|
8188
|
+
const rows: Knowledge[] = []
|
|
8189
|
+
for (let offset = 0;; offset += 500) {
|
|
8190
|
+
const page = await knowledge.list({ excludeWithdrawn: true, limit: 500, offset })
|
|
8191
|
+
rows.push(...page.knowledge.filter(row => row.status !== 'deprecated').map(row => assessKnowledgeEvidence(row, db)).map(row => ({ ...row, sources: row.sources.filter(source => !source.withdrawn && source.evidenceStatus !== 'missing') })).filter(row => row.sources.length > 0))
|
|
8192
|
+
if (offset + page.knowledge.length >= page.total) break
|
|
8193
|
+
}
|
|
8194
|
+
return rows
|
|
8195
|
+
}
|
|
8196
|
+
|
|
8197
|
+
private async materializationGroups(store: PipelineStore, context: PipelineContext): Promise<{ groups: MaterializationGroup[]; blocked: number }> {
|
|
8198
|
+
const unresolved = store.unresolvedKnowledge()
|
|
8199
|
+
const groups = new Map<string, { coordinate: CardCoordinate; knowledge: Knowledge[] }>()
|
|
8200
|
+
let blocked = 0
|
|
8201
|
+
const groupFor = (coordinate: CardCoordinate) => {
|
|
8202
|
+
const key = coordinateKeyOf(coordinate)
|
|
8203
|
+
if (!groups.has(key)) groups.set(key, { coordinate, knowledge: [] })
|
|
8204
|
+
return groups.get(key)!
|
|
8205
|
+
}
|
|
8206
|
+
for (const row of await this.activePipelineKnowledge()) {
|
|
8207
|
+
const coordinate = { objectId: row.objectId, actionId: row.actionId, sceneId: row.sceneId }
|
|
8208
|
+
if (coordinateIsEmpty(coordinate)) { blocked += 1; continue }
|
|
8209
|
+
groupFor(coordinate).knowledge.push(row)
|
|
8210
|
+
}
|
|
8211
|
+
const db = await this.storeHandle()
|
|
8212
|
+
const eventsByCoordinate = new Map<string, EventEntity[]>()
|
|
8213
|
+
for (const event of db?.events() ?? []) {
|
|
8214
|
+
const coordinate = event.coordinateIds
|
|
8215
|
+
if (!coordinate.objectId || !coordinate.actionId || !coordinate.sceneId) continue
|
|
8216
|
+
const key = coordinateKeyOf(coordinate)
|
|
8217
|
+
const bucket = eventsByCoordinate.get(key) ?? []
|
|
8218
|
+
bucket.push(event)
|
|
8219
|
+
eventsByCoordinate.set(key, bucket)
|
|
8220
|
+
}
|
|
8221
|
+
const cards = await this.loadCards()
|
|
8222
|
+
const byCoordinate = new Map(cards.filter(card => card.coordinate).map(card => [coordinateKeyOf(card.coordinate!), card]))
|
|
8223
|
+
const result: MaterializationGroup[] = []
|
|
8224
|
+
for (const [key, value] of groups) {
|
|
8225
|
+
const names = await this.coordinateNames(value.coordinate)
|
|
8226
|
+
const existing = byCoordinate.get(key)
|
|
8227
|
+
// Event 只作为卡片引用;它的增删和正文变化不得触发 Knowledge 重新组织。
|
|
8228
|
+
const fingerprint = contentHashOf(JSON.stringify({ knowledge: value.knowledge.map(row => ({ id: row.id, text: row.text, status: row.status, sources: row.sources.filter(source => !source.withdrawn) })).sort((a, b) => a.id.localeCompare(b.id)), profile: context.profile, organize: context.organize, model: context.organize ? { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort } : undefined }))
|
|
8229
|
+
const receipt = store.materialization(key)
|
|
8230
|
+
const current = existing && receipt?.fingerprint === fingerprint && receipt.cardId === existing.entityId
|
|
8231
|
+
const contributingSources = [...new Set(value.knowledge.flatMap(row => row.sources.map(source => source.documentId)))].filter(Boolean).sort()
|
|
8232
|
+
const needsNormalization = value.knowledge.some(row => unresolved.ids.has(row.id)) || contributingSources.some(id => {
|
|
8233
|
+
const source = db?.source(id)
|
|
8234
|
+
const head = store.head(id)
|
|
8235
|
+
return head.normalized ? head.normalized.sourceVersion !== source?.version || head.extracted?.id !== head.normalized.id : value.knowledge.some(row => row.sources.some(evidence => evidence.documentId === id && evidence.toVersion !== undefined && evidence.toVersion !== source?.version))
|
|
8236
|
+
})
|
|
8237
|
+
if (needsNormalization) blocked += value.knowledge.length
|
|
8238
|
+
result.push({ key, title: existing?.title ?? cardNameFor(names.object || null, names.action || null, names.scene || null), coordinate: value.coordinate, knowledgeCount: value.knowledge.length, eventCount: eventsByCoordinate.get(key)?.length ?? 0, sourceIds: contributingSources, ...(existing ? { cardId: existing.entityId } : {}), state: needsNormalization ? 'blocked' : current ? 'current' : existing ? 'changed' : 'new', ...(needsNormalization ? { reason: '分组中的来源已变化、新抽取待归一或坐标有歧义,请先完成归一' } : {}), fingerprint })
|
|
8239
|
+
}
|
|
8240
|
+
for (const card of cards) {
|
|
8241
|
+
if (!card.coordinate || groups.has(coordinateKeyOf(card.coordinate)) || !store.materialization(coordinateKeyOf(card.coordinate)) || card.status === 'deprecated') continue
|
|
8242
|
+
result.push({ key: coordinateKeyOf(card.coordinate), title: card.title, coordinate: card.coordinate, knowledgeCount: 0, eventCount: 0, sourceIds: card.sourceRefs, cardId: card.entityId, state: card.sourceRefs.some(id => unresolved.sources.has(id)) ? 'blocked' : 'withdrawn', ...(card.sourceRefs.some(id => unresolved.sources.has(id)) ? { reason: '来源中的坐标仍有歧义,请先完成归一' } : {}), fingerprint: 'withdrawn' })
|
|
8243
|
+
}
|
|
8244
|
+
return { groups: result.sort((a, b) => a.title.localeCompare(b.title)), blocked }
|
|
8245
|
+
}
|
|
8246
|
+
|
|
8247
|
+
private async materializePipelineGroup(store: PipelineStore, group: MaterializationGroup, context: PipelineContext, output: (text: string) => void): Promise<void> {
|
|
8248
|
+
const db = await this.storeHandle()
|
|
8249
|
+
if (!db) throw new Error('知识库不可用')
|
|
8250
|
+
const fresh = (await this.materializationGroups(store, context)).groups.find(item => item.key === group.key)
|
|
8251
|
+
if (!fresh || fresh.fingerprint !== group.fingerprint) throw new Error('卡片分组在处理期间已变化,请刷新并重试')
|
|
8252
|
+
if (fresh.state === 'blocked') throw new Error(fresh.reason)
|
|
8253
|
+
const storedExisting = group.cardId ? await this.readCard(group.cardId) : undefined
|
|
8254
|
+
const existing = storedExisting
|
|
8255
|
+
? { ...storedExisting, content: stripEventReferenceSection(storedExisting.content) }
|
|
8256
|
+
: undefined
|
|
8257
|
+
if (group.state === 'withdrawn') {
|
|
8258
|
+
if (!existing) return
|
|
8259
|
+
const written = await this.writeCard({ ...existing, status: 'deprecated', origin: 'compile' })
|
|
8260
|
+
if (!written.ok) throw new Error(written.error.message)
|
|
8261
|
+
store.materialized(group.key, group.fingerprint, existing.entityId)
|
|
8262
|
+
return
|
|
8263
|
+
}
|
|
8264
|
+
const allKnowledge = await this.activePipelineKnowledge()
|
|
8265
|
+
const knowledge = allKnowledge.filter(row => coordinateKeyOf({ objectId: row.objectId, actionId: row.actionId, sceneId: row.sceneId }) === group.key)
|
|
8266
|
+
const events = db.events().filter(event => coordinateKeyOf(event.coordinateIds) === group.key)
|
|
8267
|
+
const profile = context.profile
|
|
8268
|
+
const entityType = (profile?.id ?? 'concept') as SchemaType
|
|
8269
|
+
let title = group.title
|
|
8270
|
+
let content = `## 知识\n\n${knowledge.map(row => `- ${row.text}`).join('\n')}`
|
|
8271
|
+
let summary = knowledge[0]?.text ?? ''
|
|
8272
|
+
let placements: CardKnowledgeRow[] = knowledge.map((row, index) => ({ knowledgeId: row.id, section: '知识', ordinal: index + 1, origin: 'compiler' }))
|
|
8273
|
+
let metadata: Record<string, unknown> = { ...(existing?.metadata ?? {}), pipeline_materialization: true }
|
|
8274
|
+
const entityId = existing?.entityId ?? `coord-${hash(group.key)}`
|
|
8275
|
+
const imagePlan = planSourceImages({
|
|
8276
|
+
cards: (await this.materializationGroups(store, context)).groups.filter(item => item.state !== 'withdrawn').map(item => ({ key: item.key, content: allKnowledge.filter(row => coordinateKeyOf({ objectId: row.objectId, actionId: row.actionId, sceneId: row.sceneId }) === item.key).map(row => row.text).join('\n\n'), sourceRefs: item.sourceIds })),
|
|
8277
|
+
sources: (await this.loadSources()).map(source => ({ sourceId: source.sourceId, content: source.content })),
|
|
8278
|
+
})
|
|
8279
|
+
const images = imagePlan.byCard.get(group.key) ?? []
|
|
8280
|
+
const related = existing ? await this.relatedForCard(existing) : {}
|
|
8281
|
+
metadata.citations = citationsFrom({ placements, knowledge: new Map(knowledge.map(row => [row.id, { source: row.sources[0]?.documentId, heading: row.sources[0]?.headingPath.at(-1) }])) })
|
|
8282
|
+
if (context.organize && knowledge.length > 0) {
|
|
8283
|
+
if (!context.model.call) throw new Error(`卡片组织模型不可用:${context.model.why ?? '请选择模型'}`)
|
|
8284
|
+
const result = await this.organizeCard({ knowledge, images, ...related, cardTitle: title, coordinate: group.coordinate, profileId: entityType, force: true, suggestedSections: profile?.sections.map(section => section.label), requiredSections: profile?.sections.filter(section => section.required).map(section => section.label), existing, call: request => context.model.call!({ ...request, onDelta: output }) })
|
|
8285
|
+
if (result.kind !== 'applied') throw new Error(result.kind === 'reused' ? '组织结果已变化,请刷新后重试' : result.warnings.join(';'))
|
|
8286
|
+
title = result.title; content = result.content; summary = result.summary; placements = result.placements
|
|
8287
|
+
metadata = { ...metadata, layout: result.layout, citations: result.citations }
|
|
8288
|
+
}
|
|
8289
|
+
for (const image of images) content = insertImage(content, image)
|
|
8290
|
+
const eventIds = events.map(event => event.id).sort()
|
|
8291
|
+
content = stripEventReferenceSection(content)
|
|
8292
|
+
metadata.event_refs = JSON.stringify(eventIds)
|
|
8293
|
+
// Freeze the selected group's facts, not an arbitrary subset of its rows.
|
|
8294
|
+
const latest = (await this.materializationGroups(store, context)).groups.find(item => item.key === group.key)
|
|
8295
|
+
if (!latest || latest.fingerprint !== group.fingerprint || latest.state === 'blocked') throw new Error('卡片组织期间事实发生变化,结果未提交,请重试')
|
|
8296
|
+
if (this.closed) throw new Error('知识库已关闭,本次物化未提交')
|
|
8297
|
+
const written = await this.writeCard({ entityType, entityId, title, content, summary, sourceRefs: group.sourceIds, status: existing?.status === 'deprecated' ? 'unreviewed' : existing?.status ?? 'unreviewed', coordinate: group.coordinate, placements, metadata, origin: 'compile' })
|
|
8298
|
+
if (!written.ok) throw new Error(written.error.message)
|
|
8299
|
+
db.setCardEvents(entityId, eventIds)
|
|
8300
|
+
store.materialized(group.key, group.fingerprint, entityId)
|
|
8301
|
+
}
|
|
8302
|
+
|
|
8303
|
+
async pipelineStatus(): Promise<PipelineStatus> {
|
|
8304
|
+
const pipeline = await this.pipeline()
|
|
8305
|
+
const sources = await this.pipelineSources()
|
|
8306
|
+
const context: PipelineContext = { model: this.modelSeamNow(), note: this.extractionNote(), profile: this.organizeProfile(), organize: this.config.cardOrganize, format: 'table' }
|
|
8307
|
+
const rows: PipelineStatus['sources'] = await Promise.all(sources.map(async source => {
|
|
8308
|
+
const head = pipeline.store.head(source.sourceId)
|
|
8309
|
+
const extracted = head.extracted?.sourceVersion === source.version && head.extracted.sourceHash === source.sourceHash
|
|
8310
|
+
const normalized = extracted && head.normalized?.id === head.extracted?.id && head.normalized?.normalized?.fingerprint === await this.normalizationFingerprint(head.normalized!, context)
|
|
8311
|
+
return { id: source.sourceId, title: source.title, version: source.version, extractionId: head.extracted?.id, normalizedId: head.normalized?.id, state: normalized ? 'normalized' as const : extracted ? 'extracted' as const : head.extracted ? 'outdated' as const : 'pending' as const }
|
|
8312
|
+
}))
|
|
8313
|
+
const { groups, blocked } = await this.materializationGroups(pipeline.store, context)
|
|
8314
|
+
const db = await this.derivedHandle()
|
|
8315
|
+
const manifest = db?.artifact('build-manifest.json')
|
|
8316
|
+
const hashes = manifest ? (JSON.parse(manifest) as { cardHashes?: Record<string, string> }).cardHashes ?? {} : {}
|
|
8317
|
+
const cards = await this.loadCards()
|
|
8318
|
+
const indexPending = cards.some(card => hashes[`${card.entityType}:${card.entityId}`] !== this.cardHashOf(card)) || Object.keys(hashes).length !== cards.length
|
|
8319
|
+
return { sources: rows, groups, blocked, pendingNew: groups.filter(group => group.state === 'new').length, pendingUpdated: groups.filter(group => group.state === 'changed').length, pendingWithdrawn: groups.filter(group => group.state === 'withdrawn').length, indexPending }
|
|
8320
|
+
}
|
|
8321
|
+
|
|
8087
8322
|
/** 抽取轨迹列表(新→旧,不含报告本体)。 */
|
|
8088
8323
|
async listExtractionRuns(query: { docId?: string; limit?: number; offset?: number } = {}): Promise<ExtractionRunPage | null> {
|
|
8089
8324
|
const store = await this.extractionRuns()
|
|
@@ -8271,11 +8506,12 @@ export class KnowledgeRuntime {
|
|
|
8271
8506
|
* 跑一趟编译的**实体**(0066)。它不做排队,也不对外承诺「你 await 它就等于你发起了
|
|
8272
8507
|
* 它」—— 那两件事分别在 `enqueueCompile()` 与 `rebuild()` / `startCompile()` 里。
|
|
8273
8508
|
*/
|
|
8274
|
-
private async compileOnce(runId: string, only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean } = {}): Promise<Envelope<RebuildReport>> {
|
|
8275
|
-
await this.environmentGuard?.()
|
|
8509
|
+
private async compileOnce(runId: string, only: 'all' | 'schema' | 'wiki' | 'embeddings' = 'all', lineage?: LineageContext, options: { extract?: boolean; forceExtract?: boolean; checkpoint?: () => Promise<void> } = {}): Promise<Envelope<RebuildReport>> {
|
|
8510
|
+
if (only === 'all' || only === 'embeddings') await this.environmentGuard?.('embedding')
|
|
8276
8511
|
this.publication = { artifacts: new Map() }
|
|
8277
8512
|
const configAtStart = JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() })
|
|
8278
8513
|
try {
|
|
8514
|
+
await options.checkpoint?.()
|
|
8279
8515
|
await this.loadCompiledState()
|
|
8280
8516
|
const result = await this.prepareCompile(runId, only, lineage, options)
|
|
8281
8517
|
await this.checkpointCompileTask(runId)
|
|
@@ -8305,6 +8541,7 @@ export class KnowledgeRuntime {
|
|
|
8305
8541
|
this.publication.artifacts.set(COMPILE_SNAPSHOT, JSON.stringify(snapshot))
|
|
8306
8542
|
await this.checkpointCompileTask(runId)
|
|
8307
8543
|
if (JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() }) !== configAtStart) throw new Error('COMPILE_CONFIG_CHANGED: 编译期间设置已变化,请重新编译')
|
|
8544
|
+
await options.checkpoint?.()
|
|
8308
8545
|
db.publish(this.publication)
|
|
8309
8546
|
this.vectorIndex = null
|
|
8310
8547
|
}
|
|
@@ -8452,7 +8689,7 @@ export class KnowledgeRuntime {
|
|
|
8452
8689
|
tally.knowledge > 0
|
|
8453
8690
|
? `抽出知识 ${tally.knowledge} 条`
|
|
8454
8691
|
: '未用模型:正文按 ## 小节切',
|
|
8455
|
-
tally.skipped > 0 ?
|
|
8692
|
+
tally.skipped > 0 ? `复用未变来源 ${tally.skipped} 份` : options.forceExtract ? '本次为手动强制重抽' : '正在检查新增或变化的来源',
|
|
8456
8693
|
]
|
|
8457
8694
|
if (done < total) {
|
|
8458
8695
|
setStep('knowledge', 'running', knowledgeFacts)
|
|
@@ -9426,7 +9663,7 @@ export class KnowledgeRuntime {
|
|
|
9426
9663
|
* 只取**命中这几张卡**的落位。用 `placements()`(全表)在这里是错的量级:
|
|
9427
9664
|
* 一次召回十来张卡,而库里可能有上千张。
|
|
9428
9665
|
*/
|
|
9429
|
-
try { placements = snapshot
|
|
9666
|
+
try { placements = snapshot ? new Map(snapshot.placements) : store.placementsOf([...new Set(hits.map(hit => hit.entityId))]) } catch { return }
|
|
9430
9667
|
/**
|
|
9431
9668
|
* 每条命中要哪几条 Knowledge:以 Card 正文切出的 heading 为准。
|
|
9432
9669
|
*
|
|
@@ -9442,7 +9679,7 @@ export class KnowledgeRuntime {
|
|
|
9442
9679
|
})
|
|
9443
9680
|
const wanted = new Set(perHit.flat().map(row => row.knowledgeId))
|
|
9444
9681
|
if (wanted.size === 0) return
|
|
9445
|
-
const rows = snapshot
|
|
9682
|
+
const rows = snapshot
|
|
9446
9683
|
? new Map(snapshot.knowledge.filter(row => wanted.has(row.id) && knowledge !== null)
|
|
9447
9684
|
.flatMap(row => {
|
|
9448
9685
|
// Deleted facts never reappear through a historical snapshot.
|
|
@@ -9536,11 +9773,14 @@ export class KnowledgeRuntime {
|
|
|
9536
9773
|
if (!query?.trim()) return failure(empty(), 'QUERY_REQUIRED', 'query 不能为空', true)
|
|
9537
9774
|
if (!Number.isInteger(recallTopK) || recallTopK < 1 || recallTopK > 50) return failure(empty(), 'LIMIT_INVALID', '召回 top_k 必须是 1 到 50 的整数', false)
|
|
9538
9775
|
if (!Number.isInteger(requestedTopN) || requestedTopN < 1 || (options.contextTopN !== undefined && requestedTopN > recallTopK)) return failure(empty(), 'CONTEXT_TOP_N_INVALID', `上下文 top_n 必须是 1 到 ${recallTopK} 的整数`, false)
|
|
9539
|
-
try {
|
|
9776
|
+
try {
|
|
9777
|
+
if (recallMode !== 'fts') await this.environmentGuard?.('embedding')
|
|
9778
|
+
if (options.rerank !== false && this.config.rerankProvider !== 'none') await this.environmentGuard?.('rerank')
|
|
9779
|
+
}
|
|
9540
9780
|
catch (cause) { return failure(empty(), 'ENVIRONMENT_NOT_READY', cause instanceof Error ? cause.message : String(cause), true, '在 Tita 设置中完成环境准备并重新自检。') }
|
|
9541
9781
|
const derived = await this.derivedHandle()
|
|
9542
9782
|
const liveCards = await this.loadCards()
|
|
9543
|
-
const liveKeys = new Set(liveCards.map(card => cardKey(card)))
|
|
9783
|
+
const liveKeys = new Set(liveCards.filter(card => card.status !== 'deprecated').map(card => cardKey(card)))
|
|
9544
9784
|
const snapshotHash = derived?.artifactHash(COMPILE_SNAPSHOT)
|
|
9545
9785
|
const cached = snapshotHash && this.recallSnapshotCache?.hash === snapshotHash ? this.recallSnapshotCache : undefined
|
|
9546
9786
|
const raw = cached ? undefined : derived?.artifact(COMPILE_SNAPSHOT)
|
|
@@ -9700,7 +9940,7 @@ export class KnowledgeRuntime {
|
|
|
9700
9940
|
].join('\n')
|
|
9701
9941
|
const reranked = await this.rerankNotes(rerankQuery, hits, this.config.rerankThreshold)
|
|
9702
9942
|
rerankScored = reranked.scored ?? []
|
|
9703
|
-
hits = reranked.hits
|
|
9943
|
+
hits = reranked.hits
|
|
9704
9944
|
rerank = reranked.result
|
|
9705
9945
|
if (reranked.warning) warnings.push(reranked.warning)
|
|
9706
9946
|
}
|
|
@@ -9709,7 +9949,7 @@ export class KnowledgeRuntime {
|
|
|
9709
9949
|
}
|
|
9710
9950
|
// 重排换的是相关性顺序,评分加权仍然要压在最上面 —— 否则「review 过的排前面一点」
|
|
9711
9951
|
// 只在关掉重排时成立,而重排默认是开着的。
|
|
9712
|
-
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId))
|
|
9952
|
+
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId)).slice(0, recallTopK)
|
|
9713
9953
|
|
|
9714
9954
|
// --- R4 · 证据片段 -----------------------------------------------------
|
|
9715
9955
|
const truth = await this.storeHandle()
|