dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -56
- package/cordis.patch.yml +36 -129
- package/eval/benchmark-extraction-format.mjs +20 -0
- package/lib/client.js +3262 -4142
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +1147 -364
- package/lib/types/axis-registry.d.ts +8 -1
- package/lib/types/axis-registry.d.ts.map +1 -1
- package/lib/types/axis-registry.js +62 -7
- package/lib/types/axis-registry.js.map +1 -1
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +3 -0
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +9 -0
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +68 -9
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/diagnosis/cards.d.ts +7 -3
- package/lib/types/diagnosis/cards.d.ts.map +1 -1
- package/lib/types/diagnosis/cards.js +13 -8
- package/lib/types/diagnosis/cards.js.map +1 -1
- package/lib/types/diagnosis/run.d.ts +4 -13
- package/lib/types/diagnosis/run.d.ts.map +1 -1
- package/lib/types/diagnosis/run.js +26 -16
- package/lib/types/diagnosis/run.js.map +1 -1
- package/lib/types/diagnosis/schema.js +2 -2
- package/lib/types/diagnosis/schema.js.map +1 -1
- package/lib/types/diagnosis/seed-source.d.ts +1 -1
- package/lib/types/diagnosis/seed-source.d.ts.map +1 -1
- package/lib/types/diagnosis/seed-source.js +5 -4
- package/lib/types/diagnosis/seed-source.js.map +1 -1
- package/lib/types/diagnosis/seed.d.ts +4 -4
- package/lib/types/diagnosis/seed.d.ts.map +1 -1
- package/lib/types/diagnosis/seed.js +6 -8
- package/lib/types/diagnosis/seed.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +18 -17
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +94 -66
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +10 -16
- package/lib/types/diagnosis/types.d.ts.map +1 -1
- package/lib/types/diagnosis/types.js.map +1 -1
- package/lib/types/environment/dsh-python.d.ts +77 -0
- package/lib/types/environment/dsh-python.d.ts.map +1 -0
- package/lib/types/environment/dsh-python.js +139 -0
- package/lib/types/environment/dsh-python.js.map +1 -0
- package/lib/types/environment/install.d.ts +9 -1
- package/lib/types/environment/install.d.ts.map +1 -1
- package/lib/types/environment/install.js +149 -24
- package/lib/types/environment/install.js.map +1 -1
- package/lib/types/environment/model-catalog.d.ts +34 -2
- package/lib/types/environment/model-catalog.d.ts.map +1 -1
- package/lib/types/environment/model-catalog.js +40 -2
- package/lib/types/environment/model-catalog.js.map +1 -1
- package/lib/types/environment/service.d.ts +13 -3
- package/lib/types/environment/service.d.ts.map +1 -1
- package/lib/types/environment/service.js +127 -23
- package/lib/types/environment/service.js.map +1 -1
- package/lib/types/environment/types.d.ts +9 -0
- package/lib/types/environment/types.d.ts.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js +13 -0
- package/lib/types/events.js.map +1 -1
- package/lib/types/extract-contract.d.ts +1 -1
- package/lib/types/extract-contract.d.ts.map +1 -1
- package/lib/types/extract-contract.js +34 -31
- package/lib/types/extract-contract.js.map +1 -1
- package/lib/types/extract-model.d.ts +2 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/extract-table-contract.d.ts +3 -0
- package/lib/types/extract-table-contract.d.ts.map +1 -0
- package/lib/types/extract-table-contract.js +30 -0
- package/lib/types/extract-table-contract.js.map +1 -0
- package/lib/types/extract-table.d.ts +8 -0
- package/lib/types/extract-table.d.ts.map +1 -0
- package/lib/types/extract-table.js +151 -0
- package/lib/types/extract-table.js.map +1 -0
- package/lib/types/extraction-runs.d.ts +30 -6
- package/lib/types/extraction-runs.d.ts.map +1 -1
- package/lib/types/extraction-runs.js +69 -8
- package/lib/types/extraction-runs.js.map +1 -1
- package/lib/types/fs-atomic.d.ts +0 -7
- package/lib/types/fs-atomic.d.ts.map +1 -1
- package/lib/types/fs-atomic.js +17 -2
- package/lib/types/fs-atomic.js.map +1 -1
- package/lib/types/images.d.ts +1 -1
- package/lib/types/images.d.ts.map +1 -1
- package/lib/types/images.js +2 -2
- package/lib/types/images.js.map +1 -1
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +232 -50
- package/lib/types/index.js.map +1 -1
- package/lib/types/intent/scope.d.ts.map +1 -1
- package/lib/types/intent/scope.js +1 -0
- package/lib/types/intent/scope.js.map +1 -1
- package/lib/types/intent/tools.d.ts +19 -0
- package/lib/types/intent/tools.d.ts.map +1 -1
- package/lib/types/intent/tools.js +37 -1
- package/lib/types/intent/tools.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +5 -16
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge.d.ts +3 -0
- package/lib/types/knowledge.d.ts.map +1 -1
- package/lib/types/knowledge.js +21 -5
- package/lib/types/knowledge.js.map +1 -1
- package/lib/types/materialize-snapshot.d.ts +49 -0
- package/lib/types/materialize-snapshot.d.ts.map +1 -0
- package/lib/types/materialize-snapshot.js +2 -0
- package/lib/types/materialize-snapshot.js.map +1 -0
- package/lib/types/model-text.d.ts +1 -1
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +5 -4
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/oplog.d.ts.map +1 -1
- package/lib/types/oplog.js +7 -0
- package/lib/types/oplog.js.map +1 -1
- package/lib/types/pipeline/engine.d.ts +60 -0
- package/lib/types/pipeline/engine.d.ts.map +1 -0
- package/lib/types/pipeline/engine.js +338 -0
- package/lib/types/pipeline/engine.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +73 -0
- package/lib/types/pipeline/store.d.ts.map +1 -0
- package/lib/types/pipeline/store.js +143 -0
- package/lib/types/pipeline/store.js.map +1 -0
- package/lib/types/pipeline/types.d.ts +101 -0
- package/lib/types/pipeline/types.d.ts.map +1 -0
- package/lib/types/pipeline/types.js +5 -0
- package/lib/types/pipeline/types.js.map +1 -0
- package/lib/types/present.d.ts +2 -0
- package/lib/types/present.d.ts.map +1 -1
- package/lib/types/present.js +9 -4
- package/lib/types/present.js.map +1 -1
- package/lib/types/retention.d.ts +39 -0
- package/lib/types/retention.d.ts.map +1 -0
- package/lib/types/retention.js +42 -0
- package/lib/types/retention.js.map +1 -0
- package/lib/types/runtime.d.ts +115 -5
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +862 -132
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/source-evidence.d.ts +9 -1
- package/lib/types/source-evidence.d.ts.map +1 -1
- package/lib/types/source-evidence.js +13 -5
- package/lib/types/source-evidence.js.map +1 -1
- package/lib/types/store/axis-references.js +3 -3
- package/lib/types/store/axis-references.js.map +1 -1
- package/lib/types/store/derived.d.ts +1 -1
- package/lib/types/store/derived.d.ts.map +1 -1
- package/lib/types/store/derived.js +41 -7
- package/lib/types/store/derived.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +18 -0
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +111 -6
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/migrate.js +1 -1
- package/lib/types/store/migrate.js.map +1 -1
- package/lib/types/trajectory-types.d.ts +172 -0
- package/lib/types/trajectory-types.d.ts.map +1 -0
- package/lib/types/trajectory-types.js +36 -0
- package/lib/types/trajectory-types.js.map +1 -0
- package/lib/types/types.d.ts +3 -4
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-finalize.d.ts +16 -0
- package/lib/types/wiki-finalize.d.ts.map +1 -1
- package/lib/types/wiki-finalize.js +26 -0
- package/lib/types/wiki-finalize.js.map +1 -1
- package/lib/types/wiki-materialize.d.ts.map +1 -1
- package/lib/types/wiki-materialize.js +3 -15
- package/lib/types/wiki-materialize.js.map +1 -1
- package/lib/types/wiki-trial-types.d.ts +3 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial.d.ts +2 -0
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +31 -7
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +6 -3
- package/preset/tita-data-agent/agent.cordis.yml +36 -129
- package/src/axis-registry.ts +57 -9
- package/src/client/AttributionEventGuide.tsx +10 -9
- package/src/client/CompileDialogs.tsx +10 -2
- package/src/client/EnvironmentSetup.tsx +76 -16
- package/src/client/ExpansionGraph.tsx +13 -7
- package/src/client/ExtractionTasksPage.tsx +543 -156
- package/src/client/ImportFilesDialog.tsx +2 -13
- package/src/client/IntentTab.tsx +3 -1
- package/src/client/KnowledgeBaseTab.tsx +139 -122
- package/src/client/KnowledgeLayerSection.tsx +19 -2
- package/src/client/KnowledgePanel.tsx +17 -3
- package/src/client/ListPager.tsx +15 -5
- package/src/client/MaterializeTrajectoryView.tsx +124 -0
- package/src/client/NodeSettings.tsx +90 -25
- package/src/client/NormalizeTrajectoryView.tsx +136 -0
- package/src/client/PipelineNotice.tsx +26 -0
- package/src/client/RecallPanel.tsx +55 -25
- package/src/client/TabGuide.tsx +4 -4
- package/src/client/TrialReportView.tsx +61 -16
- package/src/client/TypeDefinitionDrawer.tsx +33 -9
- package/src/client/WikiCardPreview.tsx +43 -10
- package/src/client/WikiFlowPanel.tsx +4 -4
- package/src/client/architecture.tsx +1 -1
- package/src/client/attribution/AttributionTab.tsx +37 -36
- package/src/client/attribution/CaseView.tsx +3 -1
- package/src/client/attribution/EventView.tsx +14 -4
- package/src/client/attribution/RunView.tsx +2 -2
- package/src/client/attribution/ViewSwitch.tsx +20 -5
- package/src/client/attribution/graph-options.ts +38 -31
- package/src/client/attribution/model.ts +16 -9
- package/src/client/graph/palette.ts +165 -0
- package/src/client/pages.css +224 -28
- package/src/client/pages.tsx +33 -11
- package/src/client/styles.css +61 -6
- package/src/client/wiki-trial.ts +30 -1
- package/src/convert/host.ts +3 -0
- package/src/convert/ingest.ts +73 -9
- package/src/diagnosis/cards.ts +13 -8
- package/src/diagnosis/run.ts +27 -17
- package/src/diagnosis/schema.ts +2 -2
- package/src/diagnosis/seed-source.ts +5 -4
- package/src/diagnosis/seed.ts +6 -8
- package/src/diagnosis/store.ts +99 -69
- package/src/diagnosis/types.ts +10 -16
- package/src/environment/dsh-python.ts +156 -0
- package/src/environment/install.ts +134 -19
- package/src/environment/model-catalog.ts +45 -3
- package/src/environment/service.ts +111 -23
- package/src/environment/types.ts +6 -0
- package/src/events.ts +12 -0
- package/src/extract-contract.ts +6 -3
- package/src/extract-model.ts +2 -0
- package/src/extract-table-contract.ts +29 -0
- package/src/extract-table.ts +126 -0
- package/src/extraction-runs.ts +70 -11
- package/src/fs-atomic.ts +19 -2
- package/src/images.ts +2 -2
- package/src/index.ts +207 -72
- package/src/intent/scope.ts +1 -0
- package/src/intent/tools.ts +38 -1
- package/src/knowledge-bases.ts +4 -12
- package/src/knowledge.ts +24 -4
- package/src/materialize-snapshot.ts +46 -0
- package/src/model-text.ts +5 -3
- package/src/oplog.ts +7 -0
- package/src/pipeline/engine.ts +312 -0
- package/src/pipeline/store.ts +147 -0
- package/src/pipeline/types.ts +92 -0
- package/src/present.ts +10 -4
- package/src/retention.ts +76 -0
- package/src/runtime.ts +853 -131
- package/src/source-evidence.ts +12 -5
- package/src/store/axis-references.ts +3 -3
- package/src/store/derived.ts +40 -7
- package/src/store/knowledge-db.ts +111 -5
- package/src/store/migrate.ts +1 -1
- package/src/trajectory-types.ts +142 -0
- package/src/types.ts +3 -4
- package/src/wiki-finalize.ts +28 -0
- package/src/wiki-materialize.ts +3 -14
- package/src/wiki-trial-types.ts +3 -0
- package/src/wiki-trial.ts +26 -7
- package/src/client/convert/ConvertPanel.tsx +0 -583
- package/src/client/convert/styles.css +0 -138
- package/src/client/convert/styles.ts +0 -49
package/lib/types/runtime.js
CHANGED
|
@@ -13,15 +13,18 @@ import { MAX_LOG_ITEMS, OperationLog, operationId } from './oplog.js';
|
|
|
13
13
|
import { ExtractionRunStore, extractionRunId } from './extraction-runs.js';
|
|
14
14
|
import { AxisRegistry, normalizeAxisName } from './axis-registry.js';
|
|
15
15
|
import { KNOWLEDGE_STATE_LABEL, KnowledgeStore } from './knowledge.js';
|
|
16
|
-
import { compositeEventIdentity, eventEntityFromRecords } from './events.js';
|
|
16
|
+
import { compositeEventIdentity, eventEntityFromRecords, stripEventReferenceSection } from './events.js';
|
|
17
17
|
import { buildAliasIndex, candidateEntityId, extractCandidates, normalizeCandidates, RELATES_TO } from './extract.js';
|
|
18
18
|
import { skeletonBody } from './extract-model.js';
|
|
19
|
-
import { applyTrialAxisResolution, normalizeKnowledge, runWikiTrial, trialSummary } from './wiki-trial.js';
|
|
20
|
-
import {
|
|
19
|
+
import { applyTrialAxisResolution, cardNameFor, normalizeKnowledge, runWikiTrial, trialSummary } from './wiki-trial.js';
|
|
20
|
+
import { EXTRACTION_CONTRACT_VERSION } from './extract-table.js';
|
|
21
|
+
import { KnowledgePipeline } from './pipeline/engine.js';
|
|
22
|
+
import { PipelineStore } from './pipeline/store.js';
|
|
23
|
+
import { candidatesFromReport, coordinateIdsOf, coordinateIsEmpty } from './wiki-materialize.js';
|
|
21
24
|
import { buildOrganizePrompt, buildOrganizeSystem, checkOrganize, citationsFrom, layoutHash, layoutOf, parseOrganize, placementsOf, renderFootnoteDefinitions, renderOrganizeBody, } from './card-organize.js';
|
|
22
25
|
import { FTS_FILE, loadSqlite, segment } from './fts.js';
|
|
23
26
|
import { attachImagesToKnowledge, insertImage, planSourceImages } from './source-images.js';
|
|
24
|
-
import { buildWikiIndex, cardKey, deadLinksOf, planRelated, relatedEntriesOf, stripRelatedSection, RELATED_METADATA_KEY, } from './wiki-finalize.js';
|
|
27
|
+
import { buildWikiIndex, cardKey, deadLinksOf, planRelated, relatedByGroupKey, relatedEntriesOf, stripRelatedSection, RELATED_METADATA_KEY, } from './wiki-finalize.js';
|
|
25
28
|
import { applyRerank, applyThreshold, evidenceExcerpts, graphBoost, matchOf, noteIndexText, reviewWeight, rrfFuse, tokenSet } from './recall-rank.js';
|
|
26
29
|
import { ASSETS_DIR, KNOWLEDGE_DB_FILE, PUBLISHED_IMAGE_MARKER, SOURCE_RAW_DIR, defaultAgentRoot, knowledgeBaseRoot } from './layout.js';
|
|
27
30
|
import { KnowledgeDb } from './store/knowledge-db.js';
|
|
@@ -233,6 +236,15 @@ const RRF_K = 60;
|
|
|
233
236
|
const RRF_WEIGHTS = { coordinate: 0.9, relation: 0.45, fts: 0.3, vector: 0.7, graph: 0.2 };
|
|
234
237
|
/** 图谱只是 boost,不是路:它的权重只作用在**已经在候选池里**的条目上(见 `graphBoost`)。 */
|
|
235
238
|
const GRAPH_BOOST_WEIGHT = RRF_WEIGHTS.graph;
|
|
239
|
+
/**
|
|
240
|
+
* 重排失败后的冷却时间。
|
|
241
|
+
*
|
|
242
|
+
* 失败要先记住,否则每次召回都会重新拉起一个注定失败的 worker(那是「秒级重试」的代价)。
|
|
243
|
+
* 但**不能永久记住**:一次超时是暂态,而写死之后 `rerankNotes` 会在开头直接返回,
|
|
244
|
+
* 于是这个标志自我封闭 —— 只有换模型、重载家当或人工重跑自检才能清掉。带上冷却,
|
|
245
|
+
* 第一次失败仍然如实报(有生产门时仍然明确失败),之后每 30 秒给一次自愈机会。
|
|
246
|
+
*/
|
|
247
|
+
const RERANK_RETRY_COOLDOWN_MS = 30_000;
|
|
236
248
|
/**
|
|
237
249
|
* 编码器健康门的判据:两段无关文本的余弦必须低于它。
|
|
238
250
|
*
|
|
@@ -684,38 +696,6 @@ function appendExtracted(existing, additions) {
|
|
|
684
696
|
}
|
|
685
697
|
return content;
|
|
686
698
|
}
|
|
687
|
-
/** 坐标 WikiCard 的事件摘要段由 Event 引用派生;全文与时间线留在独立 Event 实体。 */
|
|
688
|
-
function withEventReferences(content, rawRefs, eventsById) {
|
|
689
|
-
let refs = [];
|
|
690
|
-
if (Array.isArray(rawRefs))
|
|
691
|
-
refs = rawRefs.filter((item) => typeof item === 'string');
|
|
692
|
-
else if (typeof rawRefs === 'string') {
|
|
693
|
-
try {
|
|
694
|
-
const parsed = JSON.parse(rawRefs);
|
|
695
|
-
if (Array.isArray(parsed))
|
|
696
|
-
refs = parsed.filter((item) => typeof item === 'string');
|
|
697
|
-
}
|
|
698
|
-
catch { /* 无效引用会作为空列表处理 */ }
|
|
699
|
-
}
|
|
700
|
-
const lines = content.split('\n');
|
|
701
|
-
const start = lines.findIndex(line => line.trim() === '## 关联 Event');
|
|
702
|
-
if (start >= 0) {
|
|
703
|
-
let end = start + 1;
|
|
704
|
-
while (end < lines.length && !/^#{1,2}\s+/.test(lines[end] ?? ''))
|
|
705
|
-
end += 1;
|
|
706
|
-
lines.splice(start, end - start);
|
|
707
|
-
}
|
|
708
|
-
const base = lines.join('\n').trimEnd();
|
|
709
|
-
const entries = [...new Set(refs)].map(id => eventsById.get(id)).filter((event) => event !== undefined);
|
|
710
|
-
if (entries.length === 0)
|
|
711
|
-
return base;
|
|
712
|
-
const bullets = entries.map(event => {
|
|
713
|
-
const occurredAt = event.occurredAt ? ` · ${event.occurredAt}` : '';
|
|
714
|
-
const summary = event.summary.trim() ? `:${event.summary.trim()}` : '';
|
|
715
|
-
return `- **${event.title}**${occurredAt}${summary}(${event.id})`;
|
|
716
|
-
});
|
|
717
|
-
return `${base ? `${base}\n\n` : ''}## 关联 Event\n\n${bullets.join('\n')}`;
|
|
718
|
-
}
|
|
719
699
|
/** 把 `incoming` 按小节折进 `existing`;结构对不上时整段追加。 */
|
|
720
700
|
function foldBody(existing, incoming) {
|
|
721
701
|
if (!existing)
|
|
@@ -1008,9 +988,10 @@ function markdownImageRefs(content) {
|
|
|
1008
988
|
* `tags` / `generated` / `sources`。它们是真 YAML 键,人能读、`grep` 得到、
|
|
1009
989
|
* 外部工具能按公开规范读走。`generated.at` 就是**采集时间**(OKF 里没有「抓取
|
|
1010
990
|
* 时间」这个概念,本文件何时被谁写出来就是它)。
|
|
1011
|
-
* 2. **我们自己的运行键**:`
|
|
1012
|
-
* `
|
|
1013
|
-
*
|
|
991
|
+
* 2. **我们自己的运行键**:`created_at` / `updated_at` / `metadata` / `lineage`。
|
|
992
|
+
* 来源身份保存在 SQLite 的 `entity_id` / `external_id` 列,不在 front matter 重复一份。
|
|
993
|
+
* OKF 明确允许自定义键,且要求消费者不得因为未知键拒收,所以运行元数据可以同处
|
|
994
|
+
* 一个 front matter。
|
|
1014
995
|
*
|
|
1015
996
|
* `metadata` 只剩**类型专属的扩展键**(转换任务 id、内容哈希、资产清单、格式…):
|
|
1016
997
|
* 那些东西 OKF 没有对应概念,但也不该混进上面那组公开字段里。
|
|
@@ -1043,8 +1024,6 @@ function sourceFrontMatter(record, provenance) {
|
|
|
1043
1024
|
...(origin.lastModified?.trim() ? { last_modified: origin.lastModified.trim() } : {}),
|
|
1044
1025
|
}));
|
|
1045
1026
|
}
|
|
1046
|
-
frontMatter.source_id = record.sourceId;
|
|
1047
|
-
frontMatter.external_id = record.externalId ?? '';
|
|
1048
1027
|
frontMatter.created_at = record.createdAt;
|
|
1049
1028
|
frontMatter.updated_at = record.updatedAt;
|
|
1050
1029
|
frontMatter.metadata = record.metadata;
|
|
@@ -1551,6 +1530,9 @@ export class KnowledgeRuntime {
|
|
|
1551
1530
|
/** 抽取轨迹(`<memoryDir>/extraction-runs.sqlite`)。同一个懒打开的理由。 */
|
|
1552
1531
|
extractionRunsStore = null;
|
|
1553
1532
|
extractionRunsReady = null;
|
|
1533
|
+
closed = false;
|
|
1534
|
+
pipelineEngine;
|
|
1535
|
+
pipelineOpening;
|
|
1554
1536
|
/** 轴注册表(`<memoryDir>/knowledge.sqlite`)。与抽取轨迹同一个懒打开的理由。 */
|
|
1555
1537
|
axesStore = null;
|
|
1556
1538
|
axesReady = null;
|
|
@@ -1630,7 +1612,7 @@ export class KnowledgeRuntime {
|
|
|
1630
1612
|
embeddingWorkerFingerprint;
|
|
1631
1613
|
/** Separate process so a rerank model failure cannot take embeddings down. */
|
|
1632
1614
|
rerankWorker = null;
|
|
1633
|
-
/** Cache the
|
|
1615
|
+
/** Cache the last rerank failure with its time; see `RERANK_RETRY_COOLDOWN_MS`. */
|
|
1634
1616
|
rerankUnavailable;
|
|
1635
1617
|
/** 解析后的 Python 解释器路径(配置为空时从家当目录解析)。 */
|
|
1636
1618
|
pythonPath;
|
|
@@ -2292,6 +2274,10 @@ export class KnowledgeRuntime {
|
|
|
2292
2274
|
return { vectors, device };
|
|
2293
2275
|
}
|
|
2294
2276
|
close() {
|
|
2277
|
+
if (this.closed)
|
|
2278
|
+
return;
|
|
2279
|
+
this.closed = true;
|
|
2280
|
+
this.pipelineEngine?.close();
|
|
2295
2281
|
for (const task of this.compileTasks.values()) {
|
|
2296
2282
|
if (['succeeded', 'failed', 'cancelled'].includes(task.progress.state))
|
|
2297
2283
|
continue;
|
|
@@ -2299,6 +2285,13 @@ export class KnowledgeRuntime {
|
|
|
2299
2285
|
task.pauseRequested = false;
|
|
2300
2286
|
task.resume?.();
|
|
2301
2287
|
}
|
|
2288
|
+
if (this.compilePending > 0) {
|
|
2289
|
+
void this.compileTail.finally(() => this.closeStores());
|
|
2290
|
+
return;
|
|
2291
|
+
}
|
|
2292
|
+
this.closeStores();
|
|
2293
|
+
}
|
|
2294
|
+
closeStores() {
|
|
2302
2295
|
this.embeddingWorker?.close();
|
|
2303
2296
|
this.embeddingWorker = null;
|
|
2304
2297
|
this.rerankWorker?.close();
|
|
@@ -2680,39 +2673,26 @@ export class KnowledgeRuntime {
|
|
|
2680
2673
|
if (written.length === 0) {
|
|
2681
2674
|
return failure({ ...empty, failed, attempted: items.length }, 'IMPORT_ALL_FAILED', `${items.length} 份来源全部写入失败,没有任何东西进入知识库`, true, failed[0] ? `第一份的原因:${failed[0].message}` : undefined);
|
|
2682
2675
|
}
|
|
2683
|
-
/**
|
|
2684
|
-
* 编译。**只要有一份写成就编**:成功的那几份已经是库里的知识,
|
|
2685
|
-
* 「因为同批里有一份失败就不编」等于把已经进来的知识又挡在门外一次。
|
|
2686
|
-
*/
|
|
2687
2676
|
let compile = null;
|
|
2688
|
-
if (input.compile
|
|
2689
|
-
let rebuilt;
|
|
2677
|
+
if (input.compile === true) {
|
|
2690
2678
|
try {
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
}
|
|
2696
|
-
if (rebuilt === undefined)
|
|
2697
|
-
return success({ written, failed, attempted: items.length, compile }, warnings);
|
|
2698
|
-
if (rebuilt.ok === false) {
|
|
2699
|
-
warnings.push(`来源已写入,但编译失败:${rebuilt.error?.message ?? '未知原因'}`);
|
|
2700
|
-
}
|
|
2701
|
-
else {
|
|
2702
|
-
const data = rebuilt.data;
|
|
2679
|
+
const before = new Map((await this.loadCards()).map(card => [card.entityId, this.cardHashOf(card)]));
|
|
2680
|
+
const pipeline = await this.pipeline();
|
|
2681
|
+
const task = await pipeline.start('all', { sourceIds: written.map(source => source.sourceId) }).completion;
|
|
2682
|
+
const after = await this.loadCards();
|
|
2703
2683
|
compile = {
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
sourcesExtracted: data.sourcesExtracted,
|
|
2710
|
-
sourcesSkipped: data.sourcesSkipped,
|
|
2711
|
-
cardsRemoved: data.cardsRemoved,
|
|
2712
|
-
embedding: data.embedding,
|
|
2713
|
-
version: data.version,
|
|
2684
|
+
taskId: task.id, state: task.state,
|
|
2685
|
+
cardsCreated: after.filter(card => !before.has(card.entityId)).length,
|
|
2686
|
+
cardsUpdated: after.filter(card => before.has(card.entityId) && before.get(card.entityId) !== this.cardHashOf(card)).length,
|
|
2687
|
+
sourcesExtracted: Math.max(0, (task.results?.extract?.done ?? 0) - (task.results?.extract?.reused ?? 0) - (task.results?.extract?.failed ?? 0)),
|
|
2688
|
+
sourcesSkipped: task.results?.extract?.reused ?? 0,
|
|
2714
2689
|
};
|
|
2715
|
-
warnings.push(...
|
|
2690
|
+
warnings.push(...task.warnings);
|
|
2691
|
+
if (task.state !== 'succeeded')
|
|
2692
|
+
warnings.push(`来源已保存;全链路未完成:${task.summary}。可在知识处理页面继续。`);
|
|
2693
|
+
}
|
|
2694
|
+
catch (cause) {
|
|
2695
|
+
warnings.push(`来源已保存;知识处理未完成:${cause instanceof Error ? cause.message : String(cause)}`);
|
|
2716
2696
|
}
|
|
2717
2697
|
}
|
|
2718
2698
|
return success({ written, failed, attempted: items.length, compile }, warnings);
|
|
@@ -2984,6 +2964,15 @@ export class KnowledgeRuntime {
|
|
|
2984
2964
|
* 失败一律返回 `null`,**不抛** —— 「知识库不可用」是一条要显示的结论,不是一个异常。
|
|
2985
2965
|
*/
|
|
2986
2966
|
storeHandleOpening;
|
|
2967
|
+
/**
|
|
2968
|
+
* 知识存储现在可用吗(0070)。
|
|
2969
|
+
*
|
|
2970
|
+
* 读路径需要它把「确认是空」与「读不到」分开:`loadCards()` 在存储不可用时返回 `[]`,
|
|
2971
|
+
* 而调用方据此做写决定(例如诊断的旧格式迁移)就会把一次读故障变成一次污染写。
|
|
2972
|
+
*/
|
|
2973
|
+
async knowledgeStoreAvailable() {
|
|
2974
|
+
return (await this.storeHandle()) !== null;
|
|
2975
|
+
}
|
|
2987
2976
|
async storeHandle() {
|
|
2988
2977
|
if (this.store !== undefined)
|
|
2989
2978
|
return this.store;
|
|
@@ -3894,10 +3883,11 @@ export class KnowledgeRuntime {
|
|
|
3894
3883
|
const store = await this.knowledgeStore();
|
|
3895
3884
|
if (store === null)
|
|
3896
3885
|
return { ok: false, reason: '知识库不可用,这次没有组织卡片(正文退回确定性布局)' };
|
|
3897
|
-
const listed = await store.list({
|
|
3886
|
+
const listed = input.knowledge ? { knowledge: input.knowledge, total: input.knowledge.length } : await store.list({
|
|
3898
3887
|
objectId: input.coordinate.objectId,
|
|
3899
3888
|
actionId: input.coordinate.actionId,
|
|
3900
3889
|
sceneId: input.coordinate.sceneId,
|
|
3890
|
+
excludeWithdrawn: true,
|
|
3901
3891
|
limit: MAX_ORGANIZE_KNOWLEDGE,
|
|
3902
3892
|
});
|
|
3903
3893
|
if (listed.knowledge.length === 0) {
|
|
@@ -4075,7 +4065,8 @@ export class KnowledgeRuntime {
|
|
|
4075
4065
|
if (call === undefined)
|
|
4076
4066
|
return failure({ card, warnings: [] }, 'MODEL_UNAVAILABLE', '当前没有可用的模型,无法重新组织这张卡片', true, '检查模型配置后再试。');
|
|
4077
4067
|
const profile = this.organizeProfile();
|
|
4078
|
-
const
|
|
4068
|
+
const organizationCard = { ...card, content: stripEventReferenceSection(card.content) };
|
|
4069
|
+
const related = await this.relatedForCard(organizationCard);
|
|
4079
4070
|
const images = await this.imagesForCard(card);
|
|
4080
4071
|
const organized = await this.organizeCard({
|
|
4081
4072
|
cardTitle: card.title,
|
|
@@ -4086,7 +4077,7 @@ export class KnowledgeRuntime {
|
|
|
4086
4077
|
...(profile === undefined ? {} : { requiredSections: profile.sections.filter(section => section.required).map(section => section.label) }),
|
|
4087
4078
|
...(related.related === undefined ? {} : { related: related.related }),
|
|
4088
4079
|
...(images.images.length === 0 ? {} : { images: images.images }),
|
|
4089
|
-
existing:
|
|
4080
|
+
existing: organizationCard,
|
|
4090
4081
|
call,
|
|
4091
4082
|
});
|
|
4092
4083
|
if (organized.kind !== 'applied') {
|
|
@@ -4097,7 +4088,7 @@ export class KnowledgeRuntime {
|
|
|
4097
4088
|
entityType: card.entityType,
|
|
4098
4089
|
entityId: card.entityId,
|
|
4099
4090
|
title: organized.title,
|
|
4100
|
-
content: organized.content,
|
|
4091
|
+
content: stripEventReferenceSection(organized.content),
|
|
4101
4092
|
sourceRefs: card.sourceRefs,
|
|
4102
4093
|
summary: organized.summary,
|
|
4103
4094
|
status: card.status,
|
|
@@ -4183,6 +4174,7 @@ export class KnowledgeRuntime {
|
|
|
4183
4174
|
}
|
|
4184
4175
|
async organizeCard(input) {
|
|
4185
4176
|
const context = await this.organizeContext({
|
|
4177
|
+
knowledge: input.knowledge,
|
|
4186
4178
|
cardTitle: input.cardTitle,
|
|
4187
4179
|
coordinate: input.coordinate,
|
|
4188
4180
|
profileId: input.profileId,
|
|
@@ -4511,7 +4503,16 @@ export class KnowledgeRuntime {
|
|
|
4511
4503
|
return cards.flatMap(card => this.compileCardNotes(card));
|
|
4512
4504
|
}
|
|
4513
4505
|
compileCardNotes(card) {
|
|
4514
|
-
|
|
4506
|
+
/**
|
|
4507
|
+
* **只排除 `deprecated`,不排除 `problem`。**
|
|
4508
|
+
*
|
|
4509
|
+
* `problem` 是人工标注的「这张卡有问题」,它仍然是知识,而召回契约里的
|
|
4510
|
+
* `include_problems` 是一个**读取时**的开关(见 `recall` 里 eligible 的那一行)。
|
|
4511
|
+
* 在编译期就把它的 note 丢掉,等于把这个开关变成死参数:无论传什么,编译产物
|
|
4512
|
+
* 里根本没有它的 note,FTS / 向量索引里也没有。所以索引层保留,过滤留给读路径。
|
|
4513
|
+
* `deprecated` 不同 —— 它是退役,编译与读取两层都该排除。
|
|
4514
|
+
*/
|
|
4515
|
+
if (card.status === 'deprecated')
|
|
4515
4516
|
return [];
|
|
4516
4517
|
const notes = [];
|
|
4517
4518
|
/**
|
|
@@ -4529,7 +4530,7 @@ export class KnowledgeRuntime {
|
|
|
4529
4530
|
* `metadata.related_cards`;这里继续剥离旧版本正文里的标记,保证存量卡片迁移期间
|
|
4530
4531
|
* 也不会把导航送进召回。
|
|
4531
4532
|
*/
|
|
4532
|
-
const recallBody = stripRelatedSection(card.content);
|
|
4533
|
+
const recallBody = stripEventReferenceSection(stripRelatedSection(card.content));
|
|
4533
4534
|
for (const [index, chunk] of splitWikiNotes({ ...card, content: recallBody }).entries()) {
|
|
4534
4535
|
const baseId = `${card.entityType}:${card.entityId}`;
|
|
4535
4536
|
const unitId = index === 0 ? baseId : `${baseId}#${index}`;
|
|
@@ -4562,13 +4563,20 @@ export class KnowledgeRuntime {
|
|
|
4562
4563
|
/** 1-hop neighbourhood of a set of wiki cards inside the compiled graph. */
|
|
4563
4564
|
oneHopGraph(keys, includeProblem, graph = this.graph) {
|
|
4564
4565
|
const visible = (node) => includeProblem || node.status !== 'problem';
|
|
4565
|
-
|
|
4566
|
+
/**
|
|
4567
|
+
* 必须读**传进来的** `graph`,不能读 `this.graph`。
|
|
4568
|
+
*
|
|
4569
|
+
* `this.graph` 是编译产物:卡片被删掉之后它不会跟着变小。调用方传的是
|
|
4570
|
+
* `visibleGraph(snapshot.graph, liveGraphKeys(truth))` —— 已按存活实体过滤过的那一份。
|
|
4571
|
+
* 用 `this.graph` 等于把删除掉的卡片留在召回结果的图里。
|
|
4572
|
+
*/
|
|
4573
|
+
const edges = graph.edges.filter(edge => keys.has(String(edge.from)) || keys.has(String(edge.to))).slice(0, 50);
|
|
4566
4574
|
const ids = new Set(keys);
|
|
4567
4575
|
for (const edge of edges) {
|
|
4568
4576
|
ids.add(String(edge.from));
|
|
4569
4577
|
ids.add(String(edge.to));
|
|
4570
4578
|
}
|
|
4571
|
-
const nodes =
|
|
4579
|
+
const nodes = graph.nodes
|
|
4572
4580
|
.filter(visible)
|
|
4573
4581
|
.filter(node => ids.has(`${node.type}:${node.id}`))
|
|
4574
4582
|
.slice(0, 50);
|
|
@@ -4595,10 +4603,11 @@ export class KnowledgeRuntime {
|
|
|
4595
4603
|
const provider = this.config.rerankProvider;
|
|
4596
4604
|
if (provider === 'none')
|
|
4597
4605
|
return { hits, result: { status: 'skipped', provider, threshold } };
|
|
4598
|
-
if (hits.length
|
|
4606
|
+
if (hits.length === 0)
|
|
4599
4607
|
return { hits, result: { status: 'skipped', provider, model: this.rerankModelPath, scored: hits.length, threshold } };
|
|
4600
|
-
|
|
4601
|
-
|
|
4608
|
+
const cooldownActive = this.rerankUnavailable !== undefined && Date.now() - this.rerankUnavailable.at < RERANK_RETRY_COOLDOWN_MS;
|
|
4609
|
+
if (this.rerankUnavailable && cooldownActive)
|
|
4610
|
+
return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error: this.rerankUnavailable.error, threshold }, warning: `重排不可用,已按融合分数排序:${this.rerankUnavailable.error}` };
|
|
4602
4611
|
try {
|
|
4603
4612
|
if (!this.rerankWorker)
|
|
4604
4613
|
this.rerankWorker = this.spawnWorker('RERANK');
|
|
@@ -4612,6 +4621,8 @@ export class KnowledgeRuntime {
|
|
|
4612
4621
|
const raw = Array.isArray(response?.scores) ? response.scores.map(Number) : [];
|
|
4613
4622
|
const outcome = applyRerank(raw, hits, threshold);
|
|
4614
4623
|
const vocabSize = Number(response?.vocabSize);
|
|
4624
|
+
// 一次成功的推理就是这条缓存的反证:清掉它,别让暂态失败变成会话级残疾。
|
|
4625
|
+
this.rerankUnavailable = undefined;
|
|
4615
4626
|
return {
|
|
4616
4627
|
hits: outcome.kept,
|
|
4617
4628
|
scored: outcome.scored,
|
|
@@ -4629,7 +4640,7 @@ export class KnowledgeRuntime {
|
|
|
4629
4640
|
// so a later recall does not respawn a doomed model every time.
|
|
4630
4641
|
this.rerankWorker?.close();
|
|
4631
4642
|
this.rerankWorker = null;
|
|
4632
|
-
this.rerankUnavailable = error;
|
|
4643
|
+
this.rerankUnavailable = { error, at: Date.now() };
|
|
4633
4644
|
return { hits, result: { status: 'unavailable', provider, model: this.rerankModelPath, error, threshold }, warning: `重排不可用,已按融合分数排序:${error}` };
|
|
4634
4645
|
}
|
|
4635
4646
|
}
|
|
@@ -4843,7 +4854,6 @@ export class KnowledgeRuntime {
|
|
|
4843
4854
|
*/
|
|
4844
4855
|
const profile = this.organizeProfile();
|
|
4845
4856
|
const profileId = profile?.id ?? 'concept';
|
|
4846
|
-
const eventsById = new Map((await this.loadEvents()).map(event => [event.id, event]));
|
|
4847
4857
|
for (const item of pending.values()) {
|
|
4848
4858
|
if (runId)
|
|
4849
4859
|
await this.checkpointCompileTask(runId);
|
|
@@ -4892,7 +4902,7 @@ export class KnowledgeRuntime {
|
|
|
4892
4902
|
&& layoutOf(existing.metadata)?.origin === 'model';
|
|
4893
4903
|
const finalTitle = preserveModelLayout ? existing.title : item.title;
|
|
4894
4904
|
const contentBeforeEvents = preserveModelLayout ? existing.content : content;
|
|
4895
|
-
const finalContent =
|
|
4905
|
+
const finalContent = stripEventReferenceSection(contentBeforeEvents);
|
|
4896
4906
|
const finalSummary = preserveModelLayout ? existing.summary : item.summary;
|
|
4897
4907
|
let finalPlacements = preserveModelLayout || item.placements === undefined
|
|
4898
4908
|
? undefined
|
|
@@ -4986,7 +4996,7 @@ export class KnowledgeRuntime {
|
|
|
4986
4996
|
for (const warning of written.warnings ?? [])
|
|
4987
4997
|
outcome.warnings.push(`卡片 ${card.entityId}:${warning}`);
|
|
4988
4998
|
}
|
|
4989
|
-
// WikiCard
|
|
4999
|
+
// WikiCard 只由 Knowledge 物化;Event 作为独立事实按坐标挂引用。
|
|
4990
5000
|
const eventSync = await this.syncEventReferences();
|
|
4991
5001
|
outcome.cardsUpdated += eventSync.updated;
|
|
4992
5002
|
outcome.warnings.push(...eventSync.warnings);
|
|
@@ -5035,8 +5045,9 @@ export class KnowledgeRuntime {
|
|
|
5035
5045
|
* 用的输入与收尾阶段那个相关块**完全相同**(同一份 `relations` + 同一个坐标投影),
|
|
5036
5046
|
* 所以「正文里能提到谁」与「末尾相关块列了谁」不会是两个答案。
|
|
5037
5047
|
*/
|
|
5038
|
-
const
|
|
5039
|
-
const
|
|
5048
|
+
const cardsWithoutLegacyEventSections = input.cards.map(card => ({ ...card, content: stripEventReferenceSection(card.content) }));
|
|
5049
|
+
const relatedPlan = planRelated(cardsWithoutLegacyEventSections, input.relations, await this.projectedRelated(input.cards));
|
|
5050
|
+
const withCoordinate = cardsWithoutLegacyEventSections
|
|
5040
5051
|
.filter(card => card.coordinate !== undefined)
|
|
5041
5052
|
.sort((left, right) => left.entityId.localeCompare(right.entityId));
|
|
5042
5053
|
if (withCoordinate.length === 0)
|
|
@@ -5366,7 +5377,8 @@ export class KnowledgeRuntime {
|
|
|
5366
5377
|
outcome.warnings.push(`Event 没有写入知识库:${cause instanceof Error ? cause.message : String(cause)}`);
|
|
5367
5378
|
}
|
|
5368
5379
|
outcome.warnings.push(...eventIngested.warnings);
|
|
5369
|
-
|
|
5380
|
+
outcome.warnings.push(...ingested.warnings);
|
|
5381
|
+
if (ingested.skipped > 0 || ingested.failures > 0 || eventIngested.skipped > 0)
|
|
5370
5382
|
extractionComplete = false;
|
|
5371
5383
|
// 落位要用库内的 id。这一份报告里每一条知识的映射都并进整轮那一份 ——
|
|
5372
5384
|
// 后面的来源可能命中同一张卡,那时落位指的行也得是活的。
|
|
@@ -5674,7 +5686,7 @@ export class KnowledgeRuntime {
|
|
|
5674
5686
|
const projected = await this.projectedRelated(cards);
|
|
5675
5687
|
const related = planRelated(cards, relations, projected);
|
|
5676
5688
|
/** 统计清理旧导航块之后的正文断链;相关卡片本身不再伪装成正文链接。 */
|
|
5677
|
-
const nextCards = cards.map(card => ({ ...card, content: stripRelatedSection(card.content) }));
|
|
5689
|
+
const nextCards = cards.map(card => ({ ...card, content: stripEventReferenceSection(stripRelatedSection(card.content)) }));
|
|
5678
5690
|
const deadLinks = deadLinksOf(nextCards);
|
|
5679
5691
|
let rewritten = 0;
|
|
5680
5692
|
for (const [index, card] of cards.entries()) {
|
|
@@ -6288,7 +6300,7 @@ export class KnowledgeRuntime {
|
|
|
6288
6300
|
if (store === null)
|
|
6289
6301
|
return { saved: false, why: '抽取轨迹库不可用(需要 Node 22.5+ 的 node:sqlite)' };
|
|
6290
6302
|
const report = input.report;
|
|
6291
|
-
const id = input.id?.trim() || extractionRunId();
|
|
6303
|
+
const id = input.id?.trim() || extractionRunId(input.stage);
|
|
6292
6304
|
const record = {
|
|
6293
6305
|
id,
|
|
6294
6306
|
kb: this.rootDir,
|
|
@@ -6300,6 +6312,7 @@ export class KnowledgeRuntime {
|
|
|
6300
6312
|
...(report?.model?.why ? { modelWhy: report.model.why } : {}),
|
|
6301
6313
|
summary: input.summary,
|
|
6302
6314
|
hasReport: true,
|
|
6315
|
+
stage: input.stage ?? 'extract',
|
|
6303
6316
|
report: input.report,
|
|
6304
6317
|
};
|
|
6305
6318
|
try {
|
|
@@ -6310,6 +6323,135 @@ export class KnowledgeRuntime {
|
|
|
6310
6323
|
return { saved: false, why: cause instanceof Error ? cause.message : String(cause) };
|
|
6311
6324
|
}
|
|
6312
6325
|
}
|
|
6326
|
+
/**
|
|
6327
|
+
* 归一段的轨迹(2026-10-18)。
|
|
6328
|
+
*
|
|
6329
|
+
* 这一段的产物以前只留在两处:任务里的流式输出(刷新就没),和事实表里最终写成的行
|
|
6330
|
+
* (只剩结果)。中间那步 —— 「模型给的这 283 个名字里,哪些命中了已有实体、哪些新建了
|
|
6331
|
+
* 候选、哪些因为不唯一没猜」—— 跑完就查不到了,而它恰好是坐标错了之后唯一能复盘的东西。
|
|
6332
|
+
*
|
|
6333
|
+
* **复用的那一次不落轨迹**:那一次什么都没做(指纹没变、一条事实都没重写),落一条
|
|
6334
|
+
* 「复用」只会把真正的运行挤出第一页。
|
|
6335
|
+
*/
|
|
6336
|
+
async recordNormalizeTrajectory(input) {
|
|
6337
|
+
const report = input.report;
|
|
6338
|
+
const axes = [['object', report.objects], ['action', report.actions], ['scene', report.scenes]].flatMap(([kind, rows]) => rows.map(axis => ({
|
|
6339
|
+
kind,
|
|
6340
|
+
name: axis.name,
|
|
6341
|
+
outcome: axis.outcome,
|
|
6342
|
+
...(axis.id ? { id: axis.id } : {}),
|
|
6343
|
+
...(axis.how ? { how: axis.how } : {}),
|
|
6344
|
+
aliases: axis.aliases,
|
|
6345
|
+
...(axis.parentMention ? { parentMention: axis.parentMention } : {}),
|
|
6346
|
+
})));
|
|
6347
|
+
const count = (outcome) => axes.filter(row => row.outcome === outcome).length;
|
|
6348
|
+
const unresolved = new Set(input.unresolvedKnowledgeIds);
|
|
6349
|
+
const payload = {
|
|
6350
|
+
stage: 'normalize',
|
|
6351
|
+
at: new Date().toISOString(),
|
|
6352
|
+
source: { id: input.source.sourceId, title: input.source.title, version: input.source.version },
|
|
6353
|
+
extractionId: input.checkpoint.id,
|
|
6354
|
+
fingerprint: input.fingerprint,
|
|
6355
|
+
mentions: report.objects.length + report.actions.length + report.scenes.length,
|
|
6356
|
+
counts: { matched: count('matched'), created: count('new'), ambiguous: count('ambiguous'), mechanical: count('no-registry') },
|
|
6357
|
+
axes: axes.slice(0, 400),
|
|
6358
|
+
knowledge: {
|
|
6359
|
+
saved: input.knowledge.saved,
|
|
6360
|
+
merged: input.knowledge.merged,
|
|
6361
|
+
skipped: input.knowledge.skipped,
|
|
6362
|
+
failures: input.knowledge.failures ?? 0,
|
|
6363
|
+
unresolved: unresolved.size,
|
|
6364
|
+
},
|
|
6365
|
+
events: { saved: input.events.saved, merged: input.events.merged, skipped: input.events.skipped },
|
|
6366
|
+
rows: report.knowledge.slice(0, 300).map(item => ({
|
|
6367
|
+
text: item.text,
|
|
6368
|
+
coordinate: [item.object, item.action, item.scene].filter(Boolean).join(' · ') || '无坐标',
|
|
6369
|
+
sources: item.sources.length,
|
|
6370
|
+
unresolved: unresolved.has(item.id),
|
|
6371
|
+
})),
|
|
6372
|
+
model: {
|
|
6373
|
+
ok: report.model.parsed === true,
|
|
6374
|
+
...(report.model.why ? { why: report.model.why } : {}),
|
|
6375
|
+
provider: report.model.request?.provider ?? input.context.model.provider,
|
|
6376
|
+
model: report.model.request?.model ?? input.context.model.model,
|
|
6377
|
+
reasoningEffort: report.model.request?.reasoningEffort ?? input.context.model.reasoningEffort,
|
|
6378
|
+
},
|
|
6379
|
+
...(report.axesContext ? { axesContext: report.axesContext } : {}),
|
|
6380
|
+
warnings: input.warnings,
|
|
6381
|
+
};
|
|
6382
|
+
const saved = await this.recordExtractionRun({
|
|
6383
|
+
stage: 'normalize',
|
|
6384
|
+
docId: input.source.sourceId,
|
|
6385
|
+
title: `${input.source.title} · 归一`,
|
|
6386
|
+
summary: `命中 ${payload.counts.matched} · 新建候选 ${payload.counts.created} · 歧义 ${payload.counts.ambiguous};写入 ${payload.knowledge.saved} 条 Knowledge / ${payload.events.saved} 个 Event`,
|
|
6387
|
+
report: payload,
|
|
6388
|
+
});
|
|
6389
|
+
if (!saved.saved)
|
|
6390
|
+
throw new Error(saved.why ?? '归一轨迹没有存下来');
|
|
6391
|
+
}
|
|
6392
|
+
/**
|
|
6393
|
+
* 物化段的轨迹(2026-10-18):一个卡片分组 → 一张卡。
|
|
6394
|
+
*
|
|
6395
|
+
* 这一段的产物是卡片本身,所以「卡里有什么」已经查得到;查不到的是**这一次为什么这么排**:
|
|
6396
|
+
* 分组指纹、卡片组织的结论(模型排的还是机械拼的)、进了哪几条知识、正文里为什么链到
|
|
6397
|
+
* 那一张卡。这些是改完提示词、想知道「这次和上次差在哪」时唯一能对着看的东西。
|
|
6398
|
+
*
|
|
6399
|
+
* 粒度是**一组**而不是一个任务:一张卡组织失败,不该让同批其它卡的轨迹一起消失。
|
|
6400
|
+
*/
|
|
6401
|
+
async recordMaterializeTrajectory(input) {
|
|
6402
|
+
const sections = [];
|
|
6403
|
+
for (const row of input.placements ?? []) {
|
|
6404
|
+
const hit = sections.find(section => section.label === row.section);
|
|
6405
|
+
if (hit)
|
|
6406
|
+
hit.count += 1;
|
|
6407
|
+
else
|
|
6408
|
+
sections.push({ label: row.section, count: 1 });
|
|
6409
|
+
}
|
|
6410
|
+
const payload = {
|
|
6411
|
+
stage: 'materialize',
|
|
6412
|
+
at: new Date().toISOString(),
|
|
6413
|
+
group: {
|
|
6414
|
+
key: input.group.key,
|
|
6415
|
+
title: input.group.title,
|
|
6416
|
+
coordinate: [input.group.coordinate.objectId, input.group.coordinate.actionId, input.group.coordinate.sceneId].filter(Boolean).join(' · ') || '无坐标',
|
|
6417
|
+
state: input.group.state,
|
|
6418
|
+
fingerprint: input.group.fingerprint,
|
|
6419
|
+
},
|
|
6420
|
+
action: input.action,
|
|
6421
|
+
card: {
|
|
6422
|
+
id: input.card.id,
|
|
6423
|
+
title: input.card.title,
|
|
6424
|
+
status: input.card.status,
|
|
6425
|
+
characters: input.card.characters,
|
|
6426
|
+
knowledge: input.knowledge.length,
|
|
6427
|
+
events: input.card.events,
|
|
6428
|
+
images: input.card.images,
|
|
6429
|
+
sections,
|
|
6430
|
+
},
|
|
6431
|
+
organize: input.organizeUsed
|
|
6432
|
+
? { used: true, ...(input.layout?.origin ? { origin: input.layout.origin } : {}), ...(input.layout?.hash ? { layoutHash: input.layout.hash } : {}) }
|
|
6433
|
+
: { used: false },
|
|
6434
|
+
rows: input.knowledge.slice(0, 300).map(row => ({ id: row.id, text: row.text, sources: row.sources.length })),
|
|
6435
|
+
sources: [...input.group.sourceIds],
|
|
6436
|
+
related: input.related.map(candidate => ({ title: candidate.title, reason: candidate.reason })),
|
|
6437
|
+
model: {
|
|
6438
|
+
ok: true,
|
|
6439
|
+
provider: input.context.model.provider,
|
|
6440
|
+
model: input.context.model.model,
|
|
6441
|
+
reasoningEffort: input.context.model.reasoningEffort,
|
|
6442
|
+
},
|
|
6443
|
+
warnings: input.warnings ?? [],
|
|
6444
|
+
};
|
|
6445
|
+
const label = payload.action === 'created' ? '新建' : payload.action === 'updated' ? '更新' : '停用';
|
|
6446
|
+
const saved = await this.recordExtractionRun({
|
|
6447
|
+
stage: 'materialize',
|
|
6448
|
+
title: `${payload.card.title || payload.group.title} · ${label}`,
|
|
6449
|
+
summary: `${label}「${payload.card.title || payload.group.title}」:${payload.card.knowledge} 条知识 · ${payload.card.events} 个事件 · ${payload.card.images} 张图 · ${payload.card.sections.length} 节${payload.organize.used ? '' : '(机械拼装,未用卡片组织)'}`,
|
|
6450
|
+
report: payload,
|
|
6451
|
+
});
|
|
6452
|
+
if (!saved.saved)
|
|
6453
|
+
throw new Error(saved.why ?? '物化轨迹没有存下来');
|
|
6454
|
+
}
|
|
6313
6455
|
/**
|
|
6314
6456
|
* 轴注册表(0063 §3):Object / Action / Scene 的 canonical 身份。
|
|
6315
6457
|
*
|
|
@@ -6390,6 +6532,7 @@ export class KnowledgeRuntime {
|
|
|
6390
6532
|
* 不能当成「没命中」。
|
|
6391
6533
|
*/
|
|
6392
6534
|
async resolveTrialAxisSet(input) {
|
|
6535
|
+
const resolve = (value) => this.resolveAxisMention({ ...value, registry: input.registry });
|
|
6393
6536
|
const groups = [
|
|
6394
6537
|
{ kind: 'object', key: 'objects' },
|
|
6395
6538
|
{ kind: 'action', key: 'actions' },
|
|
@@ -6400,7 +6543,7 @@ export class KnowledgeRuntime {
|
|
|
6400
6543
|
const axisKey = (kind, mention) => `${kind}\u0000${normalizeAxisName(mention)}`;
|
|
6401
6544
|
const supplementalEntries = { object: [], action: [], scene: [] };
|
|
6402
6545
|
try {
|
|
6403
|
-
const registry = await this.axisRegistry();
|
|
6546
|
+
const registry = input.registry ?? await this.axisRegistry();
|
|
6404
6547
|
if (registry !== null) {
|
|
6405
6548
|
// 初次抽取的候选清单可能被截断。再按本批轴 mention 做一次本地召回,补进未下发的已有坐标。
|
|
6406
6549
|
const query = groups.flatMap(group => input.axes[group.key].flatMap(axis => [
|
|
@@ -6455,7 +6598,7 @@ export class KnowledgeRuntime {
|
|
|
6455
6598
|
const key = axisKey(group.kind, axis.name);
|
|
6456
6599
|
let lookup;
|
|
6457
6600
|
try {
|
|
6458
|
-
lookup = await
|
|
6601
|
+
lookup = await resolve({
|
|
6459
6602
|
kind: group.kind,
|
|
6460
6603
|
mention: axis.name,
|
|
6461
6604
|
parentMention: axis.parentMention,
|
|
@@ -6519,7 +6662,7 @@ export class KnowledgeRuntime {
|
|
|
6519
6662
|
continue;
|
|
6520
6663
|
}
|
|
6521
6664
|
// 没有可供模型比较的现有候选时,沿用注册表的新建候选路径。
|
|
6522
|
-
const created = await
|
|
6665
|
+
const created = await resolve({
|
|
6523
6666
|
kind: group.kind,
|
|
6524
6667
|
mention: axis.name,
|
|
6525
6668
|
parentMention: axis.parentMention,
|
|
@@ -6629,7 +6772,7 @@ export class KnowledgeRuntime {
|
|
|
6629
6772
|
const key = axisKey(item.kind, item.axis.name);
|
|
6630
6773
|
const judgment = decisions.get(key);
|
|
6631
6774
|
if (judgment?.decision === 'new') {
|
|
6632
|
-
const resolution = await
|
|
6775
|
+
const resolution = await resolve({
|
|
6633
6776
|
kind: item.kind,
|
|
6634
6777
|
mention: item.axis.name,
|
|
6635
6778
|
parentMention: item.axis.parentMention,
|
|
@@ -6643,7 +6786,7 @@ export class KnowledgeRuntime {
|
|
|
6643
6786
|
if (judgment?.decision === 'match') {
|
|
6644
6787
|
const candidate = item.candidates.find(value => value.id === judgment.candidateId);
|
|
6645
6788
|
if (candidate) {
|
|
6646
|
-
const resolution = await
|
|
6789
|
+
const resolution = await resolve({
|
|
6647
6790
|
kind: item.kind,
|
|
6648
6791
|
mention: item.axis.name,
|
|
6649
6792
|
parentMention: item.axis.parentMention,
|
|
@@ -6672,7 +6815,7 @@ export class KnowledgeRuntime {
|
|
|
6672
6815
|
};
|
|
6673
6816
|
}
|
|
6674
6817
|
async resolveAxisMention(input) {
|
|
6675
|
-
const registry = await this.axisRegistry();
|
|
6818
|
+
const registry = input.registry ?? await this.axisRegistry();
|
|
6676
6819
|
if (registry === null)
|
|
6677
6820
|
return null;
|
|
6678
6821
|
const runId = input.runId ?? '';
|
|
@@ -6688,7 +6831,7 @@ export class KnowledgeRuntime {
|
|
|
6688
6831
|
if (parent.entity)
|
|
6689
6832
|
parentId = parent.entity.id;
|
|
6690
6833
|
}
|
|
6691
|
-
const preferId = await this.axisIdByName(input.kind, input.preferMention);
|
|
6834
|
+
const preferId = await this.axisIdByName(input.kind, input.preferMention, registry);
|
|
6692
6835
|
return registry.resolve({
|
|
6693
6836
|
kind: input.kind, mention: input.mention,
|
|
6694
6837
|
...(parentId ? { parentId } : {}), ...(runId ? { runId } : {}),
|
|
@@ -6704,11 +6847,11 @@ export class KnowledgeRuntime {
|
|
|
6704
6847
|
* 「是已有的某个东西」而那个名字在库里指两个实体时,把它当成指认就等于替人做了
|
|
6705
6848
|
* 「哪一个」的决定 —— 而注册表的规矩是「不唯一就不猜」。
|
|
6706
6849
|
*/
|
|
6707
|
-
async axisIdByName(kind, mention) {
|
|
6850
|
+
async axisIdByName(kind, mention, supplied) {
|
|
6708
6851
|
const name = (mention ?? '').trim();
|
|
6709
6852
|
if (!name)
|
|
6710
6853
|
return undefined;
|
|
6711
|
-
const registry = await this.axisRegistry();
|
|
6854
|
+
const registry = supplied ?? await this.axisRegistry();
|
|
6712
6855
|
if (registry === null)
|
|
6713
6856
|
return undefined;
|
|
6714
6857
|
try {
|
|
@@ -7022,7 +7165,7 @@ export class KnowledgeRuntime {
|
|
|
7022
7165
|
const cardOrphans = cards.filter(card => card.sourceRefs.every(ref => !live.has(ref))).length;
|
|
7023
7166
|
const events = (await this.loadEvents()).filter(event => event.records.some(record => !live.has(record.sourceId))).length;
|
|
7024
7167
|
/**
|
|
7025
|
-
* 未被引用的来源图片也进这张账(2026-10-05)。`
|
|
7168
|
+
* 未被引用的来源图片也进这张账(2026-10-05)。`image_publish` 写入的独立图片会被识别并保留。
|
|
7026
7169
|
*
|
|
7027
7170
|
* 少了它,「来源与卡片都清完了、只剩一堆没人引用的图」这种状态在界面上**一个字都没有**
|
|
7028
7171
|
* —— 清理孤儿的入口因此不出现,而用户唯一能看到的读数(「收录 N 个文件」)又把它算成
|
|
@@ -7059,7 +7202,7 @@ export class KnowledgeRuntime {
|
|
|
7059
7202
|
}
|
|
7060
7203
|
/**
|
|
7061
7204
|
* 还有没有人在引用这个桶 —— 判据是来源正文、来源 front matter 的 `metadata.assets`、
|
|
7062
|
-
* 卡片正文出现过 `_assets/<bucket>/`,或者它是 `
|
|
7205
|
+
* 卡片正文出现过 `_assets/<bucket>/`,或者它是 `image_publish` 写入的独立图床图片。
|
|
7063
7206
|
* 独立图片没有来源/卡片引用是正常状态,必须保留。
|
|
7064
7207
|
*
|
|
7065
7208
|
* 为什么三处都要看:桶名是**内容哈希的前 12 位**(`convert/ingest.ts`),不是来源 id。
|
|
@@ -7103,7 +7246,7 @@ export class KnowledgeRuntime {
|
|
|
7103
7246
|
return unreferenced;
|
|
7104
7247
|
}
|
|
7105
7248
|
/**
|
|
7106
|
-
* `
|
|
7249
|
+
* `image_publish` 的图片不需要被来源或卡片引用。新写入的桶有明确标记;同时兼容
|
|
7107
7250
|
* 标记加入前已发布的图片:它们以图片自身的 SHA-256 前 12 位作为桶名。
|
|
7108
7251
|
*/
|
|
7109
7252
|
async isPublishedImageBucket(name) {
|
|
@@ -7111,7 +7254,7 @@ export class KnowledgeRuntime {
|
|
|
7111
7254
|
if (existsSync(join(root, name, PUBLISHED_IMAGE_MARKER)))
|
|
7112
7255
|
return true;
|
|
7113
7256
|
}
|
|
7114
|
-
//
|
|
7257
|
+
// 历史版本发布的图片只会写入知识库根目录的 assets/,因此只在这里做哈希识别。
|
|
7115
7258
|
if (!/^[a-f0-9]{12}$/i.test(name))
|
|
7116
7259
|
return false;
|
|
7117
7260
|
const dir = join(this.rootDir, ASSETS_DIR, name);
|
|
@@ -7362,11 +7505,13 @@ export class KnowledgeRuntime {
|
|
|
7362
7505
|
async ingestTrialKnowledge(input) {
|
|
7363
7506
|
const store = await this.knowledgeStore();
|
|
7364
7507
|
if (store === null)
|
|
7365
|
-
return { saved: 0, merged: 0, skipped: input.knowledge.length, ids: {} };
|
|
7508
|
+
return { saved: 0, merged: 0, skipped: input.knowledge.length, failures: 0, warnings: [], ids: {} };
|
|
7366
7509
|
const axes = { objects: input.objects, actions: input.actions, scenes: input.scenes };
|
|
7367
7510
|
let saved = 0;
|
|
7368
7511
|
let merged = 0;
|
|
7369
7512
|
let skipped = 0;
|
|
7513
|
+
let failures = 0;
|
|
7514
|
+
const warnings = [];
|
|
7370
7515
|
const ids = {};
|
|
7371
7516
|
for (const item of input.knowledge) {
|
|
7372
7517
|
const usable = item.sources.filter(source => source.found);
|
|
@@ -7405,11 +7550,20 @@ export class KnowledgeRuntime {
|
|
|
7405
7550
|
*/
|
|
7406
7551
|
ids[item.id] = result.knowledge.id;
|
|
7407
7552
|
}
|
|
7408
|
-
catch {
|
|
7409
|
-
|
|
7553
|
+
catch (cause) {
|
|
7554
|
+
/*
|
|
7555
|
+
* 写入失败与「引文没通过核验」必须分开报。
|
|
7556
|
+
*
|
|
7557
|
+
* 原来这里是一个 `catch { skipped += 1 }`:磁盘满、SQLITE_BUSY、约束冲突
|
|
7558
|
+
* 全都被算成「引文没核验过」,原因被丢掉,而下游只看到同一句话 ——
|
|
7559
|
+
* 运维读到的是一份数据问题,实际是基础设施问题,于是无限重试。
|
|
7560
|
+
* Event 那条路径本来就把消息收进 warnings(见下面),这里对齐它。
|
|
7561
|
+
*/
|
|
7562
|
+
failures += 1;
|
|
7563
|
+
warnings.push(`Knowledge「${item.text.slice(0, 40)}」写入失败:${cause instanceof Error ? cause.message : String(cause)}`);
|
|
7410
7564
|
}
|
|
7411
7565
|
}
|
|
7412
|
-
return { saved, merged, skipped, ids };
|
|
7566
|
+
return { saved, merged, skipped, failures, warnings, ids };
|
|
7413
7567
|
}
|
|
7414
7568
|
/**
|
|
7415
7569
|
* 把抽取到的 Event 写成独立事件实体,并按身份规则追加来源记录。
|
|
@@ -7461,14 +7615,12 @@ export class KnowledgeRuntime {
|
|
|
7461
7615
|
refs.push(event.id);
|
|
7462
7616
|
eventsByCoordinate.set(key, refs);
|
|
7463
7617
|
}
|
|
7464
|
-
const eventIndex = new Map(events.map(event => [event.id, event]));
|
|
7465
7618
|
const warnings = [];
|
|
7466
7619
|
let updated = 0;
|
|
7467
7620
|
for (const card of await this.loadCards()) {
|
|
7468
|
-
const refs = card.coordinate === undefined
|
|
7621
|
+
const refs = card.status === 'deprecated' || card.coordinate === undefined || card.entityType === 'diagnosis/case'
|
|
7469
7622
|
? []
|
|
7470
7623
|
: (eventsByCoordinate.get(coordinateKeyOf(card.coordinate)) ?? []).sort();
|
|
7471
|
-
db.setCardEvents(card.entityId, refs);
|
|
7472
7624
|
const oldRaw = card.metadata.event_refs;
|
|
7473
7625
|
let oldRefs = [];
|
|
7474
7626
|
let oldRefsValid = false;
|
|
@@ -7486,13 +7638,17 @@ export class KnowledgeRuntime {
|
|
|
7486
7638
|
oldRefs = oldRaw.filter((item) => typeof item === 'string').sort();
|
|
7487
7639
|
oldRefsValid = true;
|
|
7488
7640
|
}
|
|
7489
|
-
const content =
|
|
7641
|
+
const content = stripEventReferenceSection(card.content);
|
|
7490
7642
|
const contentChanged = content !== card.content;
|
|
7491
7643
|
const refsChanged = !oldRefsValid || JSON.stringify(refs) !== JSON.stringify(oldRefs);
|
|
7492
|
-
if (!refsChanged && !contentChanged)
|
|
7644
|
+
if (!refsChanged && !contentChanged) {
|
|
7645
|
+
db.setCardEvents(card.entityId, refs);
|
|
7493
7646
|
continue;
|
|
7494
|
-
|
|
7647
|
+
}
|
|
7648
|
+
if (!oldRefsValid && refs.length === 0 && oldRaw === undefined && !contentChanged) {
|
|
7649
|
+
db.setCardEvents(card.entityId, refs);
|
|
7495
7650
|
continue;
|
|
7651
|
+
}
|
|
7496
7652
|
const written = await this.writeCard({
|
|
7497
7653
|
...card,
|
|
7498
7654
|
sourceRefs: card.sourceRefs,
|
|
@@ -7501,9 +7657,11 @@ export class KnowledgeRuntime {
|
|
|
7501
7657
|
origin: 'internal',
|
|
7502
7658
|
});
|
|
7503
7659
|
if (written.ok === false)
|
|
7504
|
-
warnings.push(
|
|
7505
|
-
else
|
|
7660
|
+
warnings.push(`WikiCard ${card.title} 的 Event 引用未更新:${written.error.message}`);
|
|
7661
|
+
else {
|
|
7662
|
+
db.setCardEvents(card.entityId, refs);
|
|
7506
7663
|
updated += 1;
|
|
7664
|
+
}
|
|
7507
7665
|
}
|
|
7508
7666
|
return { updated, warnings };
|
|
7509
7667
|
}
|
|
@@ -7524,10 +7682,6 @@ export class KnowledgeRuntime {
|
|
|
7524
7682
|
const ids = {};
|
|
7525
7683
|
for (const event of db.events()) {
|
|
7526
7684
|
const remaining = event.records.filter(record => record.sourceId !== input.sourceId);
|
|
7527
|
-
if (remaining.length === 0) {
|
|
7528
|
-
db.removeEvent(event.id);
|
|
7529
|
-
continue;
|
|
7530
|
-
}
|
|
7531
7685
|
const state = {
|
|
7532
7686
|
id: event.id,
|
|
7533
7687
|
identityKey: event.identityKey,
|
|
@@ -7629,8 +7783,12 @@ export class KnowledgeRuntime {
|
|
|
7629
7783
|
saved += 1;
|
|
7630
7784
|
}
|
|
7631
7785
|
for (const state of states.values()) {
|
|
7632
|
-
if (!state.dirty
|
|
7786
|
+
if (!state.dirty)
|
|
7787
|
+
continue;
|
|
7788
|
+
if (state.records.length === 0) {
|
|
7789
|
+
db.removeEvent(state.id);
|
|
7633
7790
|
continue;
|
|
7791
|
+
}
|
|
7634
7792
|
const records = state.records;
|
|
7635
7793
|
const entity = eventEntityFromRecords({
|
|
7636
7794
|
id: state.id,
|
|
@@ -7650,7 +7808,565 @@ export class KnowledgeRuntime {
|
|
|
7650
7808
|
}
|
|
7651
7809
|
return { saved, merged, skipped, ids, warnings };
|
|
7652
7810
|
}
|
|
7653
|
-
/**
|
|
7811
|
+
/** The stage runner shares this base's existing writer queue and audit database. */
|
|
7812
|
+
async pipeline() {
|
|
7813
|
+
if (this.closed)
|
|
7814
|
+
throw new Error('知识库已关闭');
|
|
7815
|
+
this.pipelineOpening ??= (async () => {
|
|
7816
|
+
const db = await this.storeHandle();
|
|
7817
|
+
const audit = await this.extractionRuns();
|
|
7818
|
+
if (this.closed)
|
|
7819
|
+
throw new Error('知识库已关闭');
|
|
7820
|
+
if (!db || !audit || !await this.knowledgeStore())
|
|
7821
|
+
throw new Error('知识库或任务历史不可用,无法启动处理任务');
|
|
7822
|
+
const store = new PipelineStore(db.connection(), audit.connection());
|
|
7823
|
+
const engine = new KnowledgePipeline(store, {
|
|
7824
|
+
capture: () => ({ model: this.modelSeamNow(), note: this.extractionNote(), profile: structuredClone(this.organizeProfile()), organize: this.config.cardOrganize, format: 'table' }),
|
|
7825
|
+
serialize: work => {
|
|
7826
|
+
this.compilePending += 1;
|
|
7827
|
+
const run = this.compileTail.then(work, work);
|
|
7828
|
+
this.compileTail = run.then(() => undefined, () => undefined).finally(() => { this.compilePending -= 1; });
|
|
7829
|
+
return run;
|
|
7830
|
+
},
|
|
7831
|
+
sources: () => this.pipelineSources(),
|
|
7832
|
+
source: sourceId => this.pipelineSource(sourceId),
|
|
7833
|
+
/**
|
|
7834
|
+
* 抽取指纹必须覆盖**真正喂给模型的那段实体清单**(0070),但要按本文提到的条目收窄。
|
|
7835
|
+
*
|
|
7836
|
+
* `extract` 把 `axisPromptBlock(source.content)` 拼进提示词(注册表里的实体名、
|
|
7837
|
+
* 别名、定义与人工指引都在里面),而这些会随人的确认、合并与模型的补定义变化。
|
|
7838
|
+
* 指纹漏掉它 → 注册表变了仍复用旧报告,模型当时的 `existing` / `dropped` 判断
|
|
7839
|
+
* 被冻住。旧路径的指纹是含注册表成分的(`axesRevision`),这里原来不对称。
|
|
7840
|
+
*
|
|
7841
|
+
* 见 `pipelineExtractionFingerprint` —— 那里还有一条必需的配套:归一成功后要**刷新**这份
|
|
7842
|
+
* 指纹,否则归一自己学到的实体(它们本来就出现在这份文档里)会让它永远无法复用。
|
|
7843
|
+
*/
|
|
7844
|
+
fingerprint: (source, context) => this.pipelineExtractionFingerprint(source, context),
|
|
7845
|
+
extract: async (source, context, onDelta) => {
|
|
7846
|
+
if (!context.model.call)
|
|
7847
|
+
throw new Error(`抽取模型不可用:${context.model.why ?? '请在编译设置中选择模型'}`);
|
|
7848
|
+
const axes = await this.axisPromptBlock(source.content);
|
|
7849
|
+
return runWikiTrial({ title: source.title, text: source.content, docId: source.sourceId, axes, note: context.note, call: context.model.call, modelRequest: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort }, outputFormat: context.format, sessionId: `knowledge:${this.baseId || 'default'}`, onDelta });
|
|
7850
|
+
},
|
|
7851
|
+
normalize: (source, checkpoint, context, force, output) => this.normalizePipelineSource(store, source, checkpoint, context, force, output),
|
|
7852
|
+
/**
|
|
7853
|
+
* 引擎只把 `snapshot` 原样传回来,所以那边看到的是 `unknown`;这里把它取回成
|
|
7854
|
+
* 真实类型(见 `materialize-snapshot.ts` 里为什么不放在 `pipeline/types.ts`)。
|
|
7855
|
+
*/
|
|
7856
|
+
groups: async (context) => {
|
|
7857
|
+
const snapshot = await this.materializationSnapshot(store, context);
|
|
7858
|
+
return { groups: snapshot.groups, blocked: snapshot.blocked, blockedAmbiguous: snapshot.blockedAmbiguous, blockedAwaitingNormalize: snapshot.blockedAwaitingNormalize, snapshot };
|
|
7859
|
+
},
|
|
7860
|
+
materialize: (group, context, output, snapshot) => this.materializePipelineGroup(store, group, context, output, snapshot),
|
|
7861
|
+
build: async (taskId, checkpoint) => {
|
|
7862
|
+
let result;
|
|
7863
|
+
try {
|
|
7864
|
+
result = await this.compileOnce(taskId, 'all', undefined, { extract: false, checkpoint });
|
|
7865
|
+
}
|
|
7866
|
+
catch (cause) {
|
|
7867
|
+
await this.settleProgressFailure(taskId, cause);
|
|
7868
|
+
throw cause;
|
|
7869
|
+
}
|
|
7870
|
+
this.settleProgressReport(taskId, result);
|
|
7871
|
+
if (!result.ok)
|
|
7872
|
+
throw new Error(result.error.message);
|
|
7873
|
+
return result.data.skipped
|
|
7874
|
+
? { summary: `索引无变化:复用 ${result.data.notesReused ?? 0} 条召回片段,仍为 v${result.data.version}。`, skipped: true }
|
|
7875
|
+
: { summary: `检索索引已更新至 v${result.data.version}。`, skipped: false };
|
|
7876
|
+
},
|
|
7877
|
+
record: async (checkpoint) => {
|
|
7878
|
+
/**
|
|
7879
|
+
* `scope: 'extract-only'` —— 这份报告的 ②③④ 段**不是这个库里真的做过的事**。
|
|
7880
|
+
*
|
|
7881
|
+
* 报告的形状来自 `runWikiTrial`(试跑:一次调用里把归一、物化、建索引的机械结果
|
|
7882
|
+
* 也顺带算出来)。但在这条流水线里,归一与物化是**独立阶段**,各有自己的任务与轨迹。
|
|
7883
|
+
* 不标这一笔,用户点开抽取轨迹会看到「物化 · 0 张 Wiki Card」并以为物化跑过了。
|
|
7884
|
+
*
|
|
7885
|
+
* 旧的全链路编译(`compileOnce`)不标:那一条路上报告里的四段是真跑过的。
|
|
7886
|
+
*/
|
|
7887
|
+
const audit = await this.recordExtractionRun({ id: checkpoint.id, docId: checkpoint.sourceId, title: checkpoint.report.title, summary: trialSummary(checkpoint.report), report: { ...checkpoint.report, scope: 'extract-only' } });
|
|
7888
|
+
if (!audit.saved)
|
|
7889
|
+
throw new Error(`抽取报告保存失败:${audit.why}`);
|
|
7890
|
+
},
|
|
7891
|
+
});
|
|
7892
|
+
this.pipelineEngine = engine;
|
|
7893
|
+
if (this.closed)
|
|
7894
|
+
engine.close();
|
|
7895
|
+
return engine;
|
|
7896
|
+
})();
|
|
7897
|
+
return this.pipelineOpening;
|
|
7898
|
+
}
|
|
7899
|
+
async pipelineSources() {
|
|
7900
|
+
const db = await this.storeHandle();
|
|
7901
|
+
if (!db)
|
|
7902
|
+
throw new Error('来源存储不可用');
|
|
7903
|
+
return db.sources().map(row => ({ ...row.record, version: row.version, sourceHash: contentHashOf(row.record.content) })).sort((a, b) => a.sourceId.localeCompare(b.sourceId));
|
|
7904
|
+
}
|
|
7905
|
+
async pipelineSource(sourceId) {
|
|
7906
|
+
const db = await this.storeHandle();
|
|
7907
|
+
if (!db)
|
|
7908
|
+
throw new Error('来源存储不可用');
|
|
7909
|
+
const row = db.source(sourceId);
|
|
7910
|
+
return row ? { ...row.record, version: row.version, sourceHash: contentHashOf(row.record.content) } : undefined;
|
|
7911
|
+
}
|
|
7912
|
+
/**
|
|
7913
|
+
* 抽取指纹(0070)。
|
|
7914
|
+
*
|
|
7915
|
+
* 判据是「这段提示词会不会变」,所以实体清单要按**本文真的提到**的条目收窄:小库发的是
|
|
7916
|
+
* 全集(`selection: 'full'`),直接哈希整块清单会让「新增一个与本文无关的实体」触发全库
|
|
7917
|
+
* 重抽 —— 而那正是既有测试钉住不许发生的退化。
|
|
7918
|
+
*
|
|
7919
|
+
* **已知的取舍**(不是疏漏):提示词里那份清单是 `promptEntries` 按相关度排/截出来的,
|
|
7920
|
+
* 而这里按字面包含关系重筛一遍。两者不完全等价 —— 「在提示词里但本文没有字面提到」的
|
|
7921
|
+
* 实体被改名时,这里不会失效。选择这条判据是因为另一条(原样哈希发出去的清单)与
|
|
7922
|
+
* 上面那条被测试钉住的产品规则直接冲突。残留风险由归一那一侧兜住:归一总是对着**实时**
|
|
7923
|
+
* 注册表重新解析坐标,所以模型当时的 `existing` 判断不会原样变成最终卡名。
|
|
7924
|
+
*
|
|
7925
|
+
* 它有两个调用点:抽取前判断能不能复用,以及归一成功后**刷新**已存的那一份。后者是必需的:
|
|
7926
|
+
* 归一学到的实体本来就出现在这份文档里,不刷新的话这份文档下次抽取永远复用不上。这与
|
|
7927
|
+
* `normalizationFingerprint` 存「补判之后」的快照是同一条规则,不是两套。
|
|
7928
|
+
*/
|
|
7929
|
+
async pipelineExtractionFingerprint(source, context) {
|
|
7930
|
+
const axes = await this.axisPromptBlock(source.content);
|
|
7931
|
+
const text = normalizeAxisName(source.content);
|
|
7932
|
+
const relevant = ['object', 'action', 'scene'].map(kind => (axes?.entries[kind] ?? [])
|
|
7933
|
+
.filter(entry => [entry.name, ...entry.aliases].map(normalizeAxisName).some(value => value.length > 0 && text.includes(value)))
|
|
7934
|
+
.map(entry => ({ id: entry.id, name: entry.name, aliases: entry.aliases, description: entry.description, guidance: entry.guidance }))
|
|
7935
|
+
.sort((left, right) => left.id.localeCompare(right.id)));
|
|
7936
|
+
return contentHashOf(JSON.stringify({ source: source.sourceHash, title: source.title, contract: EXTRACTION_CONTRACT_VERSION, note: context.note, format: context.format, axes: relevant, model: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort } }));
|
|
7937
|
+
}
|
|
7938
|
+
async normalizationFingerprint(checkpoint, context) {
|
|
7939
|
+
const registry = await this.axisRegistry();
|
|
7940
|
+
const report = checkpoint.normalized?.report ?? checkpoint.report;
|
|
7941
|
+
const query = [...report.objects, ...report.actions, ...report.scenes].flatMap(axis => [axis.name, axis.existing ?? '', axis.definition ?? '', axis.parentMention ?? '']).join('\n');
|
|
7942
|
+
const candidates = await registry?.promptEntries(24, query);
|
|
7943
|
+
const mentions = [...checkpoint.report.objects, ...checkpoint.report.actions, ...checkpoint.report.scenes].flatMap(axis => [axis.name, axis.existing ?? '', axis.parentMention ?? '']).map(normalizeAxisName).filter(Boolean);
|
|
7944
|
+
const resolvedIds = new Set([...report.objects, ...report.actions, ...report.scenes].map(axis => axis.id).filter(Boolean));
|
|
7945
|
+
// Ignore unrelated filler rows and registry recency; retain candidates that can affect these mentions.
|
|
7946
|
+
const relevant = Object.fromEntries(Object.entries(candidates?.entries ?? {}).map(([kind, entries]) => [kind, entries.filter(entry => resolvedIds.has(entry.id) || mentions.some(mention => [entry.name, ...entry.aliases, entry.description].map(normalizeAxisName).some(value => value.includes(mention) || (value.length > 1 && mention.includes(value))))).sort((a, b) => a.id.localeCompare(b.id))]));
|
|
7947
|
+
return contentHashOf(JSON.stringify({ extraction: checkpoint.id, model: { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort }, candidates: relevant }));
|
|
7948
|
+
}
|
|
7949
|
+
async normalizePipelineSource(store, source, checkpoint, context, force, output) {
|
|
7950
|
+
const fingerprint = await this.normalizationFingerprint(checkpoint, context);
|
|
7951
|
+
if (!force && checkpoint.normalized?.fingerprint === fingerprint && store.head(source.sourceId).normalized?.id === checkpoint.id)
|
|
7952
|
+
return { reused: true, warnings: [] };
|
|
7953
|
+
if (!checkpoint.report.model.response)
|
|
7954
|
+
throw new Error('抽取记录没有原始返回,无法归一;请重新抽取该文档');
|
|
7955
|
+
const registry = await this.axisRegistry();
|
|
7956
|
+
if (!registry)
|
|
7957
|
+
throw new Error('实体注册表不可用');
|
|
7958
|
+
const draft = await registry.draft();
|
|
7959
|
+
try {
|
|
7960
|
+
const axesBlock = await this.axisPromptBlock(source.content);
|
|
7961
|
+
const call = context.model.call ? request => context.model.call({ ...request, onDelta: output }) : undefined;
|
|
7962
|
+
const report = await runWikiTrial({
|
|
7963
|
+
title: source.title, text: source.content, docId: source.sourceId, axes: axesBlock,
|
|
7964
|
+
// Reparse the captured response; this callback never requests another extraction.
|
|
7965
|
+
call: async () => checkpoint.report.model.response, modelRequest: checkpoint.report.model.request,
|
|
7966
|
+
resolveAxisSet: axes => this.resolveTrialAxisSet({ registry: draft.registry, axes, axesBlock, runId: checkpoint.id, call, modelRequest: context.model }),
|
|
7967
|
+
});
|
|
7968
|
+
if (!report.model.parsed)
|
|
7969
|
+
throw new Error(report.model.why ?? '抽取结果无法归一');
|
|
7970
|
+
const db = await this.storeHandle();
|
|
7971
|
+
const current = db?.source(source.sourceId);
|
|
7972
|
+
if (this.closed)
|
|
7973
|
+
throw new Error('知识库已关闭,本次归一未提交');
|
|
7974
|
+
if (!db || !current || current.version !== source.version || contentHashOf(current.record.content) !== source.sourceHash)
|
|
7975
|
+
throw new Error('来源在归一期间已变化,请重新抽取最新版本');
|
|
7976
|
+
const sql = db.connection();
|
|
7977
|
+
sql.exec('SAVEPOINT pipeline_normalize');
|
|
7978
|
+
try {
|
|
7979
|
+
draft.commit();
|
|
7980
|
+
// Preserve evidence history while replacing this source's active contribution.
|
|
7981
|
+
sql.prepare('UPDATE knowledge_sources SET withdrawn=1 WHERE document_id=?').run(source.sourceId);
|
|
7982
|
+
const knowledge = await this.ingestTrialKnowledge({ runId: checkpoint.id, knowledge: report.knowledge, objects: report.objects, actions: report.actions, scenes: report.scenes, sourceVersions: { [source.sourceId]: source.version } });
|
|
7983
|
+
const events = await this.ingestTrialEvents({ sourceId: source.sourceId, runId: checkpoint.id, events: report.events, objects: report.objects, actions: report.actions, scenes: report.scenes });
|
|
7984
|
+
if (knowledge.skipped || knowledge.failures || events.skipped)
|
|
7985
|
+
throw new Error(`出处核验或写入失败:${knowledge.skipped} 条 Knowledge 引文未通过核验、${knowledge.failures} 条写入失败、${events.skipped} 个 Event;本次归一没有提交`);
|
|
7986
|
+
const unresolvedKnowledgeIds = report.knowledge.filter(item => [['object', report.objects], ['action', report.actions], ['scene', report.scenes]].some(([kind, axes]) => {
|
|
7987
|
+
const name = item[kind];
|
|
7988
|
+
const axis = name ? axes.find(axis => axis.name === name) : undefined;
|
|
7989
|
+
return name !== null && (!axis || axis.outcome === 'ambiguous' || axis.outcome === 'no-registry');
|
|
7990
|
+
})).map(item => knowledge.ids[item.id]).filter(Boolean);
|
|
7991
|
+
const normalized = { report, knowledgeIds: knowledge.ids, eventIds: events.ids, unresolvedKnowledgeIds, fingerprint: '' };
|
|
7992
|
+
// Store the post-resolution candidate snapshot so learning its own entities does not invalidate this run.
|
|
7993
|
+
normalized.fingerprint = await this.normalizationFingerprint({ ...checkpoint, normalized }, context);
|
|
7994
|
+
/**
|
|
7995
|
+
* 归一学到的实体本来就在这份文档的提示词里,所以它会把「抽取指纹」改掉。
|
|
7996
|
+
* 把已存的那一份刷新成**学完之后**的值,否则这份文档下次抽取永远复用不上
|
|
7997
|
+
* (与上面那行存「补判之后」的快照同一条规则)。
|
|
7998
|
+
*/
|
|
7999
|
+
store.refreshFingerprint(checkpoint.id, await this.pipelineExtractionFingerprint(source, context));
|
|
8000
|
+
store.saveNormalization(checkpoint, normalized);
|
|
8001
|
+
sql.exec('RELEASE pipeline_normalize');
|
|
8002
|
+
// Updating Event references is a cheap relation refresh. It does not reorganize or index card content.
|
|
8003
|
+
const eventReferences = await this.syncEventReferences();
|
|
8004
|
+
const warnings = [...knowledge.warnings, ...events.warnings, ...eventReferences.warnings, ...(unresolvedKnowledgeIds.length ? [`${unresolvedKnowledgeIds.length} 条 Knowledge 的坐标仍有歧义,保留事实与出处,暂不物化。`] : [])];
|
|
8005
|
+
/**
|
|
8006
|
+
* 轨迹在**提交之后**才落(`RELEASE` 之后):写在 savepoint 里面的话,回滚了还会留下
|
|
8007
|
+
* 一条「归一成功了」的记录。
|
|
8008
|
+
*
|
|
8009
|
+
* 落不下来只记一条警告,不把这一段判失败 —— 事实已经写进知识库了,这时候抛错会让人
|
|
8010
|
+
* 以为没生效。抽取那一段是反过来的(它的产物只有报告,存不下就等于什么都没做)。
|
|
8011
|
+
*/
|
|
8012
|
+
try {
|
|
8013
|
+
await this.recordNormalizeTrajectory({ source, checkpoint, report, knowledge, events, unresolvedKnowledgeIds, fingerprint: normalized.fingerprint, warnings, context });
|
|
8014
|
+
}
|
|
8015
|
+
catch (cause) {
|
|
8016
|
+
warnings.push(`归一轨迹没有存下来:${cause instanceof Error ? cause.message : String(cause)}`);
|
|
8017
|
+
}
|
|
8018
|
+
return { reused: false, warnings };
|
|
8019
|
+
}
|
|
8020
|
+
catch (cause) {
|
|
8021
|
+
sql.exec('ROLLBACK TO pipeline_normalize; RELEASE pipeline_normalize');
|
|
8022
|
+
throw cause;
|
|
8023
|
+
}
|
|
8024
|
+
}
|
|
8025
|
+
finally {
|
|
8026
|
+
draft.close();
|
|
8027
|
+
}
|
|
8028
|
+
}
|
|
8029
|
+
async activePipelineKnowledge() {
|
|
8030
|
+
const knowledge = await this.knowledgeStore();
|
|
8031
|
+
const db = await this.storeHandle();
|
|
8032
|
+
if (!knowledge || !db)
|
|
8033
|
+
return [];
|
|
8034
|
+
/**
|
|
8035
|
+
* **来源正文预读一次**,别在每一行上重读整份文档。
|
|
8036
|
+
*
|
|
8037
|
+
* `assessSource` 要对每条 knowledge-source 判定「引文是否还在当前版本里」,而那需要
|
|
8038
|
+
* 该文档的正文。原来的写法是 `db.source(id)`:每行一次全表 join + 整份正文 + 全文
|
|
8039
|
+
* flatten + 子串匹配 —— 5000 条知识 × 每份 50 KB 就是几百 MB 的重复工作,而这只是
|
|
8040
|
+
* 一次调用。正文在这里读一次(一条查询),下面按 documentId 查表。
|
|
8041
|
+
*/
|
|
8042
|
+
const bodies = new Map();
|
|
8043
|
+
for (const row of db.sources())
|
|
8044
|
+
bodies.set(row.record.sourceId, { version: row.version, content: row.record.content });
|
|
8045
|
+
const rows = [];
|
|
8046
|
+
for (let offset = 0;; offset += 500) {
|
|
8047
|
+
const page = await knowledge.list({ excludeWithdrawn: true, limit: 500, offset });
|
|
8048
|
+
rows.push(...page.knowledge.filter(row => row.status !== 'deprecated').map(row => assessKnowledgeEvidence(row, db, bodies)).map(row => ({ ...row, sources: row.sources.filter(source => !source.withdrawn && source.evidenceStatus !== 'missing') })).filter(row => row.sources.length > 0));
|
|
8049
|
+
if (offset + page.knowledge.length >= page.total)
|
|
8050
|
+
break;
|
|
8051
|
+
}
|
|
8052
|
+
return rows;
|
|
8053
|
+
}
|
|
8054
|
+
/**
|
|
8055
|
+
* 这一组现在能不能物化(有没有来源变了、新抽取待归一、或坐标还有歧义)。
|
|
8056
|
+
*
|
|
8057
|
+
* 抽成一处是因为它有**两个**调用点:阶段开始时的全量快照,与单组写回之前的复查。
|
|
8058
|
+
* 各写一份的话,「什么算变化」迟早会有两套说法。
|
|
8059
|
+
*/
|
|
8060
|
+
needsNormalizationForGroup(store, knowledge, unresolved, db) {
|
|
8061
|
+
return knowledge.some(row => unresolved.ids.has(row.id)) || [...new Set(knowledge.flatMap(row => row.sources.map(source => source.documentId)))].filter(Boolean).some(id => {
|
|
8062
|
+
const source = db?.source(id);
|
|
8063
|
+
const head = store.head(id);
|
|
8064
|
+
return head.normalized ? head.normalized.sourceVersion !== source?.version || head.extracted?.id !== head.normalized.id : knowledge.some(row => row.sources.some(evidence => evidence.documentId === id && evidence.toVersion !== undefined && evidence.toVersion !== source?.version));
|
|
8065
|
+
});
|
|
8066
|
+
}
|
|
8067
|
+
/** 相关推荐的整库计划。它是增强,所以算不出来时给空表而不是把编译卡住。 */
|
|
8068
|
+
async relatedPlan() {
|
|
8069
|
+
const plan = new Map();
|
|
8070
|
+
try {
|
|
8071
|
+
const cards = await this.loadCards();
|
|
8072
|
+
const relations = Object.values(await this.readExtractedRelations()).flat();
|
|
8073
|
+
for (const [key, entries] of planRelated(cards, relations, await this.projectedRelated(cards))) {
|
|
8074
|
+
if (entries.length > 0)
|
|
8075
|
+
plan.set(key, entries.map(entry => ({ title: entry.title, entityType: entry.entityType, reason: entry.reason })));
|
|
8076
|
+
}
|
|
8077
|
+
}
|
|
8078
|
+
catch { /* 增强项:算不出来就不给清单 */ }
|
|
8079
|
+
return plan;
|
|
8080
|
+
}
|
|
8081
|
+
/**
|
|
8082
|
+
* 一个分组的**事实指纹**:只有它变了才需要重新成卡。
|
|
8083
|
+
*
|
|
8084
|
+
* 抽成一处是因为它有**两个**调用点:阶段开始时的全库快照,与单组写回之前的复查。
|
|
8085
|
+
* 两处必须用同一个表达式,否则「什么算变化」就有两套说法。
|
|
8086
|
+
*/
|
|
8087
|
+
pipelineGroupFingerprint(knowledge, context) {
|
|
8088
|
+
return contentHashOf(JSON.stringify({ knowledge: knowledge.map(row => ({ id: row.id, text: row.text, status: row.status, sources: row.sources.filter(source => !source.withdrawn) })).sort((a, b) => a.id.localeCompare(b.id)), profile: context.profile, organize: context.organize, model: context.organize ? { provider: context.model.provider, model: context.model.model, reasoningEffort: context.model.reasoningEffort } : undefined }));
|
|
8089
|
+
}
|
|
8090
|
+
/**
|
|
8091
|
+
* **只读这一组**的事实指纹,写回之前复查用。
|
|
8092
|
+
*
|
|
8093
|
+
* 快照在阶段内是稳定的,但那有个前提:写入者都走同一条串行队列。别的进程、或同一个库的
|
|
8094
|
+
* 另一条路径(强制重归一复用同一个检查点 id、诊断载入示例、删除流程)不受它约束 ——
|
|
8095
|
+
* 那时「分组的知识行变了,而 head 指针没变」是可能的,只看 head 会漏掉,于是拿陈旧的
|
|
8096
|
+
* 行去写卡片(丢更新)。这里按坐标只读这一组,代价与该组的知识条数成正比,不扫全库。
|
|
8097
|
+
*/
|
|
8098
|
+
async freshGroupFingerprint(coordinate, context, db) {
|
|
8099
|
+
const store = await this.knowledgeStore();
|
|
8100
|
+
if (!store)
|
|
8101
|
+
return '';
|
|
8102
|
+
const raw = [];
|
|
8103
|
+
for (let offset = 0;; offset += 500) {
|
|
8104
|
+
const page = await store.list({ objectId: coordinate.objectId, actionId: coordinate.actionId, sceneId: coordinate.sceneId, excludeWithdrawn: true, limit: 500, offset });
|
|
8105
|
+
raw.push(...page.knowledge);
|
|
8106
|
+
if (offset + page.knowledge.length >= page.total)
|
|
8107
|
+
break;
|
|
8108
|
+
}
|
|
8109
|
+
// 只读这一组**用到**的那些来源正文,而不是全库的。
|
|
8110
|
+
const documentIds = [...new Set(raw.flatMap(row => row.sources.map(source => source.documentId)).filter(Boolean))];
|
|
8111
|
+
const bodies = new Map();
|
|
8112
|
+
for (const id of documentIds) {
|
|
8113
|
+
const source = db.source(id);
|
|
8114
|
+
if (source)
|
|
8115
|
+
bodies.set(id, { version: source.version, content: source.record.content });
|
|
8116
|
+
}
|
|
8117
|
+
const rows = raw
|
|
8118
|
+
.filter(row => row.status !== 'deprecated')
|
|
8119
|
+
.map(row => assessKnowledgeEvidence(row, db, bodies))
|
|
8120
|
+
.map(row => ({ ...row, sources: row.sources.filter(source => !source.withdrawn && source.evidenceStatus !== 'missing') }))
|
|
8121
|
+
.filter(row => row.sources.length > 0);
|
|
8122
|
+
return this.pipelineGroupFingerprint(rows, context);
|
|
8123
|
+
}
|
|
8124
|
+
/**
|
|
8125
|
+
* 物化阶段的**一次**全库快照。
|
|
8126
|
+
*
|
|
8127
|
+
* 逐组现算会变成 O(组数 × 语料):每个分组要扫 4 遍全库(三次分组、一次知识),
|
|
8128
|
+
* 再加上每组一次图片分发与一次相关推荐 —— 几百张卡时单线程跑不完。这里全部算一次。
|
|
8129
|
+
*/
|
|
8130
|
+
async materializationSnapshot(store, context, options = {}) {
|
|
8131
|
+
/**
|
|
8132
|
+
* `plans: false` 给**只读状态**的调用方(`pipelineStatus`):它只要 groups / blocked /
|
|
8133
|
+
* 各类计数,而图片分发与相关推荐是整库计算(全部来源正文 + 全部卡片 + 关系投影),
|
|
8134
|
+
* 让它跟着 2.5 秒一次的状态轮询跑,等于把整个语料每 2.5 秒重算一遍。
|
|
8135
|
+
*/
|
|
8136
|
+
const plans = options.plans !== false;
|
|
8137
|
+
const unresolved = store.unresolvedKnowledge();
|
|
8138
|
+
const grouped = new Map();
|
|
8139
|
+
let blockedAmbiguous = 0;
|
|
8140
|
+
let blockedAwaitingNormalize = 0;
|
|
8141
|
+
const groupFor = (coordinate) => {
|
|
8142
|
+
const key = coordinateKeyOf(coordinate);
|
|
8143
|
+
if (!grouped.has(key))
|
|
8144
|
+
grouped.set(key, { coordinate, knowledge: [] });
|
|
8145
|
+
return grouped.get(key);
|
|
8146
|
+
};
|
|
8147
|
+
for (const row of await this.activePipelineKnowledge()) {
|
|
8148
|
+
const coordinate = { objectId: row.objectId, actionId: row.actionId, sceneId: row.sceneId };
|
|
8149
|
+
if (coordinateIsEmpty(coordinate)) {
|
|
8150
|
+
blockedAmbiguous += 1;
|
|
8151
|
+
continue;
|
|
8152
|
+
}
|
|
8153
|
+
groupFor(coordinate).knowledge.push(row);
|
|
8154
|
+
}
|
|
8155
|
+
const db = await this.storeHandle();
|
|
8156
|
+
const events = new Map();
|
|
8157
|
+
for (const event of db?.events() ?? []) {
|
|
8158
|
+
const coordinate = event.coordinateIds;
|
|
8159
|
+
if (!coordinate.objectId || !coordinate.actionId || !coordinate.sceneId)
|
|
8160
|
+
continue;
|
|
8161
|
+
const key = coordinateKeyOf(coordinate);
|
|
8162
|
+
const bucket = events.get(key) ?? [];
|
|
8163
|
+
bucket.push(event);
|
|
8164
|
+
events.set(key, bucket);
|
|
8165
|
+
}
|
|
8166
|
+
const cards = await this.loadCards();
|
|
8167
|
+
const byCoordinate = new Map(cards.filter(card => card.coordinate).map(card => [coordinateKeyOf(card.coordinate), card]));
|
|
8168
|
+
const groups = [];
|
|
8169
|
+
const knowledge = new Map();
|
|
8170
|
+
for (const [key, value] of grouped) {
|
|
8171
|
+
const names = await this.coordinateNames(value.coordinate);
|
|
8172
|
+
const existing = byCoordinate.get(key);
|
|
8173
|
+
// Event 只作为卡片引用;它的增删和正文变化不得触发 Knowledge 重新组织。
|
|
8174
|
+
const fingerprint = this.pipelineGroupFingerprint(value.knowledge, context);
|
|
8175
|
+
const receipt = store.materialization(key);
|
|
8176
|
+
const current = existing && receipt?.fingerprint === fingerprint && receipt.cardId === existing.entityId;
|
|
8177
|
+
const contributingSources = [...new Set(value.knowledge.flatMap(row => row.sources.map(source => source.documentId)))].filter(Boolean).sort();
|
|
8178
|
+
const needsNormalization = this.needsNormalizationForGroup(store, value.knowledge, unresolved, db);
|
|
8179
|
+
if (needsNormalization)
|
|
8180
|
+
blockedAwaitingNormalize += value.knowledge.length;
|
|
8181
|
+
knowledge.set(key, value.knowledge);
|
|
8182
|
+
groups.push({ key, title: existing?.title ?? cardNameFor(names.object || null, names.action || null, names.scene || null), coordinate: value.coordinate, knowledgeCount: value.knowledge.length, eventCount: events.get(key)?.length ?? 0, sourceIds: contributingSources, ...(existing ? { cardId: existing.entityId } : {}), state: needsNormalization ? 'blocked' : current ? 'current' : existing ? 'changed' : 'new', ...(needsNormalization ? { reason: '分组中的来源已变化、新抽取待归一或坐标有歧义,请先完成归一' } : {}), fingerprint });
|
|
8183
|
+
}
|
|
8184
|
+
for (const card of cards) {
|
|
8185
|
+
if (!card.coordinate || grouped.has(coordinateKeyOf(card.coordinate)) || !store.materialization(coordinateKeyOf(card.coordinate)) || card.status === 'deprecated')
|
|
8186
|
+
continue;
|
|
8187
|
+
groups.push({ key: coordinateKeyOf(card.coordinate), title: card.title, coordinate: card.coordinate, knowledgeCount: 0, eventCount: 0, sourceIds: card.sourceRefs, cardId: card.entityId, state: card.sourceRefs.some(id => unresolved.sources.has(id)) ? 'blocked' : 'withdrawn', ...(card.sourceRefs.some(id => unresolved.sources.has(id)) ? { reason: '来源中的坐标仍有歧义,请先完成归一' } : {}), fingerprint: 'withdrawn' });
|
|
8188
|
+
}
|
|
8189
|
+
const sources = plans ? (await this.loadSources()).map(source => ({ sourceId: source.sourceId, content: source.content })) : [];
|
|
8190
|
+
const imagePlan = plans
|
|
8191
|
+
? planSourceImages({
|
|
8192
|
+
cards: groups.filter(item => item.state !== 'withdrawn').map(item => ({ key: item.key, content: (knowledge.get(item.key) ?? []).map(row => row.text).join('\n\n'), sourceRefs: item.sourceIds })),
|
|
8193
|
+
sources,
|
|
8194
|
+
})
|
|
8195
|
+
: { byCard: new Map() };
|
|
8196
|
+
/**
|
|
8197
|
+
* 相关推荐是按**卡片身份**(`type:entityId`)算的,而分组键是坐标 —— 两套键永远对不上,
|
|
8198
|
+
* 所以必须在这里把身份映射到分组键。少了这一步,`organizeCard` 拿不到「可引用的邻居」,
|
|
8199
|
+
* 而模型一旦照样写出 `[[名字]]`,`checkOrganize` 会判 `ORGANIZE_UNKNOWN_REFERENCE`、
|
|
8200
|
+
* 整组物化失败(卡片永远写不出来)。
|
|
8201
|
+
*
|
|
8202
|
+
* 只对**已经有卡片**的分组建映射:新建的卡片在相关计划里本来就不存在(与旧行为一致)。
|
|
8203
|
+
* 键用的是卡片自己的类型,与 `relatedForCard(existing)` 一致。
|
|
8204
|
+
*/
|
|
8205
|
+
const relatedByKey = new Map();
|
|
8206
|
+
if (plans) {
|
|
8207
|
+
const plan = await this.relatedPlan();
|
|
8208
|
+
for (const [key, entries] of relatedByGroupKey(groups, groupKey => {
|
|
8209
|
+
const card = byCoordinate.get(groupKey);
|
|
8210
|
+
return card === undefined ? undefined : cardKey(card);
|
|
8211
|
+
}, plan)) {
|
|
8212
|
+
relatedByKey.set(key, entries);
|
|
8213
|
+
}
|
|
8214
|
+
}
|
|
8215
|
+
return {
|
|
8216
|
+
groups: groups.sort((a, b) => a.title.localeCompare(b.title)),
|
|
8217
|
+
blocked: blockedAmbiguous + blockedAwaitingNormalize,
|
|
8218
|
+
blockedAmbiguous, blockedAwaitingNormalize,
|
|
8219
|
+
knowledge, events, sources, images: imagePlan.byCard, related: relatedByKey,
|
|
8220
|
+
};
|
|
8221
|
+
}
|
|
8222
|
+
async materializePipelineGroup(store, group, context, output, snapshot) {
|
|
8223
|
+
const db = await this.storeHandle();
|
|
8224
|
+
if (!db)
|
|
8225
|
+
throw new Error('知识库不可用');
|
|
8226
|
+
/**
|
|
8227
|
+
* 复查读**同一份快照**:语料在物化阶段内不会变(归一排在编译队列的后面),
|
|
8228
|
+
* 所以分组指纹是稳定的;这里只对目标组重算「是否等待归一」。
|
|
8229
|
+
* 会变的是人工裁决 —— 那一条在写回之前单独重读(见下面的 `statusAtWrite`)。
|
|
8230
|
+
*/
|
|
8231
|
+
const known = snapshot.groups.find(item => item.key === group.key);
|
|
8232
|
+
if (!known || known.fingerprint !== group.fingerprint)
|
|
8233
|
+
throw new Error('卡片分组在处理期间已变化,请刷新并重试');
|
|
8234
|
+
const knowledge = snapshot.knowledge.get(group.key) ?? [];
|
|
8235
|
+
if (this.needsNormalizationForGroup(store, knowledge, store.unresolvedKnowledge(), db))
|
|
8236
|
+
throw new Error(known.reason ?? '分组中的来源已变化、新抽取待归一或坐标有歧义,请先完成归一');
|
|
8237
|
+
const storedExisting = group.cardId ? await this.readCard(group.cardId) : undefined;
|
|
8238
|
+
const existing = storedExisting
|
|
8239
|
+
? { ...storedExisting, content: stripEventReferenceSection(storedExisting.content) }
|
|
8240
|
+
: undefined;
|
|
8241
|
+
if (group.state === 'withdrawn') {
|
|
8242
|
+
if (!existing)
|
|
8243
|
+
return;
|
|
8244
|
+
/**
|
|
8245
|
+
* 撤回也要过同一道人工裁决的闸门(0070):这一条原来在它**后面**才判,
|
|
8246
|
+
* 于是「有人在处理期间给这张卡做了 review / 停用」会被 `{...existing, status:'deprecated'}`
|
|
8247
|
+
* 静默覆盖。
|
|
8248
|
+
*/
|
|
8249
|
+
const statusBeforeDeprecate = (await this.readCard(existing.entityId))?.status;
|
|
8250
|
+
if (statusBeforeDeprecate === undefined)
|
|
8251
|
+
throw new Error('卡片在处理期间已被删除,本次物化未提交,请刷新后重试');
|
|
8252
|
+
if (statusBeforeDeprecate !== existing.status)
|
|
8253
|
+
throw new Error(`卡片在处理期间被人工改动(状态 ${existing.status} → ${statusBeforeDeprecate}),本次物化未提交,请重试`);
|
|
8254
|
+
const written = await this.writeCard({ ...existing, status: 'deprecated', origin: 'compile' });
|
|
8255
|
+
if (!written.ok)
|
|
8256
|
+
throw new Error(written.error.message);
|
|
8257
|
+
store.materialized(group.key, group.fingerprint, existing.entityId);
|
|
8258
|
+
// 停用也是一次物化:卡片从可召回里退出,这一步发生过,就该留一条轨迹。
|
|
8259
|
+
await this.recordMaterializeTrajectory({
|
|
8260
|
+
group, action: 'withdrawn', knowledge: [],
|
|
8261
|
+
card: { id: existing.entityId, title: existing.title, status: 'deprecated', characters: existing.content.length, events: 0, images: 0 },
|
|
8262
|
+
organizeUsed: false, related: [], context,
|
|
8263
|
+
}).catch(() => undefined);
|
|
8264
|
+
return;
|
|
8265
|
+
}
|
|
8266
|
+
const events = snapshot.events.get(group.key) ?? [];
|
|
8267
|
+
const profile = context.profile;
|
|
8268
|
+
const entityType = (profile?.id ?? 'concept');
|
|
8269
|
+
let title = group.title;
|
|
8270
|
+
let content = `## 知识\n\n${knowledge.map(row => `- ${row.text}`).join('\n')}`;
|
|
8271
|
+
let summary = knowledge[0]?.text ?? '';
|
|
8272
|
+
let placements = knowledge.map((row, index) => ({ knowledgeId: row.id, section: '知识', ordinal: index + 1, origin: 'compiler' }));
|
|
8273
|
+
let metadata = { ...(existing?.metadata ?? {}), pipeline_materialization: true };
|
|
8274
|
+
const entityId = existing?.entityId ?? `coord-${hash(group.key)}`;
|
|
8275
|
+
const images = snapshot.images.get(group.key) ?? [];
|
|
8276
|
+
const related = existing && snapshot.related.has(group.key) ? { related: snapshot.related.get(group.key) } : {};
|
|
8277
|
+
metadata.citations = citationsFrom({ placements, knowledge: new Map(knowledge.map(row => [row.id, { source: row.sources[0]?.documentId, heading: row.sources[0]?.headingPath.at(-1) }])) });
|
|
8278
|
+
if (context.organize && knowledge.length > 0) {
|
|
8279
|
+
if (!context.model.call)
|
|
8280
|
+
throw new Error(`卡片组织模型不可用:${context.model.why ?? '请选择模型'}`);
|
|
8281
|
+
const result = await this.organizeCard({ knowledge, images, ...related, cardTitle: title, coordinate: group.coordinate, profileId: entityType, force: true, suggestedSections: profile?.sections.map(section => section.label), requiredSections: profile?.sections.filter(section => section.required).map(section => section.label), existing, call: request => context.model.call({ ...request, onDelta: output }) });
|
|
8282
|
+
if (result.kind !== 'applied')
|
|
8283
|
+
throw new Error(result.kind === 'reused' ? '组织结果已变化,请刷新后重试' : result.warnings.join(';'));
|
|
8284
|
+
title = result.title;
|
|
8285
|
+
content = result.content;
|
|
8286
|
+
summary = result.summary;
|
|
8287
|
+
placements = result.placements;
|
|
8288
|
+
metadata = { ...metadata, layout: result.layout, citations: result.citations };
|
|
8289
|
+
}
|
|
8290
|
+
for (const image of images)
|
|
8291
|
+
content = insertImage(content, image);
|
|
8292
|
+
const eventIds = events.map(event => event.id).sort();
|
|
8293
|
+
content = stripEventReferenceSection(content);
|
|
8294
|
+
metadata.event_refs = JSON.stringify(eventIds);
|
|
8295
|
+
/**
|
|
8296
|
+
* 冻结的是这一组的事实,而且要在**写之前**再核一次(0070)。
|
|
8297
|
+
*
|
|
8298
|
+
* 两条都要:斜杠前那条查的是「来源是不是等归一」(head 指针级别),它看不见
|
|
8299
|
+
* 「只有知识行变了」;这条按坐标重算整组指纹,能把那种变化也挡住。
|
|
8300
|
+
*/
|
|
8301
|
+
if (this.needsNormalizationForGroup(store, knowledge, store.unresolvedKnowledge(), db))
|
|
8302
|
+
throw new Error('卡片组织期间事实发生变化,结果未提交,请重试');
|
|
8303
|
+
if (await this.freshGroupFingerprint(group.coordinate, context, db) !== group.fingerprint)
|
|
8304
|
+
throw new Error('卡片分组在处理期间事实发生了变化,结果未提交,请重试');
|
|
8305
|
+
if (this.closed)
|
|
8306
|
+
throw new Error('知识库已关闭,本次物化未提交');
|
|
8307
|
+
/**
|
|
8308
|
+
* 状态必须**在写之前**重读,并且和读到的旧值比对。
|
|
8309
|
+
*
|
|
8310
|
+
* `existing` 是在模型调用(`organizeCard`,秒到几十秒)**之前**读的,而 `status` 是
|
|
8311
|
+
* 显式传下去的 —— `writeCard` 的「不传 = 保留库里已有的」保护因此不生效。用户在
|
|
8312
|
+
* 这段时间里点了「停用」,旧值会把人工裁决静默覆盖掉,卡片又回到索引里,
|
|
8313
|
+
* 而且没有任何地方报错。所以:值变了就**拒绝提交**,让人重试,而不是猜哪个对。
|
|
8314
|
+
*/
|
|
8315
|
+
const statusAtWrite = entityId ? (await this.readCard(entityId))?.status : undefined;
|
|
8316
|
+
// 「卡片没了」与「状态被改了」是两件事:前者要重建成新卡,后者要人重试。分开说。
|
|
8317
|
+
if (existing !== undefined && statusAtWrite === undefined)
|
|
8318
|
+
throw new Error('卡片在处理期间已被删除,本次物化未提交,请刷新后重试');
|
|
8319
|
+
if (statusAtWrite !== existing?.status) {
|
|
8320
|
+
throw new Error(`卡片在处理期间被人工改动(状态 ${existing?.status ?? '未知'} → ${statusAtWrite ?? '未知'}),本次物化未提交,请重试`);
|
|
8321
|
+
}
|
|
8322
|
+
const written = await this.writeCard({ entityType, entityId, title, content, summary, sourceRefs: group.sourceIds, status: statusAtWrite === 'deprecated' ? 'unreviewed' : statusAtWrite ?? 'unreviewed', coordinate: group.coordinate, placements, metadata, origin: 'compile' });
|
|
8323
|
+
if (!written.ok)
|
|
8324
|
+
throw new Error(written.error.message);
|
|
8325
|
+
db.setCardEvents(entityId, eventIds);
|
|
8326
|
+
store.materialized(group.key, group.fingerprint, entityId);
|
|
8327
|
+
/**
|
|
8328
|
+
* 卡片已经写好了,轨迹落不下来不该把它判失败(同归一那条理由)。所以这里吞掉异常 ——
|
|
8329
|
+
* 但**不吞在别处**:写卡失败仍然抛。
|
|
8330
|
+
*/
|
|
8331
|
+
await this.recordMaterializeTrajectory({
|
|
8332
|
+
group,
|
|
8333
|
+
action: existing === undefined ? 'created' : 'updated',
|
|
8334
|
+
knowledge,
|
|
8335
|
+
placements,
|
|
8336
|
+
organizeUsed: Boolean(context.organize && knowledge.length > 0),
|
|
8337
|
+
...(layoutOf(metadata) ? { layout: layoutOf(metadata) } : {}),
|
|
8338
|
+
related: snapshot.related.get(group.key) ?? [],
|
|
8339
|
+
card: {
|
|
8340
|
+
id: entityId,
|
|
8341
|
+
title,
|
|
8342
|
+
status: statusAtWrite === 'deprecated' ? 'unreviewed' : statusAtWrite ?? 'unreviewed',
|
|
8343
|
+
characters: content.length,
|
|
8344
|
+
events: eventIds.length,
|
|
8345
|
+
images: images.length,
|
|
8346
|
+
},
|
|
8347
|
+
context,
|
|
8348
|
+
}).catch(() => undefined);
|
|
8349
|
+
}
|
|
8350
|
+
async pipelineStatus() {
|
|
8351
|
+
const pipeline = await this.pipeline();
|
|
8352
|
+
const sources = await this.pipelineSources();
|
|
8353
|
+
const context = { model: this.modelSeamNow(), note: this.extractionNote(), profile: this.organizeProfile(), organize: this.config.cardOrganize, format: 'table' };
|
|
8354
|
+
const rows = await Promise.all(sources.map(async (source) => {
|
|
8355
|
+
const head = pipeline.store.head(source.sourceId);
|
|
8356
|
+
const extracted = head.extracted?.sourceVersion === source.version && head.extracted.sourceHash === source.sourceHash;
|
|
8357
|
+
const normalized = extracted && head.normalized?.id === head.extracted?.id && head.normalized?.normalized?.fingerprint === await this.normalizationFingerprint(head.normalized, context);
|
|
8358
|
+
return { id: source.sourceId, title: source.title, version: source.version, extractionId: head.extracted?.id, normalizedId: head.normalized?.id, state: normalized ? 'normalized' : extracted ? 'extracted' : head.extracted ? 'outdated' : 'pending' };
|
|
8359
|
+
}));
|
|
8360
|
+
// 状态查询不要图片分发与相关推荐:它每 2.5 秒被轮询一次,而那两项是整库计算。
|
|
8361
|
+
const { groups, blocked } = await this.materializationSnapshot(pipeline.store, context, { plans: false });
|
|
8362
|
+
const db = await this.derivedHandle();
|
|
8363
|
+
const manifest = db?.artifact('build-manifest.json');
|
|
8364
|
+
const hashes = manifest ? JSON.parse(manifest).cardHashes ?? {} : {};
|
|
8365
|
+
const cards = await this.loadCards();
|
|
8366
|
+
const indexPending = cards.some(card => hashes[`${card.entityType}:${card.entityId}`] !== this.cardHashOf(card)) || Object.keys(hashes).length !== cards.length;
|
|
8367
|
+
return { sources: rows, groups, blocked, pendingNew: groups.filter(group => group.state === 'new').length, pendingUpdated: groups.filter(group => group.state === 'changed').length, pendingWithdrawn: groups.filter(group => group.state === 'withdrawn').length, indexPending };
|
|
8368
|
+
}
|
|
8369
|
+
/** 轨迹列表(新→旧,不含报告本体)。`stage` 省略 = 三段一起。 */
|
|
7654
8370
|
async listExtractionRuns(query = {}) {
|
|
7655
8371
|
const store = await this.extractionRuns();
|
|
7656
8372
|
if (store === null)
|
|
@@ -7853,10 +8569,12 @@ export class KnowledgeRuntime {
|
|
|
7853
8569
|
* 它」—— 那两件事分别在 `enqueueCompile()` 与 `rebuild()` / `startCompile()` 里。
|
|
7854
8570
|
*/
|
|
7855
8571
|
async compileOnce(runId, only = 'all', lineage, options = {}) {
|
|
7856
|
-
|
|
8572
|
+
if (only === 'all' || only === 'embeddings')
|
|
8573
|
+
await this.environmentGuard?.('embedding');
|
|
7857
8574
|
this.publication = { artifacts: new Map() };
|
|
7858
8575
|
const configAtStart = JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() });
|
|
7859
8576
|
try {
|
|
8577
|
+
await options.checkpoint?.();
|
|
7860
8578
|
await this.loadCompiledState();
|
|
7861
8579
|
const result = await this.prepareCompile(runId, only, lineage, options);
|
|
7862
8580
|
await this.checkpointCompileTask(runId);
|
|
@@ -7889,6 +8607,7 @@ export class KnowledgeRuntime {
|
|
|
7889
8607
|
await this.checkpointCompileTask(runId);
|
|
7890
8608
|
if (JSON.stringify({ config: this.config, fingerprint: this.embeddingFingerprint() }) !== configAtStart)
|
|
7891
8609
|
throw new Error('COMPILE_CONFIG_CHANGED: 编译期间设置已变化,请重新编译');
|
|
8610
|
+
await options.checkpoint?.();
|
|
7892
8611
|
db.publish(this.publication);
|
|
7893
8612
|
this.vectorIndex = null;
|
|
7894
8613
|
}
|
|
@@ -8042,7 +8761,7 @@ export class KnowledgeRuntime {
|
|
|
8042
8761
|
tally.knowledge > 0
|
|
8043
8762
|
? `抽出知识 ${tally.knowledge} 条`
|
|
8044
8763
|
: '未用模型:正文按 ## 小节切',
|
|
8045
|
-
tally.skipped > 0 ?
|
|
8764
|
+
tally.skipped > 0 ? `复用未变来源 ${tally.skipped} 份` : options.forceExtract ? '本次为手动强制重抽' : '正在检查新增或变化的来源',
|
|
8046
8765
|
];
|
|
8047
8766
|
if (done < total) {
|
|
8048
8767
|
setStep('knowledge', 'running', knowledgeFacts);
|
|
@@ -8717,7 +9436,7 @@ export class KnowledgeRuntime {
|
|
|
8717
9436
|
if (!(captured?.ready ?? derived.hasFts()))
|
|
8718
9437
|
return { ranking: [], rows: [], diagnostic: { route: 'fts', status: 'degraded', reason: '关键词索引还没建(在 derived.db 里),先编译一次' } };
|
|
8719
9438
|
try {
|
|
8720
|
-
const found = (captured?.hits ?? derived.searchFts(query, pool)).filter(hit => eligibleIds.has(hit.unitId));
|
|
9439
|
+
const found = (captured?.hits ?? derived.searchFts(query, pool, eligibleIds)).filter(hit => eligibleIds.has(hit.unitId));
|
|
8721
9440
|
/**
|
|
8722
9441
|
* 覆盖率门槛(见 `MIN_QUERY_COVERAGE`):只蹭到一两个高频单字的命中在这里淘汰,
|
|
8723
9442
|
* 于是「整批都弱」也能弃答 —— 这是相对阈值做不到的那一半。
|
|
@@ -9043,7 +9762,7 @@ export class KnowledgeRuntime {
|
|
|
9043
9762
|
* 一次召回十来张卡,而库里可能有上千张。
|
|
9044
9763
|
*/
|
|
9045
9764
|
try {
|
|
9046
|
-
placements = snapshot
|
|
9765
|
+
placements = snapshot ? new Map(snapshot.placements) : store.placementsOf([...new Set(hits.map(hit => hit.entityId))]);
|
|
9047
9766
|
}
|
|
9048
9767
|
catch {
|
|
9049
9768
|
return;
|
|
@@ -9065,7 +9784,7 @@ export class KnowledgeRuntime {
|
|
|
9065
9784
|
const wanted = new Set(perHit.flat().map(row => row.knowledgeId));
|
|
9066
9785
|
if (wanted.size === 0)
|
|
9067
9786
|
return;
|
|
9068
|
-
const rows = snapshot
|
|
9787
|
+
const rows = snapshot
|
|
9069
9788
|
? new Map(snapshot.knowledge.filter(row => wanted.has(row.id) && knowledge !== null)
|
|
9070
9789
|
.flatMap(row => {
|
|
9071
9790
|
// Deleted facts never reappear through a historical snapshot.
|
|
@@ -9159,14 +9878,17 @@ export class KnowledgeRuntime {
|
|
|
9159
9878
|
if (!Number.isInteger(requestedTopN) || requestedTopN < 1 || (options.contextTopN !== undefined && requestedTopN > recallTopK))
|
|
9160
9879
|
return failure(empty(), 'CONTEXT_TOP_N_INVALID', `上下文 top_n 必须是 1 到 ${recallTopK} 的整数`, false);
|
|
9161
9880
|
try {
|
|
9162
|
-
|
|
9881
|
+
if (recallMode !== 'fts')
|
|
9882
|
+
await this.environmentGuard?.('embedding');
|
|
9883
|
+
if (options.rerank !== false && this.config.rerankProvider !== 'none')
|
|
9884
|
+
await this.environmentGuard?.('rerank');
|
|
9163
9885
|
}
|
|
9164
9886
|
catch (cause) {
|
|
9165
9887
|
return failure(empty(), 'ENVIRONMENT_NOT_READY', cause instanceof Error ? cause.message : String(cause), true, '在 Tita 设置中完成环境准备并重新自检。');
|
|
9166
9888
|
}
|
|
9167
9889
|
const derived = await this.derivedHandle();
|
|
9168
9890
|
const liveCards = await this.loadCards();
|
|
9169
|
-
const liveKeys = new Set(liveCards.map(card => cardKey(card)));
|
|
9891
|
+
const liveKeys = new Set(liveCards.filter(card => card.status !== 'deprecated').map(card => cardKey(card)));
|
|
9170
9892
|
const snapshotHash = derived?.artifactHash(COMPILE_SNAPSHOT);
|
|
9171
9893
|
const cached = snapshotHash && this.recallSnapshotCache?.hash === snapshotHash ? this.recallSnapshotCache : undefined;
|
|
9172
9894
|
const raw = cached ? undefined : derived?.artifact(COMPILE_SNAPSHOT);
|
|
@@ -9188,7 +9910,6 @@ export class KnowledgeRuntime {
|
|
|
9188
9910
|
}
|
|
9189
9911
|
if (snapshot.notes.length === 0)
|
|
9190
9912
|
return failure(empty(), 'INDEX_NOT_READY', '知识索引尚未构建,请先编译知识库', true, '先执行 rebuild_knowledge,再进行召回。');
|
|
9191
|
-
const capturedFts = { ready: derived?.hasFts() ?? false, hits: derived?.searchFts(query, candidatePool) ?? [] };
|
|
9192
9913
|
const capturedVector = cached ? cached.vector : decodeVectorIndex(derived?.vectorIndex(), ENCODER_FINGERPRINT);
|
|
9193
9914
|
if (this.environmentGuard && recallMode !== 'fts' && (!capturedVector
|
|
9194
9915
|
|| capturedVector.manifest.provider !== this.config.embeddingProvider
|
|
@@ -9220,6 +9941,15 @@ export class KnowledgeRuntime {
|
|
|
9220
9941
|
const eligible = snapshot.notes.filter(unit => liveKeys.has(`${unit.entityType}:${unit.entityId}`)
|
|
9221
9942
|
&& (includeProblems || currentByKey.get(`${unit.entityType}:${unit.entityId}`)?.status !== 'problem'));
|
|
9222
9943
|
const eligibleIds = new Set(eligible.map(unit => unit.unitId));
|
|
9944
|
+
/**
|
|
9945
|
+
* 关键词路的可见性必须在 **SQL 里、在 LIMIT 之前**收窄。
|
|
9946
|
+
*
|
|
9947
|
+
* 原来这里是先 `LIMIT candidatePool` 再在 JS 里 `filter(eligibleIds)`:删除或停用
|
|
9948
|
+
* 卡片之后索引不会立刻重建,那些死 unitId 仍占着候选位,把能用的候选挤出窗口 ——
|
|
9949
|
+
* 关键词路于是报「0 命中」,整次召回按弃答处理(面板上也看不出原因)。
|
|
9950
|
+
* 向量路一直是先过滤再取 top-k(见下面的扫描循环),这里是它漏掉的那一半。
|
|
9951
|
+
*/
|
|
9952
|
+
const capturedFts = { ready: derived?.hasFts() ?? false, hits: derived?.searchFts(query, candidatePool, eligibleIds) ?? [] };
|
|
9223
9953
|
const queryTerms = [...new Set(segment(query))];
|
|
9224
9954
|
const warnings = [];
|
|
9225
9955
|
// 一份 SourceRaw 语料,两处要用:关键词路的**查询词覆盖率**必须对着「建索引时那份
|
|
@@ -9327,7 +10057,7 @@ export class KnowledgeRuntime {
|
|
|
9327
10057
|
].join('\n');
|
|
9328
10058
|
const reranked = await this.rerankNotes(rerankQuery, hits, this.config.rerankThreshold);
|
|
9329
10059
|
rerankScored = reranked.scored ?? [];
|
|
9330
|
-
hits = reranked.hits
|
|
10060
|
+
hits = reranked.hits;
|
|
9331
10061
|
rerank = reranked.result;
|
|
9332
10062
|
if (reranked.warning)
|
|
9333
10063
|
warnings.push(reranked.warning);
|
|
@@ -9337,7 +10067,7 @@ export class KnowledgeRuntime {
|
|
|
9337
10067
|
}
|
|
9338
10068
|
// 重排换的是相关性顺序,评分加权仍然要压在最上面 —— 否则「review 过的排前面一点」
|
|
9339
10069
|
// 只在关掉重排时成立,而重排默认是开着的。
|
|
9340
|
-
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId));
|
|
10070
|
+
hits = [...hits].sort((left, right) => rankKey(right) - rankKey(left) || left.unitId.localeCompare(right.unitId)).slice(0, recallTopK);
|
|
9341
10071
|
// --- R4 · 证据片段 -----------------------------------------------------
|
|
9342
10072
|
const truth = await this.storeHandle();
|
|
9343
10073
|
cards = cards.filter(card => truth?.card(card.entityId) !== undefined && card.sourceRefs.every(ref => truth?.source(ref) !== undefined));
|