dsh-data-domain-knowledge 0.1.1-alpha.2 → 0.1.1-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -16
- package/lib/client.js +1042 -2097
- package/lib/contract-refs.d.ts +0 -1
- package/lib/doc-page.js +1 -1
- package/lib/pages.js +2489 -1060
- package/lib/types/agent-behavior-rules.d.ts.map +1 -1
- package/lib/types/agent-behavior-rules.js +3 -2
- package/lib/types/agent-behavior-rules.js.map +1 -1
- package/lib/types/axis-contract.js +1 -1
- package/lib/types/axis-contract.js.map +1 -1
- package/lib/types/card-organize.d.ts +2 -1
- package/lib/types/card-organize.d.ts.map +1 -1
- package/lib/types/card-organize.js +5 -3
- package/lib/types/card-organize.js.map +1 -1
- package/lib/types/config-schema.d.ts +1 -1
- package/lib/types/config-schema.js +1 -1
- package/lib/types/convert/host.d.ts +2 -3
- package/lib/types/convert/host.d.ts.map +1 -1
- package/lib/types/convert/host.js +11 -2
- package/lib/types/convert/host.js.map +1 -1
- package/lib/types/convert/ingest.d.ts +14 -20
- package/lib/types/convert/ingest.d.ts.map +1 -1
- package/lib/types/convert/ingest.js +8 -5
- package/lib/types/convert/ingest.js.map +1 -1
- package/lib/types/diagnosis/store.d.ts +2 -2
- package/lib/types/diagnosis/store.d.ts.map +1 -1
- package/lib/types/diagnosis/store.js +10 -2
- package/lib/types/diagnosis/store.js.map +1 -1
- package/lib/types/diagnosis/types.d.ts +1 -1
- package/lib/types/diagnosis/types.js +1 -1
- package/lib/types/extract-model.d.ts +3 -0
- package/lib/types/extract-model.d.ts.map +1 -1
- package/lib/types/extract-model.js.map +1 -1
- package/lib/types/images.js +1 -1
- package/lib/types/index.d.ts +0 -4
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +99 -91
- package/lib/types/index.js.map +1 -1
- package/lib/types/knowledge-bases.d.ts +3 -0
- package/lib/types/knowledge-bases.d.ts.map +1 -1
- package/lib/types/knowledge-bases.js +22 -2
- package/lib/types/knowledge-bases.js.map +1 -1
- package/lib/types/knowledge-contract.d.ts +6 -0
- package/lib/types/knowledge-contract.d.ts.map +1 -1
- package/lib/types/knowledge-contract.js +6 -2
- package/lib/types/knowledge-contract.js.map +1 -1
- package/lib/types/layout.d.ts +9 -26
- package/lib/types/layout.d.ts.map +1 -1
- package/lib/types/layout.js +10 -28
- package/lib/types/layout.js.map +1 -1
- package/lib/types/materialize-snapshot.d.ts +5 -0
- package/lib/types/materialize-snapshot.d.ts.map +1 -1
- package/lib/types/migrate.d.ts +3 -1
- package/lib/types/migrate.d.ts.map +1 -1
- package/lib/types/migrate.js +3 -1
- package/lib/types/migrate.js.map +1 -1
- package/lib/types/model-text.d.ts +7 -0
- package/lib/types/model-text.d.ts.map +1 -1
- package/lib/types/model-text.js +31 -9
- package/lib/types/model-text.js.map +1 -1
- package/lib/types/organize-contract.d.ts.map +1 -1
- package/lib/types/organize-contract.js +2 -1
- package/lib/types/organize-contract.js.map +1 -1
- package/lib/types/pipeline/engine.d.ts +28 -4
- package/lib/types/pipeline/engine.d.ts.map +1 -1
- package/lib/types/pipeline/engine.js +68 -15
- package/lib/types/pipeline/engine.js.map +1 -1
- package/lib/types/pipeline/progress.d.ts +12 -0
- package/lib/types/pipeline/progress.d.ts.map +1 -0
- package/lib/types/pipeline/progress.js +37 -0
- package/lib/types/pipeline/progress.js.map +1 -0
- package/lib/types/pipeline/resumable-extract.d.ts +45 -0
- package/lib/types/pipeline/resumable-extract.d.ts.map +1 -0
- package/lib/types/pipeline/resumable-extract.js +262 -0
- package/lib/types/pipeline/resumable-extract.js.map +1 -0
- package/lib/types/pipeline/store.d.ts +8 -2
- package/lib/types/pipeline/store.d.ts.map +1 -1
- package/lib/types/pipeline/store.js +27 -5
- package/lib/types/pipeline/store.js.map +1 -1
- package/lib/types/pipeline/types.d.ts +56 -0
- package/lib/types/pipeline/types.d.ts.map +1 -1
- package/lib/types/pipeline/types.js +18 -1
- package/lib/types/pipeline/types.js.map +1 -1
- package/lib/types/present.d.ts +1 -1
- package/lib/types/present.js +1 -1
- package/lib/types/preset-ids.d.ts +0 -1
- package/lib/types/preset-ids.d.ts.map +1 -1
- package/lib/types/preset-ids.js +0 -3
- package/lib/types/preset-ids.js.map +1 -1
- package/lib/types/query-analysis.d.ts +0 -5
- package/lib/types/query-analysis.d.ts.map +1 -1
- package/lib/types/query-analysis.js +0 -16
- package/lib/types/query-analysis.js.map +1 -1
- package/lib/types/recall-delivery.d.ts +36 -0
- package/lib/types/recall-delivery.d.ts.map +1 -0
- package/lib/types/recall-delivery.js +16 -0
- package/lib/types/recall-delivery.js.map +1 -0
- package/lib/types/retrieval/scope.d.ts +17 -0
- package/lib/types/retrieval/scope.d.ts.map +1 -0
- package/lib/types/retrieval/scope.js +11 -0
- package/lib/types/retrieval/scope.js.map +1 -0
- package/lib/types/runtime.d.ts +75 -14
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +454 -198
- package/lib/types/runtime.js.map +1 -1
- package/lib/types/settings-volatile.d.ts +0 -7
- package/lib/types/settings-volatile.d.ts.map +1 -1
- package/lib/types/settings-volatile.js +0 -7
- package/lib/types/settings-volatile.js.map +1 -1
- package/lib/types/source-identity.d.ts +34 -0
- package/lib/types/source-identity.d.ts.map +1 -0
- package/lib/types/source-identity.js +85 -0
- package/lib/types/source-identity.js.map +1 -0
- package/lib/types/source-read.d.ts +18 -0
- package/lib/types/source-read.d.ts.map +1 -0
- package/lib/types/source-read.js +49 -0
- package/lib/types/source-read.js.map +1 -0
- package/lib/types/store/axis-references.d.ts.map +1 -1
- package/lib/types/store/axis-references.js +5 -3
- package/lib/types/store/axis-references.js.map +1 -1
- package/lib/types/store/knowledge-db.d.ts +5 -29
- package/lib/types/store/knowledge-db.d.ts.map +1 -1
- package/lib/types/store/knowledge-db.js +57 -2
- package/lib/types/store/knowledge-db.js.map +1 -1
- package/lib/types/store/source-identity.d.ts +8 -0
- package/lib/types/store/source-identity.d.ts.map +1 -0
- package/lib/types/store/source-identity.js +56 -0
- package/lib/types/store/source-identity.js.map +1 -0
- package/lib/types/types.d.ts +46 -5
- package/lib/types/types.d.ts.map +1 -1
- package/lib/types/types.js +6 -3
- package/lib/types/types.js.map +1 -1
- package/lib/types/wiki-availability.d.ts +11 -0
- package/lib/types/wiki-availability.d.ts.map +1 -0
- package/lib/types/wiki-availability.js +39 -0
- package/lib/types/wiki-availability.js.map +1 -0
- package/lib/types/wiki-finalize.d.ts +4 -2
- package/lib/types/wiki-finalize.d.ts.map +1 -1
- package/lib/types/wiki-finalize.js +9 -27
- package/lib/types/wiki-finalize.js.map +1 -1
- package/lib/types/wiki-links.d.ts +35 -0
- package/lib/types/wiki-links.d.ts.map +1 -0
- package/lib/types/wiki-links.js +129 -0
- package/lib/types/wiki-links.js.map +1 -0
- package/lib/types/wiki-trial-types.d.ts +3 -0
- package/lib/types/wiki-trial-types.d.ts.map +1 -1
- package/lib/types/wiki-trial.d.ts +3 -3
- package/lib/types/wiki-trial.d.ts.map +1 -1
- package/lib/types/wiki-trial.js +14 -3
- package/lib/types/wiki-trial.js.map +1 -1
- package/package.json +16 -16
- package/scripts/doctor.mjs +0 -7
- package/scripts/setup-embedding.sh +0 -0
- package/src/agent-behavior-rules.ts +3 -2
- package/src/axis-contract.ts +1 -1
- package/src/card-organize.ts +5 -3
- package/src/client/AttributionEventGuide.tsx +3 -3
- package/src/client/AxisMatrix.tsx +86 -19
- package/src/client/AxisRegistrySection.tsx +1 -1
- package/src/client/CompileSettingsView.tsx +473 -0
- package/src/client/DirectoryPicker.tsx +0 -2
- package/src/client/ExtractionTasksPage.tsx +557 -423
- package/src/client/GuideArchitectureCanvas.tsx +5 -12
- package/src/client/ImportFilesDialog.tsx +1 -0
- package/src/client/InfoHint.tsx +1 -1
- package/src/client/KnowledgeBaseTab.tsx +141 -148
- package/src/client/KnowledgePanel.tsx +16 -9
- package/src/client/OperationLogTab.tsx +1 -1
- package/src/client/PipelineRunView.tsx +173 -0
- package/src/client/RecallPanel.tsx +5 -16
- package/src/client/TabGuide.tsx +3 -7
- package/src/client/WikiFlowPanel.tsx +10 -11
- package/src/client/arch/NodeLogicPanel.tsx +3 -3
- package/src/client/arch/model.ts +5 -113
- package/src/client/architecture.css +1 -1
- package/src/client/architecture.tsx +13 -27
- package/src/client/attribution/AttributionTab.tsx +141 -10
- package/src/client/pages.css +305 -180
- package/src/client/pages.tsx +7 -12
- package/src/client/styles.css +15 -96
- package/src/config-schema.ts +1 -1
- package/src/convert/host.ts +13 -5
- package/src/convert/ingest.ts +20 -7
- package/src/diagnosis/store.ts +7 -2
- package/src/diagnosis/types.ts +1 -1
- package/src/extract-model.ts +3 -0
- package/src/images.ts +1 -1
- package/src/index.ts +92 -97
- package/src/knowledge-bases.ts +23 -3
- package/src/knowledge-contract.ts +7 -2
- package/src/layout.ts +10 -32
- package/src/materialize-snapshot.ts +1 -0
- package/src/migrate.ts +3 -1
- package/src/model-text.ts +33 -11
- package/src/organize-contract.ts +2 -1
- package/src/pipeline/engine.ts +85 -22
- package/src/pipeline/progress.ts +40 -0
- package/src/pipeline/resumable-extract.ts +233 -0
- package/src/pipeline/store.ts +28 -6
- package/src/pipeline/types.ts +47 -2
- package/src/present.ts +1 -1
- package/src/preset-ids.ts +0 -4
- package/src/query-analysis.ts +0 -20
- package/src/recall-delivery.ts +19 -0
- package/src/retrieval/scope.ts +10 -0
- package/src/runtime.ts +412 -206
- package/src/settings-volatile.ts +0 -7
- package/src/source-identity.ts +85 -0
- package/src/source-read.ts +47 -0
- package/src/store/axis-references.ts +5 -3
- package/src/store/knowledge-db.ts +54 -2
- package/src/store/source-identity.ts +42 -0
- package/src/types.ts +18 -5
- package/src/wiki-availability.ts +37 -0
- package/src/wiki-finalize.ts +9 -25
- package/src/wiki-links.ts +120 -0
- package/src/wiki-trial-types.ts +3 -2
- package/src/wiki-trial.ts +17 -6
- package/lib/types/intent/engine.d.ts +0 -122
- package/lib/types/intent/engine.d.ts.map +0 -1
- package/lib/types/intent/engine.js +0 -488
- package/lib/types/intent/engine.js.map +0 -1
- package/lib/types/intent/host.d.ts +0 -68
- package/lib/types/intent/host.d.ts.map +0 -1
- package/lib/types/intent/host.js +0 -249
- package/lib/types/intent/host.js.map +0 -1
- package/lib/types/intent/scope.d.ts +0 -25
- package/lib/types/intent/scope.d.ts.map +0 -1
- package/lib/types/intent/scope.js +0 -116
- package/lib/types/intent/scope.js.map +0 -1
- package/lib/types/intent/store.d.ts +0 -33
- package/lib/types/intent/store.d.ts.map +0 -1
- package/lib/types/intent/store.js +0 -239
- package/lib/types/intent/store.js.map +0 -1
- package/lib/types/intent/tools.d.ts +0 -62
- package/lib/types/intent/tools.d.ts.map +0 -1
- package/lib/types/intent/tools.js +0 -130
- package/lib/types/intent/tools.js.map +0 -1
- package/lib/types/intent/types.d.ts +0 -170
- package/lib/types/intent/types.d.ts.map +0 -1
- package/lib/types/intent/types.js +0 -56
- package/lib/types/intent/types.js.map +0 -1
- package/src/client/IntentLogic.tsx +0 -192
- package/src/client/IntentTab.tsx +0 -343
- package/src/client/TypeDefinitionDrawer.tsx +0 -195
- package/src/intent/README.md +0 -42
- package/src/intent/engine.ts +0 -515
- package/src/intent/host.ts +0 -315
- package/src/intent/scope.ts +0 -125
- package/src/intent/store.ts +0 -218
- package/src/intent/tools.ts +0 -154
- package/src/intent/types.ts +0 -229
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import type { ModelCall } from '../extract-model.js'
|
|
3
|
+
import { callModelWithRetry } from '../model-text.js'
|
|
4
|
+
import { decodeExtractionTable } from '../extract-table.js'
|
|
5
|
+
import { normalizeKnowledgeExtraction } from '../knowledge-contract.js'
|
|
6
|
+
import { runWikiTrial, headingPathAt, locateQuote, type AxisPromptBlock } from '../wiki-trial.js'
|
|
7
|
+
import type { TrialReport } from '../wiki-trial-types.js'
|
|
8
|
+
import type { PipelineStore } from './store.js'
|
|
9
|
+
import { EXTRACTION_RECOVERY_LIMITS, type PipelineActivity, type PipelineActivityUpdate } from './types.js'
|
|
10
|
+
|
|
11
|
+
type Row = Record<string, any>
|
|
12
|
+
export const EXTRACTION_PLAN_VERSION = 'semantic-chunks-v1'
|
|
13
|
+
const hash = (value: string): string => createHash('sha256').update(value).digest('hex')
|
|
14
|
+
|
|
15
|
+
/** Paragraph boundaries preserve fenced code, tables and lists as atomic blocks. No text is discarded. */
|
|
16
|
+
export function splitExtractionText(text: string, target = 6000): string[] {
|
|
17
|
+
const blocks: string[] = []
|
|
18
|
+
let block = '', fence = ''
|
|
19
|
+
for (const line of text.split(/(?<=\n)/)) {
|
|
20
|
+
const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]
|
|
21
|
+
if (marker) { if (!fence) fence = marker; else if (marker[0] === fence[0] && marker.length >= fence.length) fence = '' }
|
|
22
|
+
block += line
|
|
23
|
+
if (!fence && /^\s*$/.test(line)) { blocks.push(block); block = '' }
|
|
24
|
+
}
|
|
25
|
+
if (block) blocks.push(block)
|
|
26
|
+
const parts: string[] = []
|
|
27
|
+
let current = ''
|
|
28
|
+
// Long prose without blank lines can still split at complete sentences.
|
|
29
|
+
const units = blocks.flatMap(value => value.length > target && !/(^|\n)\s*(?:[`~]{3}|\||[-*+] |\d+[.)] )/.test(value)
|
|
30
|
+
? value.split(/(?<=[。!?.!?])(?=\s|[^\x00-\x7F])/) : [value])
|
|
31
|
+
for (const value of units) {
|
|
32
|
+
if (current.trim() && value.trim() && current.length + value.length > target) { parts.push(current); current = '' }
|
|
33
|
+
current += value
|
|
34
|
+
}
|
|
35
|
+
if (current) parts.push(current)
|
|
36
|
+
return parts.length ? parts : [text]
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function payload(raw: string): Row {
|
|
40
|
+
const body = raw.trim().replace(/^```(?:json|toon|text)?\s*\n/, '').replace(/\n```\s*$/, '')
|
|
41
|
+
return /^format:\s*knowledge-table-v[12]\s*$/m.test(body)
|
|
42
|
+
? decodeExtractionTable(body) : normalizeKnowledgeExtraction(JSON.parse(body)) as Row
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Merge already validated records; IDs are local to each response, quotes still point into the full source. */
|
|
46
|
+
export function mergeExtractionPayloads(values: Row[], chunked: boolean): Row {
|
|
47
|
+
const merged: Row = { subject: values[0]?.subject ?? '', knowledge: [], events: [], objects: [], actions: [], scenes: [], dropped: [] }
|
|
48
|
+
values.forEach((value, index) => {
|
|
49
|
+
for (const key of ['knowledge', 'events', 'objects', 'actions', 'scenes', 'dropped']) {
|
|
50
|
+
for (const original of value[key] ?? []) {
|
|
51
|
+
const row = structuredClone(original)
|
|
52
|
+
if (row.id) row.id = `part${index + 1}_${row.id}`
|
|
53
|
+
// Full-document completeness cannot be inferred from a partial view.
|
|
54
|
+
if (chunked && row.data?.procedure?.completeness === 'complete') {
|
|
55
|
+
row.data.procedure.completeness = 'partial'
|
|
56
|
+
row.data.procedure.limitations.push('由文档分片抽取,跨章节方法完整性尚未核验')
|
|
57
|
+
}
|
|
58
|
+
merged[key].push(row)
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
})
|
|
62
|
+
return merged
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface ResumableExtractionInput {
|
|
66
|
+
sourceId: string
|
|
67
|
+
title: string
|
|
68
|
+
text: string
|
|
69
|
+
fingerprint: string
|
|
70
|
+
store: PipelineStore
|
|
71
|
+
force?: boolean
|
|
72
|
+
axes?: AxisPromptBlock
|
|
73
|
+
note?: string
|
|
74
|
+
call: ModelCall
|
|
75
|
+
modelRequest: { provider?: string; model?: string; reasoningEffort?: string; maxTokens?: number }
|
|
76
|
+
sessionId: string
|
|
77
|
+
outputFormat: 'table' | 'json'
|
|
78
|
+
onDelta: (text: string) => void
|
|
79
|
+
progress: (text: string, activity?: PipelineActivityUpdate) => void
|
|
80
|
+
checkpoint: () => Promise<void>
|
|
81
|
+
signal: AbortSignal
|
|
82
|
+
/** Injectable limits for bounded fault tests. */
|
|
83
|
+
targetChars?: number
|
|
84
|
+
timeoutMs?: number
|
|
85
|
+
recoveryTimeoutMs?: number
|
|
86
|
+
requestLimit?: number
|
|
87
|
+
now?: () => number
|
|
88
|
+
sleep?: (ms: number) => Promise<void>
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** Execution cache only. Activation remains the pipeline's document-level transaction. */
|
|
92
|
+
export async function runResumableExtraction(input: ResumableExtractionInput): Promise<TrialReport> {
|
|
93
|
+
const scope = hash(`${EXTRACTION_PLAN_VERSION}:${input.sourceId}:${input.fingerprint}`)
|
|
94
|
+
input.store.clearOtherChunks(scope, input.sourceId)
|
|
95
|
+
if (input.force) input.store.clearChunks(scope)
|
|
96
|
+
const parts = splitExtractionText(input.text, input.targetChars)
|
|
97
|
+
const values: Row[] = []
|
|
98
|
+
let requests = 0, completed = 0, failed = 0, reused = 0
|
|
99
|
+
// A restored split tree already has more leaves than its original plan.
|
|
100
|
+
const leafCount = (text: string, start: number): number => {
|
|
101
|
+
const saved = input.store.chunk(hash(`${scope}:${start}:${text}`))
|
|
102
|
+
if (!saved?.__split) return 1
|
|
103
|
+
let offset = start
|
|
104
|
+
return (saved.__split as string[]).reduce((sum, child) => {
|
|
105
|
+
const count = leafCount(child, offset); offset += child.length
|
|
106
|
+
return sum + count
|
|
107
|
+
}, 0)
|
|
108
|
+
}
|
|
109
|
+
let planOffset = 0
|
|
110
|
+
let total = parts.reduce((sum, part) => { const count = leafCount(part, planOffset); planOffset += part.length; return sum + count }, 0)
|
|
111
|
+
const progress = (phase: PipelineActivity['phase'], text: string, attempt?: number): void => input.progress(text, {
|
|
112
|
+
phase, objectId: input.sourceId, objectTitle: input.title,
|
|
113
|
+
chunks: { initial: parts.length, total, saved: completed, reused, failed }, ...(attempt ? { attempt } : {}),
|
|
114
|
+
})
|
|
115
|
+
progress('preparing', `抽取计划:初始 ${parts.length} 片,当前 ${total} 片`)
|
|
116
|
+
const now = input.now ?? Date.now
|
|
117
|
+
const recoveryMs = input.recoveryTimeoutMs ?? EXTRACTION_RECOVERY_LIMITS.durationMs
|
|
118
|
+
const deadline = now() + recoveryMs
|
|
119
|
+
const requestBudget = input.requestLimit ?? Math.max(EXTRACTION_RECOVERY_LIMITS.minimumRequests, parts.length * EXTRACTION_RECOVERY_LIMITS.requestsPerInitialChunk)
|
|
120
|
+
const budgetError = (): Error | undefined => {
|
|
121
|
+
const timedOut = now() >= deadline
|
|
122
|
+
if (!timedOut && requests < requestBudget) return undefined
|
|
123
|
+
const reason = timedOut ? `本次文档抽取已达到 ${recoveryMs >= 60_000 ? `${recoveryMs / 60_000} 分钟` : `${recoveryMs / 1000} 秒`}时间上限` : `本次文档抽取已达到 ${requestBudget} 次请求上限(含重试与拆分)`
|
|
124
|
+
return Object.assign(new Error(`${reason};已保存 ${completed}/${total} 个分片,本轮已发 ${requests} 次请求。已保存分片会复用,请继续未完成部分。此限制与单次最大输出 token 无关。`), { code: timedOut ? 'EXTRACTION_TIME_BUDGET' : 'EXTRACTION_REQUEST_BUDGET' })
|
|
125
|
+
}
|
|
126
|
+
const requireBudget = (): void => {
|
|
127
|
+
const error = budgetError()
|
|
128
|
+
if (error) { progress('failed', error.message); throw error }
|
|
129
|
+
}
|
|
130
|
+
const visit = async (text: string, start: number, depth: number): Promise<void> => {
|
|
131
|
+
await input.checkpoint()
|
|
132
|
+
input.signal.throwIfAborted()
|
|
133
|
+
const key = hash(`${scope}:${start}:${text}`)
|
|
134
|
+
const saved = input.store.chunk(key)
|
|
135
|
+
if (saved?.__split) {
|
|
136
|
+
let offset = start
|
|
137
|
+
for (const child of saved.__split as string[]) { await visit(child, offset, depth + 1); offset += child.length }
|
|
138
|
+
return
|
|
139
|
+
}
|
|
140
|
+
if (saved) { values.push(saved); completed++; reused++; progress('reused', `已复用分片;已保存 ${completed}/${total} 片`); return }
|
|
141
|
+
requireBudget()
|
|
142
|
+
progress('waiting-model', `等待模型返回;已保存 ${completed}/${total} 片`)
|
|
143
|
+
let lastReport: TrialReport | undefined
|
|
144
|
+
let attempt = 0
|
|
145
|
+
try {
|
|
146
|
+
await callModelWithRetry(async onDelta => {
|
|
147
|
+
await input.checkpoint()
|
|
148
|
+
input.signal.throwIfAborted()
|
|
149
|
+
requireBudget()
|
|
150
|
+
requests++
|
|
151
|
+
attempt++
|
|
152
|
+
progress('waiting-model', `正在请求分片;第 ${attempt} 次尝试,已保存 ${completed}/${total} 片`, attempt)
|
|
153
|
+
const timeout = AbortSignal.timeout(Math.min(input.timeoutMs ?? EXTRACTION_RECOVERY_LIMITS.requestTimeoutMs, Math.max(1, deadline - now())))
|
|
154
|
+
const signal = AbortSignal.any([input.signal, timeout])
|
|
155
|
+
lastReport = undefined
|
|
156
|
+
const startedAt = Date.now()
|
|
157
|
+
let live = true, preview = ''
|
|
158
|
+
const heading = headingPathAt(input.text, start).join(' / ')
|
|
159
|
+
let callError: unknown
|
|
160
|
+
let firstDeltaMs: number | undefined
|
|
161
|
+
const reportPromise = runWikiTrial({ title: input.title, text, docId: input.sourceId, axes: input.axes,
|
|
162
|
+
note: `${input.note ?? ''}\n当前材料是文档分片。前文标题定位(不能作为引文证据):\n${heading}\n只引用本片原文;方法依赖未提供的上下文时标记 partial 并记录缺项,不补造步骤。`,
|
|
163
|
+
modelRequest: input.modelRequest, sessionId: input.sessionId, outputFormat: input.outputFormat,
|
|
164
|
+
call: async request => { try { return await input.call({ ...request, signal, retryAttempts: 1 }) } catch (error) { callError = error; throw error } },
|
|
165
|
+
onDelta: text => { if (live) {
|
|
166
|
+
if (firstDeltaMs === undefined) { firstDeltaMs = Date.now() - startedAt; progress('receiving', '正在接收模型输出', attempt) }
|
|
167
|
+
preview = (preview + text).slice(-4 * 1024 * 1024); onDelta(text)
|
|
168
|
+
} },
|
|
169
|
+
})
|
|
170
|
+
let abortListener: (() => void) | undefined
|
|
171
|
+
try {
|
|
172
|
+
lastReport = await Promise.race([reportPromise, new Promise<never>((_, reject) => {
|
|
173
|
+
abortListener = () => reject(input.signal.aborted ? input.signal.reason : now() >= deadline ? budgetError() : new Error('request timed out'))
|
|
174
|
+
signal.addEventListener('abort', abortListener, { once: true })
|
|
175
|
+
if (signal.aborted) abortListener()
|
|
176
|
+
})])
|
|
177
|
+
if (lastReport.model.parsed !== true && callError) throw callError
|
|
178
|
+
if (lastReport.model.parsed !== true) throw new Error(lastReport.model.why ?? (lastReport.exceptions.map(e => e.message).join(';') || '分片校验未通过'))
|
|
179
|
+
progress('validating', '正在校验分片与原文引文', attempt)
|
|
180
|
+
const raw = lastReport.model.response!
|
|
181
|
+
const value = payload(raw)
|
|
182
|
+
for (const row of [...(value.knowledge ?? []), ...(value.events ?? [])]) {
|
|
183
|
+
for (const item of [row, ...(row.timeline ?? [])]) for (const source of item.sources ?? []) {
|
|
184
|
+
source.extraction_range = { start, end: start + text.length }
|
|
185
|
+
const local = locateQuote(text, source.quote)
|
|
186
|
+
if (local.locator) source.heading_path = headingPathAt(input.text, start + local.locator.start)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
input.signal.throwIfAborted()
|
|
190
|
+
progress('saving', '正在保存已校验分片', attempt)
|
|
191
|
+
input.store.chunkAttempt(key, scope, input.sourceId, { outcome: 'success', elapsedMs: Date.now() - startedAt, firstDeltaMs, report: lastReport })
|
|
192
|
+
input.store.saveChunk(key, scope, input.sourceId, value)
|
|
193
|
+
return raw
|
|
194
|
+
} catch (error) {
|
|
195
|
+
input.store.chunkAttempt(key, scope, input.sourceId, { elapsedMs: Date.now() - startedAt, error: String(error), code: (error as Row)?.code, requestId: (error as Row)?.requestId, status: (error as Row)?.status, firstDeltaMs, response: preview, report: lastReport })
|
|
196
|
+
throw error
|
|
197
|
+
} finally { live = false; if (abortListener) signal.removeEventListener('abort', abortListener) }
|
|
198
|
+
}, { attempts: 2, delaysMs: [2000], sleep: input.sleep, signal: input.signal, onDelta: input.onDelta,
|
|
199
|
+
onRetry: () => progress('retrying', `请求超时或连接中断,等待自动重试;已保存 ${completed}/${total} 片`, attempt + 1),
|
|
200
|
+
})
|
|
201
|
+
values.push(input.store.chunk(key)!); completed++
|
|
202
|
+
progress('saving', `已保存 ${completed}/${total} 个抽取分片`)
|
|
203
|
+
} catch (error) {
|
|
204
|
+
input.signal.throwIfAborted()
|
|
205
|
+
if ((error as { code?: string })?.code === 'EXTRACTION_TIME_BUDGET' || (error as { code?: string })?.code === 'EXTRACTION_REQUEST_BUDGET') throw error
|
|
206
|
+
if (/TIMEOUT|timed?\s*out|超时|OUTPUT_TRUNCATED/i.test(String(error)) && depth < 2) {
|
|
207
|
+
const children = splitExtractionText(text, Math.max(1000, Math.floor(text.length / 2)))
|
|
208
|
+
if (children.length > 1) {
|
|
209
|
+
let offsetForPlan = start
|
|
210
|
+
total += children.reduce((sum, child) => { const count = leafCount(child, offsetForPlan); offsetForPlan += child.length; return sum + count }, 0) - 1
|
|
211
|
+
progress('splitting', `连续超时或输出过长,当前分片拆为 ${children.length} 片;计划更新为 ${total} 片`)
|
|
212
|
+
input.store.saveChunk(key, scope, input.sourceId, { __split: children })
|
|
213
|
+
let offset = start
|
|
214
|
+
for (const child of children) { await visit(child, offset, depth + 1); offset += child.length }
|
|
215
|
+
return
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
failed++
|
|
219
|
+
progress('failed', `分片未完成,已保存 ${completed}/${total} 片;${String(error)}`)
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
let offset = 0
|
|
223
|
+
for (const part of parts) { await visit(part, offset, 0); offset += part.length }
|
|
224
|
+
if (failed) return runWikiTrial({ title: input.title, text: input.text, docId: input.sourceId, modelRequest: input.modelRequest,
|
|
225
|
+
call: async () => { throw new Error(`抽取等待续跑:已保存 ${completed} 片,${failed} 片未完成;请继续未完成部分`) },
|
|
226
|
+
})
|
|
227
|
+
await input.checkpoint()
|
|
228
|
+
progress('validating', '分片已保存,正在合并并校验整份文档')
|
|
229
|
+
return runWikiTrial({ title: input.title, text: input.text, docId: input.sourceId, axes: input.axes, note: input.note,
|
|
230
|
+
modelRequest: input.modelRequest, sessionId: input.sessionId, outputFormat: input.outputFormat,
|
|
231
|
+
call: async () => JSON.stringify(mergeExtractionPayloads(values, parts.length > 1 || values.length > 1)),
|
|
232
|
+
})
|
|
233
|
+
}
|
package/src/pipeline/store.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { SqliteDatabase } from '../fts.js'
|
|
2
2
|
import { applyRetention } from '../retention.js'
|
|
3
3
|
import type { TrialReport } from '../wiki-trial-types.js'
|
|
4
|
-
import type { PipelineEvent, PipelineTask } from './types.js'
|
|
4
|
+
import type { PipelineActivity, PipelineEvent, PipelineTask } from './types.js'
|
|
5
5
|
|
|
6
6
|
export interface ExtractionCheckpoint {
|
|
7
7
|
id: string
|
|
@@ -18,6 +18,8 @@ export interface ExtractionCheckpoint {
|
|
|
18
18
|
export class PipelineStore {
|
|
19
19
|
constructor(readonly truth: SqliteDatabase, private readonly audit: SqliteDatabase) {
|
|
20
20
|
truth.exec(`
|
|
21
|
+
CREATE TABLE IF NOT EXISTS pipeline_chunks (id TEXT PRIMARY KEY, scope TEXT NOT NULL, source_id TEXT NOT NULL, payload TEXT NOT NULL);
|
|
22
|
+
CREATE INDEX IF NOT EXISTS pipeline_chunks_scope ON pipeline_chunks(scope);
|
|
21
23
|
CREATE TABLE IF NOT EXISTS pipeline_extractions (
|
|
22
24
|
id TEXT PRIMARY KEY, source_id TEXT NOT NULL, source_version INTEGER NOT NULL,
|
|
23
25
|
source_hash TEXT NOT NULL, fingerprint TEXT NOT NULL, report TEXT NOT NULL,
|
|
@@ -28,6 +30,7 @@ export class PipelineStore {
|
|
|
28
30
|
CREATE TABLE IF NOT EXISTS pipeline_materializations (group_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, card_id TEXT NOT NULL);
|
|
29
31
|
`)
|
|
30
32
|
audit.exec(`
|
|
33
|
+
CREATE TABLE IF NOT EXISTS pipeline_chunk_attempts (id INTEGER PRIMARY KEY AUTOINCREMENT, chunk_id TEXT NOT NULL, scope TEXT NOT NULL, source_id TEXT NOT NULL, payload TEXT NOT NULL);
|
|
31
34
|
CREATE TABLE IF NOT EXISTS pipeline_tasks (id TEXT PRIMARY KEY, queued_at TEXT NOT NULL, state TEXT NOT NULL, payload TEXT NOT NULL);
|
|
32
35
|
CREATE INDEX IF NOT EXISTS pipeline_tasks_time ON pipeline_tasks(queued_at DESC);
|
|
33
36
|
CREATE TABLE IF NOT EXISTS pipeline_events (
|
|
@@ -38,14 +41,29 @@ export class PipelineStore {
|
|
|
38
41
|
`)
|
|
39
42
|
const eventColumns = audit.prepare('PRAGMA table_info(pipeline_events)').all() as Array<{ name: string }>
|
|
40
43
|
if (!eventColumns.some(column => column.name === 'stage')) audit.exec('ALTER TABLE pipeline_events ADD COLUMN stage TEXT')
|
|
44
|
+
if (!eventColumns.some(column => column.name === 'activity')) audit.exec('ALTER TABLE pipeline_events ADD COLUMN activity TEXT')
|
|
41
45
|
// A new runtime never claims that an abandoned request is still running.
|
|
42
46
|
for (const task of (audit.prepare("SELECT payload FROM pipeline_tasks WHERE state IN ('queued','running','pausing','paused','cancelling')").all() as Array<{ payload: string }>).map(row => JSON.parse(row.payload) as PipelineTask)) {
|
|
43
47
|
if (['queued', 'running', 'pausing', 'paused', 'cancelling'].includes(task.state)) {
|
|
44
|
-
this.putTask({ ...task, state: 'interrupted', finishedAt: new Date().toISOString(), summary: '服务已重新启动;已完成的阶段保留,可重新运行此任务。' })
|
|
48
|
+
this.putTask({ ...task, state: 'interrupted', recoverableOnRestart: task.state === 'queued' || task.state === 'running', finishedAt: new Date().toISOString(), summary: '服务已重新启动;已完成的阶段保留,可重新运行此任务。' })
|
|
45
49
|
}
|
|
46
50
|
}
|
|
47
51
|
}
|
|
48
52
|
|
|
53
|
+
chunk(id: string): Record<string, any> | undefined {
|
|
54
|
+
const row = this.truth.prepare('SELECT payload FROM pipeline_chunks WHERE id=?').get(id) as { payload: string } | undefined
|
|
55
|
+
return row ? JSON.parse(row.payload) : undefined
|
|
56
|
+
}
|
|
57
|
+
saveChunk(id: string, scope: string, sourceId: string, value: unknown): void {
|
|
58
|
+
this.truth.prepare('INSERT INTO pipeline_chunks(id,scope,source_id,payload) VALUES (?,?,?,?) ON CONFLICT(id) DO NOTHING').run(id, scope, sourceId, JSON.stringify(value))
|
|
59
|
+
}
|
|
60
|
+
clearOtherChunks(scope: string, sourceId: string): void { this.truth.prepare('DELETE FROM pipeline_chunks WHERE source_id=? AND scope<>?').run(sourceId, scope) }
|
|
61
|
+
clearChunks(scope: string): void { this.truth.prepare('DELETE FROM pipeline_chunks WHERE scope=?').run(scope) }
|
|
62
|
+
chunkAttempt(id: string, scope: string, sourceId: string, value: unknown): void {
|
|
63
|
+
this.audit.prepare('INSERT INTO pipeline_chunk_attempts(chunk_id,scope,source_id,payload) VALUES (?,?,?,?)').run(id, scope, sourceId, JSON.stringify(value))
|
|
64
|
+
applyRetention(this.audit, 'pipeline_chunk_attempts', { table: 'pipeline_chunk_attempts', key: 'id', keep: 200 })
|
|
65
|
+
}
|
|
66
|
+
|
|
49
67
|
putTask(task: PipelineTask): void {
|
|
50
68
|
this.audit.prepare('INSERT INTO pipeline_tasks(id, queued_at, state, payload) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET state=excluded.state, payload=excluded.payload')
|
|
51
69
|
.run(task.id, task.queuedAt, task.state, JSON.stringify(task))
|
|
@@ -75,9 +93,13 @@ export class PipelineStore {
|
|
|
75
93
|
const total = Number((this.audit.prepare('SELECT COUNT(*) AS n FROM pipeline_tasks').get() as { n: number }).n)
|
|
76
94
|
return { tasks: rows.map(row => JSON.parse(row.payload) as PipelineTask), total }
|
|
77
95
|
}
|
|
78
|
-
|
|
79
|
-
this.audit.prepare(
|
|
80
|
-
|
|
96
|
+
activeTasks(): PipelineTask[] {
|
|
97
|
+
const rows = this.audit.prepare("SELECT payload FROM pipeline_tasks WHERE state IN ('queued','running','pausing','paused','cancelling') ORDER BY queued_at").all() as Array<{ payload: string }>
|
|
98
|
+
return rows.map(row => JSON.parse(row.payload) as PipelineTask)
|
|
99
|
+
}
|
|
100
|
+
event(taskId: string, kind: PipelineEvent['kind'], text: string, sourceId?: string, stage?: PipelineEvent['stage'], activity?: PipelineActivity): void {
|
|
101
|
+
this.audit.prepare('INSERT INTO pipeline_events(task_id,kind,text,source_id,at,stage,activity) VALUES (?,?,?,?,?,?,?)')
|
|
102
|
+
.run(taskId, kind, text, sourceId ?? null, new Date().toISOString(), stage ?? null, activity ? JSON.stringify(activity) : null)
|
|
81
103
|
}
|
|
82
104
|
events(taskId: string, after = 0, limit = 100): { events: PipelineEvent[]; next: number; hasMore: boolean } {
|
|
83
105
|
const take = Number.isFinite(limit) ? Math.max(1, Math.min(500, Math.trunc(limit))) : 100
|
|
@@ -85,7 +107,7 @@ export class PipelineStore {
|
|
|
85
107
|
const rows = this.audit.prepare('SELECT * FROM pipeline_events WHERE task_id=? AND seq>? ORDER BY seq LIMIT ?')
|
|
86
108
|
.all(taskId, after, take + 1) as Array<Record<string, unknown>>
|
|
87
109
|
const hasMore = rows.length > take
|
|
88
|
-
const events = rows.slice(0, take).map(row => ({ seq: Number(row.seq), taskId: String(row.task_id), kind: String(row.kind) as PipelineEvent['kind'], text: String(row.text), ...(row.source_id ? { sourceId: String(row.source_id) } : {}), ...(row.stage ? { stage: String(row.stage) as PipelineEvent['stage'] } : {}), at: String(row.at) }))
|
|
110
|
+
const events = rows.slice(0, take).map(row => ({ seq: Number(row.seq), taskId: String(row.task_id), kind: String(row.kind) as PipelineEvent['kind'], text: String(row.text), ...(row.source_id ? { sourceId: String(row.source_id) } : {}), ...(row.stage ? { stage: String(row.stage) as PipelineEvent['stage'] } : {}), ...(row.activity ? { activity: JSON.parse(String(row.activity)) as PipelineActivity } : {}), at: String(row.at) }))
|
|
89
111
|
return { events, next: events.at(-1)?.seq ?? after, hasMore }
|
|
90
112
|
}
|
|
91
113
|
checkpoint(id: string): ExtractionCheckpoint | undefined {
|
package/src/pipeline/types.ts
CHANGED
|
@@ -2,6 +2,34 @@
|
|
|
2
2
|
export type PipelineStage = 'extract' | 'normalize' | 'derive' | 'materialize' | 'build' | 'all'
|
|
3
3
|
export type PipelineState = 'queued' | 'running' | 'pausing' | 'paused' | 'cancelling' | 'succeeded' | 'partial' | 'failed' | 'cancelled' | 'interrupted'
|
|
4
4
|
export type PipelineAction = 'pause' | 'resume' | 'cancel'
|
|
5
|
+
export type PipelineWorkStage = Exclude<PipelineStage, 'all'>
|
|
6
|
+
export const PIPELINE_STAGE_ORDER: readonly PipelineWorkStage[] = ['extract', 'normalize', 'derive', 'materialize', 'build']
|
|
7
|
+
export const EXTRACTION_TIMEOUT_OPTIONS = [30, 60, 100] as const
|
|
8
|
+
export type ExtractionTimeoutMinutes = typeof EXTRACTION_TIMEOUT_OPTIONS[number]
|
|
9
|
+
export const DEFAULT_EXTRACTION_TIMEOUT_MINUTES: ExtractionTimeoutMinutes = 30
|
|
10
|
+
export function isExtractionTimeoutMinutes(value: unknown): value is ExtractionTimeoutMinutes {
|
|
11
|
+
return typeof value === 'number' && EXTRACTION_TIMEOUT_OPTIONS.some(option => option === value)
|
|
12
|
+
}
|
|
13
|
+
export function normalizeExtractionTimeoutMinutes(value: unknown): ExtractionTimeoutMinutes {
|
|
14
|
+
return isExtractionTimeoutMinutes(value) ? value : DEFAULT_EXTRACTION_TIMEOUT_MINUTES
|
|
15
|
+
}
|
|
16
|
+
export const EXTRACTION_RECOVERY_LIMITS = { durationMs: DEFAULT_EXTRACTION_TIMEOUT_MINUTES * 60_000, requestTimeoutMs: 180_000, minimumRequests: 12, requestsPerInitialChunk: 8 } as const
|
|
17
|
+
|
|
18
|
+
/** Durable execution observations, independent of library freshness and historical results. */
|
|
19
|
+
export interface PipelineActivity {
|
|
20
|
+
phase: 'preparing' | 'processing' | 'waiting-model' | 'receiving' | 'validating' | 'saving' | 'retrying' | 'splitting' | 'reused' | 'completed' | 'failed'
|
|
21
|
+
at: string
|
|
22
|
+
objectId?: string
|
|
23
|
+
objectTitle?: string
|
|
24
|
+
chunks?: { initial: number; total: number; saved: number; reused: number; failed: number }
|
|
25
|
+
attempt?: number
|
|
26
|
+
}
|
|
27
|
+
export type PipelineActivityUpdate = Omit<PipelineActivity, 'at'>
|
|
28
|
+
export interface PipelineItemState {
|
|
29
|
+
state: 'processing' | 'succeeded' | 'reused' | 'failed'
|
|
30
|
+
title: string
|
|
31
|
+
summary?: string
|
|
32
|
+
}
|
|
5
33
|
|
|
6
34
|
export interface PipelineSelection {
|
|
7
35
|
sourceIds?: string[]
|
|
@@ -13,6 +41,7 @@ export interface PipelineModel {
|
|
|
13
41
|
provider?: string
|
|
14
42
|
model?: string
|
|
15
43
|
reasoningEffort?: string
|
|
44
|
+
maxTokens?: number
|
|
16
45
|
}
|
|
17
46
|
|
|
18
47
|
export interface PipelineTask {
|
|
@@ -21,6 +50,8 @@ export interface PipelineTask {
|
|
|
21
50
|
state: PipelineState
|
|
22
51
|
selection: PipelineSelection
|
|
23
52
|
model: PipelineModel
|
|
53
|
+
/** Captured at task creation; each source receives a separate time budget. */
|
|
54
|
+
extractionTimeoutMinutes?: ExtractionTimeoutMinutes
|
|
24
55
|
queuedAt: string
|
|
25
56
|
startedAt?: string
|
|
26
57
|
finishedAt?: string
|
|
@@ -31,9 +62,13 @@ export interface PipelineTask {
|
|
|
31
62
|
failed: number
|
|
32
63
|
summary: string
|
|
33
64
|
warnings: string[]
|
|
34
|
-
results?: Partial<Record<Exclude<PipelineStage, 'all'>, { done: number; total: number; reused: number; failed: number; summary?: string; skipped?: boolean }>>
|
|
65
|
+
results?: Partial<Record<Exclude<PipelineStage, 'all'>, { done: number; total: number; reused: number; failed: number; summary?: string; skipped?: boolean; blocked?: number }>>
|
|
66
|
+
recoverableOnRestart?: boolean
|
|
67
|
+
recoveryTaskId?: string
|
|
35
68
|
legacy?: boolean
|
|
36
69
|
reportId?: string
|
|
70
|
+
activity?: PipelineActivity
|
|
71
|
+
items?: Partial<Record<PipelineWorkStage, Record<string, PipelineItemState>>>
|
|
37
72
|
}
|
|
38
73
|
|
|
39
74
|
export interface PipelineEvent {
|
|
@@ -44,6 +79,7 @@ export interface PipelineEvent {
|
|
|
44
79
|
sourceId?: string
|
|
45
80
|
stage?: Exclude<PipelineStage, 'all'>
|
|
46
81
|
at: string
|
|
82
|
+
activity?: PipelineActivity
|
|
47
83
|
}
|
|
48
84
|
|
|
49
85
|
export interface MaterializationGroup {
|
|
@@ -74,6 +110,8 @@ export interface MaterializationBatch {
|
|
|
74
110
|
blocked: number
|
|
75
111
|
blockedAmbiguous: number
|
|
76
112
|
blockedAwaitingNormalize: number
|
|
113
|
+
/** Rows outside publishable groups still retain their source scope. */
|
|
114
|
+
blockedItems?: Array<{ knowledgeId: string; sourceIds: string[]; groupKey?: string }>
|
|
77
115
|
snapshot: unknown
|
|
78
116
|
}
|
|
79
117
|
|
|
@@ -87,7 +125,14 @@ export interface PipelineStatus {
|
|
|
87
125
|
indexPending: boolean
|
|
88
126
|
}
|
|
89
127
|
|
|
128
|
+
/**
|
|
129
|
+
* 五段流水线的唯一显示名。
|
|
130
|
+
*
|
|
131
|
+
* 2026-10-28 统一成「动词 + 对象」的短名:原来「归一三轴与记录」「Case 提炼 Procedure」
|
|
132
|
+
* 是两句不同句式的说明,混在同一条步骤轨上读起来像三套词。名短了,段页头才放得下
|
|
133
|
+
* 一句「它做什么」的说明;完整解释属于 `STAGE_DETAIL`,不属于名字。
|
|
134
|
+
*/
|
|
90
135
|
export const PIPELINE_STAGE_LABEL: Record<PipelineStage, string> = {
|
|
91
|
-
extract: '抽取 Knowledge', normalize: '
|
|
136
|
+
extract: '抽取 Knowledge', normalize: '归一坐标', derive: '提炼 Procedure', materialize: '物化 WikiCard', build: '构建索引', all: '全链路运行',
|
|
92
137
|
}
|
|
93
138
|
export const PIPELINE_ACTIVE_STATES: readonly PipelineState[] = ['queued', 'running', 'pausing', 'paused', 'cancelling']
|
package/src/present.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* 1. `presentationMeta` —— 结构化的小账本,随 `tool/result` 落进会话日志,
|
|
14
14
|
* Web 端的卡片读它渲染(`src/client/KnowledgeToolRow.tsx`)。它必须
|
|
15
15
|
* **可 JSON 序列化**(会话日志会整份校验),所以这里不塞函数、不塞 Date;
|
|
16
|
-
* 2. `present*View` ——
|
|
16
|
+
* 2. `present*View` —— 与工具名无关的渲染参数(标题 / 分类 / 一段 Markdown),
|
|
17
17
|
* 给终端界面和「没有专用卡片的界面」用。
|
|
18
18
|
*
|
|
19
19
|
* ## 一条规矩:数字必须来自真实的产物
|
package/src/preset-ids.ts
CHANGED
package/src/query-analysis.ts
CHANGED
|
@@ -2,8 +2,6 @@ import type { AxisEntity, AxisKind, AxisRelation } from './axis-registry.js'
|
|
|
2
2
|
import { normalizeAxisName } from './axis-registry.js'
|
|
3
3
|
import type { CardCoordinate } from './types.js'
|
|
4
4
|
|
|
5
|
-
export type QueryIntent = 'lookup' | 'explain' | 'procedure' | 'troubleshoot' | 'compare' | 'unknown'
|
|
6
|
-
|
|
7
5
|
export interface QueryAxisMatch {
|
|
8
6
|
id: string
|
|
9
7
|
kind: AxisKind
|
|
@@ -30,7 +28,6 @@ export interface QueryRelationCandidate extends QueryCoordinateCandidate {
|
|
|
30
28
|
|
|
31
29
|
export interface QueryAnalysis {
|
|
32
30
|
query: string
|
|
33
|
-
intent: { id: QueryIntent; confidence: number }
|
|
34
31
|
axes: Record<AxisKind, QueryAxisMatch[]>
|
|
35
32
|
coordinates: QueryCoordinateCandidate[]
|
|
36
33
|
unresolved: string[]
|
|
@@ -39,22 +36,6 @@ export interface QueryAnalysis {
|
|
|
39
36
|
type RegistryEntity = AxisEntity & { aliases: string[] }
|
|
40
37
|
type CoordinateCard = { cardKey: string; coordinate?: CardCoordinate }
|
|
41
38
|
|
|
42
|
-
const INTENT_RULES: Array<{ id: Exclude<QueryIntent, 'unknown'>; confidence: number; terms: string[] }> = [
|
|
43
|
-
{ id: 'troubleshoot', confidence: 0.9, terms: ['怎么排查', '如何排查', '异常', '报错', '失败', '原因', '为什么', '定位', '故障', '不工作'] },
|
|
44
|
-
{ id: 'procedure', confidence: 0.88, terms: ['怎么做', '如何做', '怎么', '如何', '步骤', '流程', '操作', '配置', '接入', '实现'] },
|
|
45
|
-
{ id: 'compare', confidence: 0.86, terms: ['区别', '差异', '对比', '比较', '哪个好', '不同点'] },
|
|
46
|
-
{ id: 'explain', confidence: 0.86, terms: ['什么是', '是什么', '定义', '含义', '解释', '指什么'] },
|
|
47
|
-
{ id: 'lookup', confidence: 0.78, terms: ['多少', '口径', '计算', '查询', '查看', '数据', '指标', '有哪些'] },
|
|
48
|
-
]
|
|
49
|
-
|
|
50
|
-
function classifyIntent(query: string): QueryAnalysis['intent'] {
|
|
51
|
-
const normalized = normalizeAxisName(query)
|
|
52
|
-
for (const rule of INTENT_RULES) {
|
|
53
|
-
if (rule.terms.some(term => normalized.includes(normalizeAxisName(term)))) return { id: rule.id, confidence: rule.confidence }
|
|
54
|
-
}
|
|
55
|
-
return { id: 'unknown', confidence: 0.35 }
|
|
56
|
-
}
|
|
57
|
-
|
|
58
39
|
function validMention(mention: string): boolean {
|
|
59
40
|
const normalized = normalizeAxisName(mention)
|
|
60
41
|
// 单字中文命中率很高,不能作为坐标证据;两个字符以上的中文或 ASCII 标识可以保留。
|
|
@@ -187,7 +168,6 @@ export function analyzeQuery(query: string, entities: RegistryEntity[], cards: C
|
|
|
187
168
|
.filter(kind => axes[kind].length === 0)
|
|
188
169
|
return {
|
|
189
170
|
query,
|
|
190
|
-
intent: classifyIntent(query),
|
|
191
171
|
axes,
|
|
192
172
|
coordinates: coordinateCandidates(axes, cards),
|
|
193
173
|
unresolved,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { ContextItem, Envelope, RecallHit, RecallResult } from './types.js'
|
|
2
|
+
|
|
3
|
+
const compactHit = (hit: RecallHit) => ({ unitId: hit.unitId, entityId: hit.entityId, entityType: hit.entityType, title: hit.title,
|
|
4
|
+
...(hit.heading ? { heading: hit.heading } : {}), score: hit.score,
|
|
5
|
+
...(hit.rerankScore === undefined ? {} : { rerankScore: hit.rerankScore }) })
|
|
6
|
+
|
|
7
|
+
/** Section references explain retrieval; the complete body is already in item.wiki. */
|
|
8
|
+
export const agentContextItem = (item: ContextItem) => ({ ...item,
|
|
9
|
+
...(item.note ? { note: compactHit(item.note) } : {}), notes: item.notes.map(compactHit),
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
/** Agent delivery carries each complete Wiki once, only inside the budgeted context. */
|
|
13
|
+
export function agentRecallEnvelope(value: Envelope<RecallResult>): Envelope<unknown> {
|
|
14
|
+
return { ...value, data: { ...value.data,
|
|
15
|
+
hits: value.data.hits.map(compactHit),
|
|
16
|
+
wiki: value.data.wiki.map(card => ({ entityId: card.entityId, entityType: card.entityType, title: card.title })),
|
|
17
|
+
context: value.data.context.map(agentContextItem),
|
|
18
|
+
} }
|
|
19
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** Omitted axes are unrestricted; an empty ID matches only an empty coordinate. */
|
|
2
|
+
export interface KnowledgeScope { objectId?: string; actionId?: string; sceneId?: string; wikiKind?: 'fact' | 'way' }
|
|
3
|
+
export function scopeError(scope: KnowledgeScope): string | undefined {
|
|
4
|
+
if ([scope.objectId, scope.actionId, scope.sceneId].some(value => value !== undefined && (typeof value !== 'string' || value !== value.trim()))) return '坐标 ID 必须是无首尾空格的字符串;空字符串仅匹配空坐标。'
|
|
5
|
+
if (scope.wikiKind !== undefined && !['fact', 'way'].includes(scope.wikiKind)) return 'wiki_kind 只能为 fact 或 way。'
|
|
6
|
+
}
|
|
7
|
+
export function matchesScope(value: { coordinate?: { objectId?: string; actionId?: string; sceneId?: string }; metadata?: Record<string, unknown> }, scope: KnowledgeScope): boolean {
|
|
8
|
+
return (['objectId', 'actionId', 'sceneId'] as const).every(axis => scope[axis] === undefined || (value.coordinate?.[axis] ?? '') === scope[axis])
|
|
9
|
+
&& (scope.wikiKind === undefined || value.metadata?.wiki_kind === scope.wikiKind)
|
|
10
|
+
}
|