@p-dsh-market/conversation-knowledge-map 0.1.15 → 0.1.17

package/README.md

@@ -11,7 +11,7 @@ DSH 的“知识视图”插件，把同一工作路径下用户明确选择的
 
 配置面板会列出运行时可用的 Provider / Model，并默认带入 DSH 默认模型；本次选择会绑定到确认令牌、Agent 调用和 `manifest.json`，不会因为默认模型变化而被静默替换。提交任务后配置对话框立即关闭，失败原因会显示在知识视图页的任务条中。
 
-摘要阶段按对话并行处理，并发上限固定为 3；每个对话按约 5000 字、优先在完整段落边界分段，只有单段本身超长时才按句末或硬边界拆分。同一对话的分段保持顺序处理。思维导图使用合并后的会话摘要；知识图谱直接使用保留来源的分段摘要，避免长对话在二次压缩后丢失实体。知识图谱根据已读取文本量和有效分段数动态计算上限，范围为 30–100 个实体、最多 200 条关系，并要求逐分段检查覆盖范围。结构化输出失败时会完全重置并最多重试 3 次；单个对话连续失败后只跳过该对话，不阻断其他对话和最终报告。“同时生成”模式下，一个最终视图失败也不会阻断另一个视图保存。进度区域和最终知识视图都会按时间线显示读取、摘要、具体失败原因、重试、跳过、合并、生成及保存过程；知识图谱时间线还会显示模型返回数量和来源校验后的保留数量。模型可以保留 thinking，但结果提取只读取最终文本并过滤 reasoning 内容；摘要和最终视图分别预留 12000 与 24000 个输出 Token。最终模型若返回未选择的 Session 引用，会过滤该引用；严格模式下无有效来源的内容项会被跳过，最终页面列出实际总结的对话、失败对话和过滤数量。
+摘要阶段按对话并行处理，并发上限固定为 3；每个对话按约 5000 字、优先在完整段落边界分段，只有单段本身超长时才按句末或硬边界拆分。同一对话的分段保持顺序处理。思维导图使用合并后的会话摘要；知识图谱直接使用保留来源的分段摘要，避免长对话在二次压缩后丢失实体。知识图谱按证据分段多次生成，最多 2 批并行；单批限制在 10–18 个实体、最多 24 条关系，若达到输出 Token 上限则只降低并重试当前批次。所有成功批次由 Host 按实体类型与名称、关系起点/终点/类型确定性去重并合并来源，最终结果再根据已读取文本量和有效分段数裁剪为 30–100 个实体、最多 200 条关系。单个批次连续失败只跳过该批，不会推翻其他成功批次。结构化输出失败时会完全重置并最多重试 3 次；单个对话摘要连续失败后只跳过该对话，不阻断其他对话和最终报告。“同时生成”模式下，一个最终视图失败也不会阻断另一个视图保存。进度区域和最终知识视图都会按时间线显示读取、摘要、具体失败原因、重试、跳过、合并、生成及保存过程；知识图谱时间线还会显示批次数、模型返回数量和合并去重后的保留数量。模型可以保留 thinking，但结果提取只读取最终文本并过滤 reasoning 内容；摘要和每次图谱调用分别预留 12000 与 24000 个输出 Token。最终模型若返回未选择的 Session 引用，会过滤该引用；严格模式下无有效来源的内容项会被跳过，最终页面列出实际总结的对话、失败对话和过滤数量。
 
 生成过程时间线支持折叠和展开：进行中的任务默认展开，已完成结果中的历史时间线默认折叠。思维导图使用递归树形布局并绘制父子连接线。知识图谱按关联度使用中心节点与内外双环布局，节点卡片内显示类型和最多两行名称，关系以带方向箭头的弱化曲线呈现，并通过描边区分推测或冲突内容；画布提供 50%–200% 缩放和一键恢复 100%，缩放不会影响实体筛选、点击和详情查看，也不会产生横向滚动条。
 
@@ -19,7 +19,7 @@ DSH 的“知识视图”插件，把同一工作路径下用户明确选择的
 
 ## 生成失败诊断
 
-Host 和生成编排器会输出带 `[conversation-knowledge-map]` 前缀的诊断日志。日志包含路由、Provider / Model、Agent Session、实时事件类型、surface/session 读取次数、提取文本长度、输出形状和错误原因，不记录 Prompt、对话正文或模型返回正文。优先查看 DSH Web Runtime 的终端日志；若 Runtime 提供 logger 服务，则同时写入该 logger。重点关注 `agent event`、`agent idle`、`agent turn failure`、`agent surface read`、`agent session read` 和 `agent output parse failed`。Runtime 在 `turn/end.reason.kind = error` 时会优先显示其 `code/message`；只有未发现 turn 错误且确实没有助手输出时，才会报告“模型没有返回 JSON 对象”。
+Host 和生成编排器会输出带 `[conversation-knowledge-map]` 前缀的诊断日志。日志包含路由、Provider / Model、Agent Session、实时事件类型、surface/session 读取次数、提取文本长度、输出形状和错误原因，不记录 Prompt 或原始对话正文。为诊断结构化输出失败，当前还会记录模型实际返回文本：不超过 32000 字符时完整记录，超过时保留首尾各 16000 字符；该内容可能包含模型整理出的对话信息，排障完成后应按运行环境的日志策略清理。解析失败日志会额外列出 Markdown JSON 围栏、首末大括号位置以及每个候选对象属于未闭合、JSON 语法错误、合法但结构不符或合法目标结构。优先查看 DSH Web Runtime 的终端日志；若 Runtime 提供 logger 服务，则同时写入该 logger。重点关注 `agent output content`、`agent output parse failed`、`agent event`、`agent idle`、`agent turn failure`、`agent surface read` 和 `agent session read`。Runtime 在 `turn/end.reason.kind = error` 时会优先显示其 `code/message`；只有未发现 turn 错误且确实没有助手输出时，才会报告“模型没有返回 JSON 对象”。
 
 ## 本地验证
 

package/lib/generation-orchestrator.js

@@ -10,6 +10,9 @@ const SUMMARY_MAX_TOKENS = 12000
 const VIEW_MAX_TOKENS = 24000
 const SUMMARY_CONCURRENCY = 3
 const MAX_MODEL_RETRIES = 3
+const GRAPH_CONCURRENCY = 2
+const GRAPH_BATCH_MAX_ENTITIES = 18
+const GRAPH_BATCH_MAX_RELATIONS = 24
 const MIN_GRAPH_ENTITIES = 30
 const MAX_GRAPH_ENTITIES = 100
 const MAX_GRAPH_RELATIONS = 200
@@ -125,6 +128,73 @@ function findJsonObject(text, kind = '') {
   return null
 }
 
+function modelOutputText(value) {
+  const text = textFromContent(value)
+  if (text) return text
+  if (value && typeof value === 'object') {
+    try { return JSON.stringify(value) } catch { /* fall through to a printable scalar */ }
+  }
+  return String(value ?? '')
+}
+
+function loggableModelOutput(value, max = 32000) {
+  const text = modelOutputText(value)
+  if (text.length <= max) return { text, loggedText: JSON.stringify(text), truncated: false }
+  const half = Math.floor(max / 2)
+  const bounded = `${text.slice(0, half)}\n…[中间 ${text.length - max} 个字符因日志长度限制被省略]…\n${text.slice(-half)}`
+  return { text, loggedText: JSON.stringify(bounded), truncated: true }
+}
+
+function jsonParseDiagnostics(value, kind = '') {
+  const text = modelOutputText(value).replace(/^\uFEFF/, '').trim()
+  const starts = []
+  for (let index = 0; index < text.length && starts.length < 12; index += 1) if (text[index] === '{') starts.push(index)
+  const candidates = []
+  for (const start of starts) {
+    let depth = 0
+    let inString = false
+    let escaped = false
+    let end = -1
+    for (let index = start; index < text.length; index += 1) {
+      const char = text[index]
+      if (inString) {
+        if (escaped) escaped = false
+        else if (char === '\\') escaped = true
+        else if (char === '"') inString = false
+        continue
+      }
+      if (char === '"') inString = true
+      else if (char === '{') depth += 1
+      else if (char === '}' && --depth === 0) { end = index; break }
+    }
+    if (end < 0) {
+      candidates.push({ start, status: 'unclosed-object' })
+      continue
+    }
+    const fragment = text.slice(start, end + 1)
+    try {
+      const parsed = JSON.parse(fragment)
+      candidates.push({
+        start,
+        end,
+        status: expectedObject(parsed, kind) ? 'valid-expected-shape' : 'valid-json-wrong-shape',
+        keys: parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? Object.keys(parsed).slice(0, 20) : []
+      })
+    } catch (error) {
+      candidates.push({ start, end, status: 'invalid-json', error: shortText(errorMessage(error), 300) })
+    }
+  }
+  return {
+    kind,
+    textLength: text.length,
+    fencedJson: /```(?:json)?\s*[\s\S]*?```/i.test(text),
+    firstBrace: text.indexOf('{'),
+    lastBrace: text.lastIndexOf('}'),
+    candidateCountInspected: candidates.length,
+    candidates
+  }
+}
+
 export function parseStructuredOutput(value, kind = '') {
   const object = asObject(value)
   if (object && expectedObject(object, kind)) return object
@@ -205,6 +275,104 @@ function graphGenerationBudget(sources, context) {
   }
 }
 
+function graphBatchBudget(context, attempt = 0) {
+  const contextChars = JSON.stringify(context || []).length
+  const reduction = Math.max(0.5, 1 - attempt * 0.2)
+  const entities = Math.max(8, Math.floor(Math.min(GRAPH_BATCH_MAX_ENTITIES, Math.max(10, Math.ceil(contextChars / 500))) * reduction))
+  return {
+    entities,
+    relations: Math.max(entities, Math.floor(Math.min(GRAPH_BATCH_MAX_RELATIONS, Math.ceil(entities * 1.35)) * reduction))
+  }
+}
+
+function mergeRefs(...groups) {
+  const bySession = new Map()
+  for (const ref of groups.flat()) {
+    const sessionId = String(ref?.sessionId || '')
+    if (!sessionId) continue
+    const seqs = bySession.get(sessionId) || new Set()
+    for (const seq of ref?.eventSeqs || []) if (Number.isInteger(seq) && seq >= 0) seqs.add(seq)
+    bySession.set(sessionId, seqs)
+  }
+  return [...bySession.entries()].slice(0, 12).map(([sessionId, seqs]) => ({ sessionId, eventSeqs: [...seqs].slice(0, 48) }))
+}
+
+function mergedConfidence(left, right) {
+  const values = new Set([left, right])
+  if (values.has('conflicted')) return 'conflicted'
+  if (values.has('inferred')) return 'inferred'
+  return 'confirmed'
+}
+
+function entityMergeKey(entity) {
+  const normalize = (value) => String(value || '').normalize('NFKC').trim().toLowerCase().replace(/\s+/g, ' ')
+  return `${normalize(entity?.type || 'concept')}\u0000${normalize(entity?.name)}`
+}
+
+function mergeKnowledgeGraphs(graphs, budget) {
+  const entitiesByKey = new Map()
+  const idMaps = []
+  let nextEntityId = 1
+  const usedEntityIds = new Set()
+  graphs.forEach((graph, graphIndex) => {
+    const idMap = new Map()
+    for (const entity of graph.entities || []) {
+      const key = entityMergeKey(entity)
+      let merged = entitiesByKey.get(key)
+      if (!merged) {
+        const requestedId = String(entity.id || '').trim()
+        let id = requestedId && !usedEntityIds.has(requestedId) ? requestedId : `entity-${nextEntityId++}`
+        while (usedEntityIds.has(id)) id = `entity-${nextEntityId++}`
+        usedEntityIds.add(id)
+        merged = { ...entity, id, sourceRefs: mergeRefs(entity.sourceRefs), order: entitiesByKey.size }
+        entitiesByKey.set(key, merged)
+      } else {
+        merged.sourceRefs = mergeRefs(merged.sourceRefs, entity.sourceRefs)
+        merged.confidence = mergedConfidence(merged.confidence, entity.confidence)
+        if (String(entity.summary || '').length > String(merged.summary || '').length) merged.summary = entity.summary
+      }
+      idMap.set(String(entity.id), merged.id)
+    }
+    idMaps[graphIndex] = idMap
+  })
+
+  const relationsByKey = new Map()
+  graphs.forEach((graph, graphIndex) => {
+    const idMap = idMaps[graphIndex]
+    for (const relation of graph.relations || []) {
+      const from = idMap.get(String(relation.from))
+      const to = idMap.get(String(relation.to))
+      if (!from || !to || from === to) continue
+      const type = String(relation.type || 'related_to').trim()
+      const key = `${from}\u0000${to}\u0000${type.toLowerCase()}`
+      const existing = relationsByKey.get(key)
+      if (existing) {
+        existing.evidence = mergeRefs(existing.evidence, relation.evidence)
+        existing.confidence = mergedConfidence(existing.confidence, relation.confidence)
+      } else {
+        relationsByKey.set(key, { ...relation, from, to, type, evidence: mergeRefs(relation.evidence), order: relationsByKey.size })
+      }
+    }
+  })
+
+  const degree = new Map()
+  for (const relation of relationsByKey.values()) {
+    degree.set(relation.from, (degree.get(relation.from) || 0) + 1)
+    degree.set(relation.to, (degree.get(relation.to) || 0) + 1)
+  }
+  const entities = [...entitiesByKey.values()]
+    .sort((left, right) => (degree.get(right.id) || 0) - (degree.get(left.id) || 0) || right.sourceRefs.length - left.sourceRefs.length || left.order - right.order)
+    .slice(0, budget.entities)
+    .map(({ order, ...entity }) => entity)
+  const retainedIds = new Set(entities.map((entity) => entity.id))
+  const relations = [...relationsByKey.values()]
+    .filter((relation) => retainedIds.has(relation.from) && retainedIds.has(relation.to))
+    .sort((left, right) => right.evidence.length - left.evidence.length || left.order - right.order)
+    .slice(0, budget.relations)
+    .map(({ order, ...relation }, index) => ({ ...relation, id: `relation-${index + 1}` }))
+  return { entities, relations }
+}
+
 async function mapWithConcurrency(items, limit, mapper) {
   const results = new Array(items.length)
   let nextIndex = 0
@@ -771,40 +939,53 @@ export class KnowledgeGenerationOrchestrator {
         }
       }
       if (request.outputMode === 'knowledge-graph' || request.outputMode === 'both') {
-        this.recordTimeline(task, 'view-start', '开始生成知识图谱。')
-        this.update(task, 'building-knowledge-graph', { progress: { percent: 86, current: sources.length, total: sources.length, label: '生成知识图谱' } })
-        const baseGraphPrompt = outputPrompt('knowledge-graph', graphContext, request.prompt, request.strict === true, { graphBudget })
-        let graphError = null
-        for (let attempt = 0; attempt <= MAX_MODEL_RETRIES; attempt += 1) {
-          try {
-            const prompt = attempt === 0 ? baseGraphPrompt : `${baseGraphPrompt}\n\n第 ${attempt} 次输出未通过校验：${shortText(errorMessage(graphError), 500)}。请完全重置并从头生成，修正结构或来源错误，同时保持对各分段的完整覆盖；只有确实超过本次上限时才精简。只输出完整、严格合法的 JSON。`
-            const value = await this.runModel({
-              kind: 'knowledge-graph', prompt, cwd: request.cwd, strict: request.strict === true,
-              selectedSessionIds: sourceSessionIds, model: request.model, signal: task.controller.signal
-            })
-            const sanitized = sanitizeKnowledgeGraphSources(value, summarizedSources, request.strict === true)
-            knowledgeGraph = validateKnowledgeGraph(sanitized.value, { selectedSessionIds: sourceSessionIds, strict: request.strict === true })
-            this.recordTimeline(task, 'graph-coverage', `知识图谱模型返回 ${Array.isArray(value?.entities) ? value.entities.length : 0} 个实体、${Array.isArray(value?.relations) ? value.relations.length : 0} 条关系；来源校验后保留 ${knowledgeGraph.entities.length} 个实体、${knowledgeGraph.relations.length} 条关系。`, { entityLimit: graphBudget.entities, relationLimit: graphBudget.relations })
-            sourceWarnings.skippedRefs += sanitized.skippedRefs
-            sourceWarnings.skippedItems += sanitized.skippedItems
-            graphError = null
-            this.recordTimeline(task, 'view-complete', '知识图谱生成完成。')
-            break
-          } catch (error) {
-            this.assertNotCancelled(task)
-            graphError = error
-            logMessage(this.logger, 'warn', 'knowledge graph attempt invalid id=%s attempt=%d error=%s', logId(task.id), attempt + 1, errorMessage(error))
-            if (attempt < MAX_MODEL_RETRIES) {
-              this.recordTimeline(task, 'retry', `知识图谱生成失败（${shortText(errorMessage(graphError), 160)}），重置重试 ${attempt + 1}/${MAX_MODEL_RETRIES}。`)
-              this.update(task, 'building-knowledge-graph', {
-                message: `知识图谱结构无效，正在重置重试 ${attempt + 1}/${MAX_MODEL_RETRIES}…`,
-                progress: { percent: 88, current: sources.length, total: sources.length, label: `知识图谱重试 ${attempt + 1}/${MAX_MODEL_RETRIES}` }
+        const graphBatches = graphContext.map((item) => [item])
+        this.recordTimeline(task, 'view-start', `开始分 ${graphBatches.length} 批生成知识图谱（最多 ${GRAPH_CONCURRENCY} 批并行）。`)
+        this.update(task, 'building-knowledge-graph', { progress: { percent: 86, current: 0, total: graphBatches.length, label: '分批生成知识图谱' } })
+        let completedGraphBatches = 0
+        const graphBatchResults = await mapWithConcurrency(graphBatches, GRAPH_CONCURRENCY, async (batch, batchIndex) => {
+          let graphError = null
+          for (let attempt = 0; attempt <= MAX_MODEL_RETRIES; attempt += 1) {
+            try {
+              const batchBudget = graphBatchBudget(batch, attempt)
+              const baseGraphPrompt = outputPrompt('knowledge-graph', batch, request.prompt, request.strict === true, { graphBudget: batchBudget })
+              const prompt = attempt === 0 ? baseGraphPrompt : `${baseGraphPrompt}\n\n本批第 ${attempt} 次输出未通过校验：${shortText(errorMessage(graphError), 500)}。请完全重置并从头生成；本次预算已降低，优先保留有明确关系和来源的实体。只输出完整、严格合法的 JSON。`
+              const value = await this.runModel({
+                kind: 'knowledge-graph', prompt, cwd: request.cwd, strict: request.strict === true,
+                selectedSessionIds: sourceSessionIds, model: request.model, signal: task.controller.signal
               })
+              const sanitized = sanitizeKnowledgeGraphSources(value, summarizedSources, request.strict === true)
+              const graph = validateKnowledgeGraph(sanitized.value, { selectedSessionIds: sourceSessionIds, strict: request.strict === true })
+              sourceWarnings.skippedRefs += sanitized.skippedRefs
+              sourceWarnings.skippedItems += sanitized.skippedItems
+              completedGraphBatches += 1
+              this.recordTimeline(task, 'graph-batch-complete', `知识图谱第 ${batchIndex + 1}/${graphBatches.length} 批完成：保留 ${graph.entities.length} 个实体、${graph.relations.length} 条关系。`, { batch: batchIndex + 1, totalBatches: graphBatches.length })
+              this.update(task, 'building-knowledge-graph', { progress: { percent: 86 + Math.floor(4 * completedGraphBatches / graphBatches.length), current: completedGraphBatches, total: graphBatches.length, label: `知识图谱批次 ${completedGraphBatches}/${graphBatches.length}` } })
+              return { graph, rawEntities: Array.isArray(value?.entities) ? value.entities.length : 0, rawRelations: Array.isArray(value?.relations) ? value.relations.length : 0, error: null }
+            } catch (error) {
+              this.assertNotCancelled(task)
+              graphError = error
+              logMessage(this.logger, 'warn', 'knowledge graph batch invalid id=%s batch=%d attempt=%d error=%s', logId(task.id), batchIndex + 1, attempt + 1, errorMessage(error))
+              if (attempt < MAX_MODEL_RETRIES) {
+                this.recordTimeline(task, 'retry', `知识图谱第 ${batchIndex + 1}/${graphBatches.length} 批失败（${shortText(errorMessage(graphError), 160)}），降低本批输出预算后重试 ${attempt + 1}/${MAX_MODEL_RETRIES}。`)
+              }
             }
           }
-        }
-        if (graphError) {
-          this.recordTimeline(task, 'view-failed', `知识图谱连续重试 ${MAX_MODEL_RETRIES} 次后仍失败，已跳过该视图。`)
+          completedGraphBatches += 1
+          this.recordTimeline(task, 'graph-batch-skipped', `知识图谱第 ${batchIndex + 1}/${graphBatches.length} 批连续失败，已跳过本批。`)
+          return { graph: null, rawEntities: 0, rawRelations: 0, error: graphError }
+        })
+        const successfulGraphBatches = graphBatchResults.filter((result) => result.graph)
+        if (successfulGraphBatches.length) {
+          const mergedGraph = mergeKnowledgeGraphs(successfulGraphBatches.map((result) => result.graph), graphBudget)
+          knowledgeGraph = validateKnowledgeGraph(mergedGraph, { selectedSessionIds: sourceSessionIds, strict: request.strict === true })
+          const rawEntities = successfulGraphBatches.reduce((total, result) => total + result.rawEntities, 0)
+          const rawRelations = successfulGraphBatches.reduce((total, result) => total + result.rawRelations, 0)
+          this.recordTimeline(task, 'graph-coverage', `知识图谱 ${successfulGraphBatches.length}/${graphBatches.length} 批成功，模型共返回 ${rawEntities} 个实体、${rawRelations} 条关系；合并去重后保留 ${knowledgeGraph.entities.length} 个实体、${knowledgeGraph.relations.length} 条关系。`, { entityLimit: graphBudget.entities, relationLimit: graphBudget.relations })
+          this.recordTimeline(task, 'view-complete', '知识图谱分批生成与合并完成。')
+        } else {
+          const graphError = graphBatchResults.find((result) => result.error)?.error || new Error('所有知识图谱批次均生成失败。')
+          this.recordTimeline(task, 'view-failed', `知识图谱 ${graphBatches.length} 个批次均失败，已跳过该视图。`)
           if (request.outputMode === 'knowledge-graph') throw graphError
         }
       }
@@ -862,13 +1043,15 @@ export class KnowledgeGenerationOrchestrator {
 
   async runModel(input) {
     const parseModelOutput = (value, source, diagnostics = {}) => {
+      const output = loggableModelOutput(value)
+      logMessage(this.logger, 'info', 'agent output content kind=%s source=%s length=%d truncated=%s content=%s', input.kind, source, output.text.length, output.truncated, output.loggedText)
       try {
         const result = parseStructuredOutput(value, input.kind)
         logMessage(this.logger, 'info', 'agent output parsed kind=%s source=%s value=%s result=%s', input.kind, source, diagnosticSummary(value), diagnosticSummary(result))
         return result
       } catch (error) {
         const surfacedError = diagnostics.agentLimit || error
-        logMessage(this.logger, 'error', 'agent output parse failed kind=%s source=%s value=%s diagnostics=%s error=%s', input.kind, source, diagnosticSummary(value), JSON.stringify(diagnostics), errorMessage(surfacedError))
+        logMessage(this.logger, 'error', 'agent output parse failed kind=%s source=%s value=%s parseDiagnostics=%s runtimeDiagnostics=%s error=%s', input.kind, source, diagnosticSummary(value), JSON.stringify(jsonParseDiagnostics(value, input.kind)), JSON.stringify(diagnostics), errorMessage(surfacedError))
         throw surfacedError
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@p-dsh-market/conversation-knowledge-map",
-  "version": "0.1.15",
+  "version": "0.1.17",
   "description": "DSH 多对话思维导图与静态知识图谱",
   "keywords": [
     "dsh",