dsh-data-domain-knowledge 0.1.0-alpha.1

package/src/convert/ingest.ts

@@ -0,0 +1,367 @@
+/**
+ * The knowledge-ingestion seam.
+ *
+ * Rules this file obeys, and why:
+ *
+ * 1. **One write entry, in-process.** Since 0018 the convert host lives inside
+ *    the knowledge plugin, so ingestion calls `KnowledgeRuntime.addSource()`
+ *    directly instead of `POST /knowledge/sources`. That removes a JSON body
+ *    cap (the old transport 400'd above ~1.2 MB) and removes port discovery,
+ *    startup ordering and error-code re-translation from the failure surface.
+ *    The structural `IngestTarget` interface is deliberately the only knowledge
+ *    API this module may touch — conversion code must not reach into the store.
+ * 2. **The kernel never writes the knowledge store.** It only produces
+ *    `convert-out/<taskId>/`. This module is the translation step.
+ * 3. **`source_type` stays the knowledge side's decision.** We propose
+ *    candidates; if the store rejects the value we surface its error verbatim
+ *    instead of silently defaulting.
+ * 4. **Images must land inside the knowledge root**, because retrieval only
+ *    reads local relative paths — and only the *first* image of a unit reaches
+ *    the multimodal embedder, so we also emit card hints that tell the Agent how
+ *    to reference them.
+ */
+
+import { cp, mkdir, stat } from 'node:fs/promises'
+import { basename, join, resolve, sep } from 'node:path'
+import type { ConvertService, IngestPayload } from 'dsh-convert-core'
+import type { LineageContext } from 'dsh-data-contracts/refs'
+import type { SourceType, SourceTypeDef } from '../types.js'
+import {
+  failure,
+  KNOWLEDGE_ASSET_DIR,
+  KNOWLEDGE_SOURCE_DIR,
+  KNOWLEDGE_WIKI_DIR,
+  success,
+  type Envelope,
+} from './types.js'
+
+/**
+ * The only slice of the knowledge store this seam is allowed to use. Declared
+ * structurally so this module never imports `runtime.ts`.
+ */
+export interface IngestTarget {
+  addSource(input: {
+    sourceType: SourceType
+    title?: string
+    content: string
+    externalId?: string
+    metadata?: Record<string, unknown>
+    sourceId?: string
+    lineage?: LineageContext
+  }): Promise<{
+    ok: boolean
+    data: { sourceId: string; path: string }
+    error?: { code?: string; message?: string; suggestion?: string } | null
+  }>
+  /**
+   * 该知识库的来源分类注册表（0013，**只读**）。
+   *
+   * 只用来把「候选分类」告诉调用方，转换模块不参与分类判定、也不改注册表，
+   * 所以它仍然是 knowledge 侧的能力，不算越界写。可选：老的测试替身可以不给。
+   */
+  sourceTypes?(): SourceTypeDef[]
+}
+
+export interface IngestOptions {
+  service: ConvertService
+  taskId: string
+  /** The live knowledge runtime; the write goes through `addSource` only. */
+  runtime: IngestTarget
+  /** Absolute knowledge root the runtime currently writes to. */
+  knowledgeRoot: string
+  /** Business classification. Omitted means "let the store decide/default". */
+  sourceType?: string
+  title?: string
+  metadata?: Record<string, unknown>
+  lineage?: Record<string, unknown>
+  /** Dry run: build the plan without copying assets or writing the source. */
+  planOnly?: boolean
+}
+
+export interface PlacedAsset {
+  rel: string
+  /** Absolute path inside the knowledge root. */
+  target: string
+  bytes: number
+}
+
+export interface IngestOutcome {
+  taskId: string
+  sourceId?: string
+  sourcePath?: string
+  knowledgeRoot: string
+  externalId: string
+  contentHash: string
+  sourceFormat: string
+  parser: string
+  sourceType: string
+  sourceTypeSupplied: boolean
+  sourceTypeCandidates: string[]
+  assets: PlacedAsset[]
+  /** Path to the source file relative to the knowledge root, e.g. `01_source_truth/x.md`. */
+  relativeSourcePath?: string
+  cardHints: CardHints
+  notes: string[]
+}
+
+export interface CardHints {
+  /** `entity_type` values accepted by the knowledge schema. */
+  entityTypes: string[]
+  sourceRefs: string[]
+  title: string
+  summary: string
+  /**
+   * Card body with image references already re-based to `02_wiki/<entityType>/`,
+   * so the first image actually reaches the multimodal embedder.
+   */
+  contentWithImages: string
+  /** The image references as they must appear inside a card body. */
+  imageRefs: string[]
+}
+
+const ENTITY_TYPES = ['concept', 'metric', 'table', 'event', 'playbook', 'case']
+
+/** Refuse any path that would escape the knowledge root. */
+export function safeJoin(root: string, ...parts: string[]): string | undefined {
+  const base = resolve(root)
+  const target = resolve(base, ...parts)
+  if (target !== base && !target.startsWith(base + sep)) return undefined
+  return target
+}
+
+function rewriteImageRefs(markdown: string, mapping: Map<string, string>): string {
+  let result = markdown
+  for (const [from, to] of mapping) {
+    result = result.split(`(${from})`).join(`(${to})`)
+  }
+  return result
+}
+
+function cardImageRef(hash: string, rel: string, entityType: string): string {
+  // A card lives at 02_wiki/<entityType>/<slug>.md; the asset lives at
+  // 01_source_truth/_assets/<hash>/<file>.
+  const depth = 2
+  const up = Array.from({ length: depth }, () => '..').join('/')
+  return `${up}/${KNOWLEDGE_SOURCE_DIR}/${KNOWLEDGE_ASSET_DIR}/${hash}/${basename(rel)}`
+}
+
+export function buildCardHints(input: {
+  payload: IngestPayload
+  sourceId: string
+  assetBucket: string
+  assetMapping: Map<string, string>
+  entityType?: string
+}): CardHints {
+  const entityType = input.entityType && ENTITY_TYPES.includes(input.entityType) ? input.entityType : 'concept'
+  const content = rewriteImageRefs(
+    input.payload.content,
+    new Map([...input.assetMapping].map(([from, to]) => [from, cardImageRef(input.assetBucket, to, entityType)])),
+  )
+  const imageRefs = input.payload.assets.map(asset => cardImageRef(input.assetBucket, asset.rel, entityType))
+  const summary = input.payload.content
+    .replace(/!\[[^\]]*\]\([^)]*\)/g, '')
+    .replace(/[#>*`_\-[\]()]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 160)
+  return {
+    entityTypes: ENTITY_TYPES,
+    sourceRefs: [input.sourceId],
+    title: input.payload.title,
+    summary,
+    contentWithImages: content,
+    imageRefs,
+  }
+}
+
+/**
+ * Turn a store rejection into an actionable suggestion without inventing one.
+ *
+ * 0013：分类候选来自**当前知识库的注册表**（含用户自定义分类），不再写死三个内置值。
+ */
+function suggestionFor(code: string, candidates: string[]): string | undefined {
+  if (code === 'SOURCE_TYPE_UNKNOWN') {
+    return `当前知识库的分类注册表里没有这个分类。可用分类：${candidates.join(' / ') || '（无）'}；要新增分类请到「知识库设置 → 来源类型」，并写清怎么判断它是这一类。`
+  }
+  if (code === 'SOURCE_TYPE_DISABLED') {
+    return `该分类已停用。可用分类：${candidates.join(' / ') || '（无）'}；要恢复请到「知识库设置 → 来源类型」重新启用。`
+  }
+  // 这里曾经有一条 `CONTENT_TOO_LARGE` 的建议（「调大 maxSourceBytes」）。0029 撤掉
+  // 那个上限之后，`addSource` 再也不会产出这个 code，留着它就成了一条指向不存在开关
+  // 的指路牌 —— 环境里已经没有任何东西会触发它。
+  if (code === 'SOURCE_TRUTH_DISABLED') {
+    return '在知识插件设置里打开「允许写入 SourceTruth」（allowSourceTruth）。'
+  }
+  return undefined
+}
+
+export async function ingestDelivery(options: IngestOptions): Promise<Envelope<IngestOutcome>> {
+  const { service, taskId } = options
+  let payload: IngestPayload
+  try {
+    payload = await service.ingestPlan(taskId)
+  } catch (cause) {
+    const error = cause as Error & { code?: string; suggestion?: string }
+    return failure(
+      emptyOutcome(taskId, options.knowledgeRoot, options.sourceType),
+      error.code ?? 'TASK_NOT_READY',
+      error.message,
+      false,
+      error.suggestion,
+    )
+  }
+
+  const bucket = payload.contentHash.slice(0, 12)
+  const assetRoot = safeJoin(options.knowledgeRoot, KNOWLEDGE_SOURCE_DIR, KNOWLEDGE_ASSET_DIR, bucket)
+  if (!assetRoot) {
+    return failure(emptyOutcome(taskId, options.knowledgeRoot, options.sourceType), 'PATH_OUTSIDE_KNOWLEDGE_ROOT', '资产目标路径越出知识库根目录。')
+  }
+
+  const assetMapping = new Map<string, string>()
+  const placed: PlacedAsset[] = []
+  const notes: string[] = []
+  const warnings: string[] = []
+
+  if (options.planOnly !== true && payload.assets.length > 0) {
+    await mkdir(assetRoot, { recursive: true })
+    for (const asset of payload.assets) {
+      const target = safeJoin(assetRoot, basename(asset.rel))
+      if (!target) {
+        warnings.push(`跳过越界资产：${asset.rel}`)
+        continue
+      }
+      try {
+        await cp(asset.path, target, { force: true })
+        const info = await stat(target)
+        placed.push({ rel: asset.rel, target, bytes: info.size })
+        assetMapping.set(asset.rel, asset.rel)
+      } catch (cause) {
+        warnings.push(`资产复制失败（${asset.rel}）：${cause instanceof Error ? cause.message : String(cause)}`)
+      }
+    }
+    if (placed.length !== payload.assets.length) {
+      notes.push(`${payload.assets.length - placed.length} 个图片资产未复制，正文中的对应引用会指向不存在的文件。`)
+    }
+  } else if (options.planOnly === true) {
+    for (const asset of payload.assets) assetMapping.set(asset.rel, asset.rel)
+    notes.push('planOnly=true：未复制资产、未写入 SourceRaw。')
+  }
+
+  const sourceContent = rewriteImageRefs(
+    payload.content,
+    new Map([...assetMapping].map(([from, to]) => [from, `${KNOWLEDGE_ASSET_DIR}/${bucket}/${basename(to)}`])),
+  )
+
+  const metadata: Record<string, unknown> = {
+    ...(options.metadata ?? {}),
+    source_format: payload.sourceFormat,
+    parser: payload.parser,
+    content_hash: payload.contentHash,
+    convert_task_id: taskId,
+    convert_contract_version: 1,
+    ...(payload.sourceUrl ? { source_url: payload.sourceUrl } : {}),
+    ...(payload.sourcePath ? { source_path: payload.sourcePath } : {}),
+    assets: placed.map(asset => `${KNOWLEDGE_ASSET_DIR}/${bucket}/${basename(asset.rel)}`),
+  }
+
+  const sourceType = options.sourceType?.trim() || 'document'
+  // 分类候选取自当前知识库的注册表（0013）：内置分类 + 用户自定义，只列启用的。
+  const sourceTypeCandidates = (options.runtime.sourceTypes?.() ?? [])
+    .filter((def: SourceTypeDef) => def.enabled)
+    .map((def: SourceTypeDef) => def.id)
+  payload.sourceTypeCandidates = sourceTypeCandidates
+  const title = options.title?.trim() || payload.title
+  const externalId = payload.externalId
+
+  if (!options.planOnly) {
+    const result = await options.runtime.addSource({
+      sourceType: sourceType as SourceType,
+      title,
+      content: sourceContent,
+      externalId,
+      metadata,
+      ...(options.lineage ? { lineage: options.lineage as LineageContext } : {}),
+    })
+
+    if (!result.ok) {
+      const code = result.error?.code ?? 'KNOWLEDGE_WRITE_FAILED'
+      return failure(
+        emptyOutcome(taskId, options.knowledgeRoot, options.sourceType),
+        code,
+        result.error?.message ?? '知识库拒绝了该来源',
+        false,
+        result.error?.suggestion ?? suggestionFor(code, sourceTypeCandidates),
+      )
+    }
+
+    const sourceId = result.data.sourceId
+    const sourcePath = result.data.path
+    const cardHints = buildCardHints({ payload, sourceId, assetBucket: bucket, assetMapping })
+    notes.push('SourceRaw 已写入；图片要进入多模态召回，还需要一张引用它们的 wiki 卡片（见 cardHints）。')
+
+    return success<IngestOutcome>(
+      {
+        taskId,
+        sourceId,
+        ...(sourcePath ? { sourcePath } : {}),
+        knowledgeRoot: options.knowledgeRoot,
+        externalId,
+        contentHash: payload.contentHash,
+        sourceFormat: payload.sourceFormat,
+        parser: payload.parser,
+        sourceType,
+        sourceTypeSupplied: Boolean(options.sourceType?.trim()),
+        sourceTypeCandidates: payload.sourceTypeCandidates,
+        assets: placed,
+        ...(sourcePath ? { relativeSourcePath: relativeTo(options.knowledgeRoot, sourcePath) } : {}),
+        cardHints,
+        notes,
+      },
+      warnings,
+    )
+  }
+
+  // Dry run: report what would happen, including the card hints.
+  const preview = buildCardHints({ payload, sourceId: '<add_source 返回的 sourceId>', assetBucket: bucket, assetMapping })
+  return success<IngestOutcome>(
+    {
+      taskId,
+      knowledgeRoot: options.knowledgeRoot,
+      externalId,
+      contentHash: payload.contentHash,
+      sourceFormat: payload.sourceFormat,
+      parser: payload.parser,
+      sourceType,
+      sourceTypeSupplied: Boolean(options.sourceType?.trim()),
+      sourceTypeCandidates: payload.sourceTypeCandidates,
+      assets: payload.assets.map(asset => ({ rel: asset.rel, target: join(assetRoot, basename(asset.rel)), bytes: asset.bytes })),
+      cardHints: preview,
+      notes: [...notes, `将写入 ${KNOWLEDGE_SOURCE_DIR}，资产放入 ${KNOWLEDGE_ASSET_DIR}/${bucket}/`],
+    },
+  )
+}
+
+function relativeTo(root: string, path: string): string {
+  const base = resolve(root)
+  const target = resolve(path)
+  return target.startsWith(base + sep) ? target.slice(base.length + 1) : target
+}
+
+function emptyOutcome(taskId: string, knowledgeRoot: string, sourceType: string | undefined): IngestOutcome {
+  return {
+    taskId,
+    knowledgeRoot,
+    externalId: '',
+    contentHash: '',
+    sourceFormat: '',
+    parser: '',
+    sourceType: sourceType?.trim() || 'document',
+    sourceTypeSupplied: Boolean(sourceType?.trim()),
+    sourceTypeCandidates: [],
+    assets: [],
+    cardHints: { entityTypes: ENTITY_TYPES, sourceRefs: [], title: '', summary: '', contentWithImages: '', imageRefs: [] },
+    notes: [],
+  }
+}
+
+export { ENTITY_TYPES, KNOWLEDGE_WIKI_DIR }

package/src/convert/settings-schema.ts

@@ -0,0 +1,27 @@
+/**
+ * 转换设置的 schema（纯数据，单独一个模块）。
+ *
+ * 与 `src/intent/settings-schema.ts` 同一个理由：0.1.7 之后它要被 `Config` 组合、
+ * 并被 `tests/settings-schema.test.mjs` 独立解析一遍，而 `host.ts` 会 `import`
+ * 工具层 —— 那条依赖在 `node --test` 里拉不起来（`dsh-tools` 的
+ * `@deepseek-ai/dsh-sandbox` 是未安装的 peer）。
+ *
+ * 0029 起这里**没有路径字段**：交付物与工作区都从知识插件的 `agentRoot` 推出来
+ * （`<agentRoot>/output/<日期>/`、`<agentRoot>/tmp/`）。一个 Agent 只有一个根路径，
+ * 所以这里再放一个 `rootDir` 就等于把「东西存在哪」这个答案劈成两半。
+ *
+ * @module
+ */
+
+import z from '@deepseek-ai/schemastery'
+import { volatileFields } from '../settings-volatile.js'
+import { DEFAULT_CONVERT_CONFIG, type ConvertPluginConfig } from './types.js'
+
+/** 显式标注是为了 TS2742（见 `intent/settings-schema.ts` 上的同一条注释）。 */
+export const CONVERT_SETTINGS_SCHEMA: Schemastery<ConvertPluginConfig> = z.object(volatileFields({
+  maxConcurrent: z.number().step(1).min(1).max(8).default(DEFAULT_CONVERT_CONFIG.maxConcurrent),
+  maxInputBytes: z.number().step(1).min(1024).default(DEFAULT_CONVERT_CONFIG.maxInputBytes),
+  retentionDays: z.number().step(1).max(365).default(DEFAULT_CONVERT_CONFIG.retentionDays),
+  toolWaitMs: z.number().step(1).min(1000).max(900000).default(DEFAULT_CONVERT_CONFIG.toolWaitMs),
+  ingestEnabled: z.boolean().default(DEFAULT_CONVERT_CONFIG.ingestEnabled),
+}))

package/src/convert/types.ts

@@ -0,0 +1,65 @@
+/**
+ * Plugin-level types, defaults and the envelope shape for the **convert seam**.
+ *
+ * These live inside `data-knowledge` since 0018: conversion is a sub-capability
+ * of the knowledge plugin, not a separately mounted plugin. The kernel itself
+ * (`dsh-convert-core`) stays an independent, optional package.
+ */
+
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+
+export const CONVERT_SETTINGS_NAMESPACE = 'tita-data-agent-convert'
+
+/** Knowledge store layout the convert seam cooperates with (issue 0013). */
+export const KNOWLEDGE_SOURCE_DIR = '01_source_truth'
+export const KNOWLEDGE_WIKI_DIR = '02_wiki'
+/** Sub-directory inside `01_source_truth` that holds converted images. */
+export const KNOWLEDGE_ASSET_DIR = '_assets'
+
+export interface ConvertPluginConfig {
+  maxConcurrent: number
+  maxInputBytes: number
+  /** Deliveries older than this are pruned on start; 0 disables. */
+  retentionDays: number
+  /** How long a tool call waits for the conversion before returning `running`. */
+  toolWaitMs: number
+  /** Allow `convert_ingest` to write into the knowledge store. */
+  ingestEnabled: boolean
+}
+
+/**
+ * @deprecated 0029：转换不再有自己的根目录。交付物落
+ * `<agentRoot>/output/<日期>/`、工作区落 `<agentRoot>/tmp/<日期>/`，
+ * 由 `layout.ts` 统一算出来。这个函数只留给**独立运行的内核**
+ * （`bin/tita-convert.mjs`）与它的单测，插件侧不再读它。
+ */
+export function defaultConvertRoot(): string {
+  if (process.env.DSH_CONVERT_ROOT) return process.env.DSH_CONVERT_ROOT
+  return join(homedir(), '.tita-data-agent', 'convert')
+}
+
+export const DEFAULT_CONVERT_CONFIG: ConvertPluginConfig = {
+  maxConcurrent: 2,
+  maxInputBytes: 256 * 1024 * 1024,
+  retentionDays: 14,
+  toolWaitMs: 120_000,
+  ingestEnabled: true,
+}
+
+export interface Envelope<T> {
+  ok: boolean
+  data: T
+  warnings?: string[]
+  error?: { code: string; message: string; suggestion?: string; retryable?: boolean }
+}
+
+export function success<T>(data: T, warnings?: string[]): Envelope<T> {
+  return { ok: true, data, ...(warnings && warnings.length > 0 ? { warnings } : {}) }
+}
+
+export function failure<T>(data: T, code: string, message: string, retryable = false, suggestion?: string): Envelope<T> {
+  return { ok: false, data, error: { code, message, retryable, ...(suggestion ? { suggestion } : {}) } }
+}
+
+export const WRITE_TOOLS = ['convert_source', 'convert_ingest'] as const