dsh-data-domain-knowledge 0.1.0-alpha.9 → 0.1.1-alpha.2

package/src/pipeline/engine.ts

@@ -0,0 +1,323 @@
+import { randomUUID } from 'node:crypto'
+import type { ModelCall } from '../extract-model.js'
+import type { SourceRaw, WikiTypeDef } from '../types.js'
+import type { TrialReport } from '../wiki-trial-types.js'
+import { PipelineStore, type ExtractionCheckpoint } from './store.js'
+import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_LABEL, type MaterializationBatch, type MaterializationGroup, type PipelineAction, type PipelineModel, type PipelineSelection, type PipelineStage, type PipelineTask } from './types.js'
+
+export interface PipelineSource extends SourceRaw { version: number; sourceHash: string }
+export interface PipelineContext {
+  model: PipelineModel & { call?: ModelCall; why?: string }
+  note: string
+  profile?: WikiTypeDef
+  organize: boolean
+  format: 'json' | 'table'
+}
+export interface PipelineBuildResult { summary: string; skipped?: boolean }
+export interface PipelineHost {
+  capture(): PipelineContext
+  serialize<T>(work: () => Promise<T>): Promise<T>
+  sources(): Promise<PipelineSource[]>
+  /** One source by id. Same shape as an element of `sources()`. */
+  source(sourceId: string): Promise<PipelineSource | undefined>
+  fingerprint(source: PipelineSource, context: PipelineContext): Promise<string>
+  extract(source: PipelineSource, context: PipelineContext, output: (text: string) => void): Promise<TrialReport>
+  normalize(source: PipelineSource, checkpoint: ExtractionCheckpoint, context: PipelineContext, force: boolean, output: (text: string) => void): Promise<{ reused: boolean; warnings: string[] }>
+  derive?(sourceIds: string[] | undefined, context: PipelineContext, force: boolean, output: (text: string) => void, checkpoint: () => Promise<void>): Promise<{ done: number; total: number; reused: number; summary: string }>
+  groups(context: PipelineContext): Promise<MaterializationBatch>
+  materialize(group: MaterializationGroup, context: PipelineContext, output: (text: string) => void, snapshot: unknown): Promise<void>
+  build(taskId: string, checkpoint: () => Promise<void>): Promise<PipelineBuildResult>
+  record(checkpoint: ExtractionCheckpoint): Promise<void>
+}
+
+class TaskCancelled extends Error {}
+interface Control { pause: boolean; cancel: boolean; resume?: () => void }
+
+/** One writer queue per base, shared with the legacy index builder. */
+export class KnowledgePipeline {
+  private closed = false
+  private controls = new Map<string, Control>()
+  constructor(readonly store: PipelineStore, private readonly host: PipelineHost) {}
+
+  start(stage: PipelineStage, selection: PipelineSelection = {}): { task: PipelineTask; completion: Promise<PipelineTask> } {
+    if (this.closed) throw new Error('知识处理已关闭')
+    if (!['extract', 'normalize', 'derive', 'materialize', 'build', 'all'].includes(stage)) throw new Error('不支持的知识处理阶段')
+    for (const value of [selection.sourceIds, selection.groupKeys]) {
+      if (value !== undefined && (!Array.isArray(value) || value.length === 0 || value.some(id => typeof id !== 'string' || !id.trim()))) throw new Error('选择范围不能为空；不传范围表示处理当前阶段的全部待办')
+    }
+    const context = this.host.capture()
+    const { call: _call, why: _why, ...model } = context.model
+    const task: PipelineTask = {
+      id: `pipeline_${randomUUID()}`, stage, state: 'queued', selection: structuredClone(selection), model,
+      queuedAt: new Date().toISOString(), done: 0, total: 0, reused: 0, failed: 0,
+      summary: '等待运行', warnings: [],
+    }
+    this.store.putTask(task)
+    this.controls.set(task.id, { pause: false, cancel: false })
+    const completion = this.host.serialize(() => this.execute(task.id, context))
+    // Starting a background task must not create an unhandled rejection.
+    void completion.catch(() => undefined)
+    return { task, completion }
+  }
+
+  close(): void {
+    this.closed = true
+    for (const control of this.controls.values()) { control.cancel = true; control.pause = false; control.resume?.() }
+  }
+
+  control(id: string, action: PipelineAction): PipelineTask {
+    const task = this.store.task(id)
+    const control = this.controls.get(id)
+    if (!task || !control || !PIPELINE_ACTIVE_STATES.includes(task.state)) throw new Error('任务已结束；请重新运行所需阶段')
+    if (action === 'pause') {
+      control.pause = true
+      this.patch(id, { state: task.state === 'queued' ? 'paused' : 'pausing', summary: '当前文档或卡片处理完成后暂停' })
+    } else if (action === 'resume') {
+      if (!control.pause) throw new Error('任务没有暂停')
+      control.pause = false
+      this.patch(id, { state: task.startedAt ? 'running' : 'queued' })
+      control.resume?.()
+      control.resume = undefined
+    } else {
+      control.cancel = true
+      control.pause = false
+      this.patch(id, { state: 'cancelling', summary: '当前文档或卡片处理完成后停止' })
+      control.resume?.()
+    }
+    return this.store.task(id)!
+  }
+
+  private patch(id: string, patch: Partial<PipelineTask>): PipelineTask {
+    const task = { ...this.store.task(id)!, ...patch }
+    this.store.putTask(task)
+    return task
+  }
+  private finishStage(id: string, outcome?: Pick<PipelineBuildResult, 'summary' | 'skipped'>): void {
+    const task = this.store.task(id)!
+    if (task.currentStage) this.patch(id, { results: { ...task.results, [task.currentStage]: { done: task.done, total: task.total, reused: task.reused, failed: task.failed, ...outcome } } })
+  }
+  private async checkpoint(id: string): Promise<void> {
+    const control = this.controls.get(id)
+    while (control) {
+      if (control.cancel) throw new TaskCancelled('任务已停止，已完成的文档与阶段会保留')
+      if (!control.pause) return
+      this.patch(id, { state: 'paused', summary: '已暂停，可继续运行' })
+      await new Promise<void>(resolve => { control.resume = resolve })
+    }
+  }
+  private output(id: string, stage: Exclude<PipelineStage, 'all'>, sourceId?: string): { append: (text: string) => void; flush: () => void } {
+    let buffer = ''
+    let last = Date.now()
+    let bytes = 0
+    let capped = false
+    const flush = (): void => {
+      if (!buffer) return
+      this.store.event(id, 'output', buffer, sourceId, stage)
+      buffer = ''
+      last = Date.now()
+    }
+    let failure: unknown
+    const timer = setInterval(() => { try { flush() } catch (cause) { failure = cause; clearInterval(timer) } }, 250)
+    timer.unref()
+    return {
+      append: text => {
+        if (failure) throw failure
+        bytes += Buffer.byteLength(text)
+        if (bytes > 4 * 1024 * 1024) {
+          if (!capped) { flush(); this.store.event(id, 'warning', '实时预览超过 4 MB；完整返回保存在本次抽取报告中。', sourceId, stage); capped = true }
+          return
+        }
+        buffer += text
+        if (buffer.length >= 1024 || Date.now() - last >= 250) flush()
+      }, flush: () => { clearInterval(timer); if (failure) throw failure; flush() },
+    }
+  }
+  private async execute(id: string, context: PipelineContext): Promise<PipelineTask> {
+    const task = this.store.task(id)!
+    const stages: Exclude<PipelineStage, 'all'>[] = task.stage === 'all' ? (this.host.derive ? ['extract', 'normalize', 'derive', 'materialize', 'build'] : ['extract', 'normalize', 'materialize', 'build']) : [task.stage]
+    try {
+      await this.checkpoint(id)
+      this.patch(id, { state: 'running', startedAt: new Date().toISOString() })
+      const allSources = await this.host.sources()
+      const sourceIds = task.selection.sourceIds === undefined ? undefined : new Set(task.selection.sourceIds)
+      if (sourceIds && [...sourceIds].some(sourceId => !allSources.some(source => source.sourceId === sourceId))) throw new Error('选择范围中有已删除或不存在的来源')
+      const sources = allSources.filter(source => !sourceIds || sourceIds.has(source.sourceId))
+      /**
+       * 坐标仍有歧义、等归一的 Knowledge 条数。它不是失败，所以既不进 `failed`、
+       * 也不提前结束链路（否则一个歧义 mention 就能让整库索引永不发布）；
+       * 但它也不是「全部完成」，所以最终状态报 `partial`。
+       */
+      let deferred = 0
+      for (const stage of stages) {
+        await this.checkpoint(id)
+        this.patch(id, { currentStage: stage, done: 0, total: 0, reused: 0, failed: 0, summary: PIPELINE_STAGE_LABEL[stage] })
+        this.store.event(id, 'progress', PIPELINE_STAGE_LABEL[stage], undefined, stage)
+        if (stage === 'derive') {
+          if (!this.host.derive) throw new Error('当前运行时不支持 Case 提炼')
+          const output = this.output(id, stage)
+          try {
+            const result = await this.host.derive(task.selection.sourceIds, context, Boolean(task.selection.force), output.append, () => this.checkpoint(id))
+            this.patch(id, result)
+            this.finishStage(id, result)
+          } finally { output.flush() }
+          continue
+        }
+        if (stage === 'build') {
+          const result = await this.host.build(id, () => this.checkpoint(id))
+          this.patch(id, { done: 1, total: 1, summary: result.summary })
+          this.finishStage(id, result)
+          continue
+        }
+        if (stage === 'materialize') {
+          const batch = await this.host.groups(context)
+          const { groups } = batch
+          const selected = task.selection.groupKeys ? new Set(task.selection.groupKeys) : undefined
+          if (selected && [...selected].some(key => !groups.some(group => group.key === key))) throw new Error('选择的卡片分组已变化，请刷新待物化列表')
+          /**
+           * 来源范围对**任何**阶段都生效，不只对 `all`：`start()` 会校验 `sourceIds`
+           * 非空，调用方因此有理由认为它生效了。只对 `all` 生效等于静默忽略选择。
+           */
+          const affectedSources = sourceIds ? new Set(sources.map(source => source.sourceId)) : undefined
+          const scoped = selected !== undefined || affectedSources !== undefined
+          const inScope = (group: MaterializationGroup): boolean => (!selected || selected.has(group.key)) && (!affectedSources || group.sourceIds.some(sourceId => affectedSources.has(sourceId)))
+          /**
+           * 待归一的量要**按选择范围**数，且两个原因分开说（0070）。
+           *
+           * 「坐标有歧义」与「来源变了、抽取结果还没归一」是两件事，报同一句话会让读的人
+           * 去查错的地方。更要紧的是范围：按 `sourceIds` / `groupKeys` 缩小的那一趟，
+           * 如果拿全库的数字报 `partial`，调用方会去重试一件它明确排除的事 —— 而它其实
+           * 全都做完了。坐标有歧义的那些行不属于任何分组，无法按来源归属，所以只在
+           * 没有选择范围时才计入。
+           */
+          const ambiguous = scoped ? 0 : batch.blockedAmbiguous
+          const awaiting = scoped
+            ? groups.filter(group => group.state === 'blocked' && inScope(group)).reduce((sum, group) => sum + group.knowledgeCount, 0)
+            : batch.blockedAwaitingNormalize
+          if (ambiguous > 0 || awaiting > 0) {
+            deferred = ambiguous + awaiting
+            const parts = [
+              ambiguous > 0 ? `${ambiguous} 条 Knowledge 因坐标歧义、方法不完整或 Case 引用过期暂不可发布` : '',
+              awaiting > 0 ? `${awaiting} 条 Knowledge 的来源已变化或新抽取待归一` : '',
+            ].filter(Boolean)
+            const message = `${parts.join('；')}，暂不物化；本次不影响索引构建。`
+            this.patch(id, { warnings: [...this.store.task(id)!.warnings, message] })
+            this.store.event(id, 'warning', message, undefined, stage)
+          }
+          const pending = groups.filter(group => group.state !== 'blocked' && inScope(group) && (task.selection.force || group.state !== 'current'))
+          this.patch(id, { total: pending.length })
+          for (const group of pending) {
+            await this.checkpoint(id)
+            const output = this.output(id, stage)
+            try { await this.host.materialize(group, context, output.append, batch.snapshot) }
+            catch (cause) {
+              if (cause instanceof TaskCancelled) throw cause
+              const message = `${group.title}：${cause instanceof Error ? cause.message : String(cause)}`
+              const current = this.store.task(id)!
+              this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
+              this.store.event(id, 'warning', message, undefined, stage)
+            }
+            finally { output.flush() }
+            const current = this.store.task(id)!
+            this.patch(id, { done: current.done + 1, summary: `已物化 ${current.done + 1}/${pending.length} 张：${group.title}` })
+          }
+          const current = this.store.task(id)!
+          /**
+           * 阶段结论要给 `summary`：任务列表的四段步轨读的是 `results[stage]`，
+           * 缺了它那一段会显示成「已完成」（成功色），而任务徽标写着「部分完成」——
+           * 同一件事在半个屏幕上两种说法。
+           */
+          this.finishStage(id, {
+            summary: `已物化 ${current.done - current.failed}/${current.total} 张${deferred > 0 ? `；${deferred} 条待归一` : ''}${affectedSources !== undefined ? `（按来源范围缩小：${affectedSources.size} 份来源）` : ''}`,
+          })
+          /**
+           * **真的**有分组失败时才提前结束（含 `build`）：这里与 `blocked` 的处理刻意不同。
+           *
+           * `blocked` 是「等归一」的待办，不是失败，所以它不阻断；而这里是某一张卡没写成功
+           * （模型报错、写库失败），此时发布索引等于把一个半成品当成品推给召回 ——
+           * 而重试的代价很小：已经物化好的那些分组指纹没变，下一次会被判成 `current` 直接跳过。
+           */
+          if (current.failed) return this.patch(id, { state: current.failed === current.total ? 'failed' : 'partial', finishedAt: new Date().toISOString(), summary: `物化结束：成功 ${current.total - current.failed}/${current.total}；已完成卡片保留，可重试。` })
+          continue
+        }
+        this.patch(id, { total: sources.length })
+        for (const source of sources) {
+          await this.checkpoint(id)
+          this.store.event(id, 'progress', `${PIPELINE_STAGE_LABEL[stage]}：${source.title}`, source.sourceId, stage)
+          const output = this.output(id, stage, source.sourceId)
+          let reused = false
+          try {
+            if (stage === 'extract') {
+              const fingerprint = await this.host.fingerprint(source, context)
+              const cached = task.selection.force ? undefined : this.store.cached(source.sourceId, source.version, fingerprint)
+              if (cached) { this.store.selectExtraction(source.sourceId, cached.id); reused = true }
+              else {
+                const report = await this.host.extract(source, context, output.append)
+                if (this.closed) throw new TaskCancelled('知识库已关闭，已完成结果保留')
+                const checkpoint: ExtractionCheckpoint = { id: `extract_${randomUUID()}`, sourceId: source.sourceId, sourceVersion: source.version, sourceHash: source.sourceHash, fingerprint, report, createdAt: new Date().toISOString() }
+                // The host checks the captured source version again before activation.
+                // Read that one source back, not the whole library: the old form re-hashed
+                // every document body once per document, which is O(N²) over a big base.
+                const current = await this.host.source(source.sourceId)
+                if (!current || current.version !== source.version || current.sourceHash !== source.sourceHash) {
+                  await this.host.record(checkpoint)
+                  this.store.event(id, 'report', checkpoint.id, source.sourceId, stage)
+                  throw new Error('来源在抽取期间已变化；返回已保留在历史中，请抽取最新版本')
+                }
+                await this.host.record(checkpoint)
+                this.store.event(id, 'report', checkpoint.id, source.sourceId, stage)
+                /**
+                 * 没通过校验时**要说清是哪一条不合格**（0070）。
+                 *
+                 * 原来只报一句 `model.why`（「抽取返回未通过校验」），"哪些引文对不上" 只留在
+                 * 审计报告里，而人看不到那一层。现在把 `exceptions` 的前三条带进错误里 ——
+                 * 面板上的失败行直接能读出原因。
+                 *
+                 * 这里**不缓存**这一份失败返回（试过，退回去了）：缓存会让下一次重跑不再调模型，
+                 * 而同一个阶段结束时的承诺是「可重试此阶段」——「重试」按下去什么都不发生，
+                 * 比多花一次模型调用糟得多。完整返回已经由 `host.record` 收进审计库。
+                 */
+                if (report.model.parsed !== true) {
+                  const detail = report.exceptions.slice(0, 3).map(item => `${item.step}：${item.message}`).join('；')
+                  throw new Error(`${report.model.why ?? '抽取返回未通过校验'}${detail === '' ? '' : `（${detail}）`}`)
+                }
+                this.store.saveExtraction(checkpoint)
+              }
+            } else {
+              const checkpoint = this.store.head(source.sourceId).extracted
+              if (!checkpoint) throw new Error('尚无抽取结果，请先运行抽取')
+              if (checkpoint.sourceVersion !== source.version || checkpoint.sourceHash !== source.sourceHash) throw new Error('抽取结果依据旧版来源，请先抽取最新版本')
+              const result = await this.host.normalize(source, checkpoint, context, task.selection.force === true, output.append)
+              reused = result.reused
+              if (result.warnings.length) this.patch(id, { warnings: [...this.store.task(id)!.warnings, ...result.warnings.map(warning => `${source.title}：${warning}`)] })
+              for (const warning of result.warnings) this.store.event(id, 'warning', warning, source.sourceId, stage)
+            }
+          } catch (cause) {
+            if (cause instanceof TaskCancelled) throw cause
+            const message = `${source.title}：${cause instanceof Error ? cause.message : String(cause)}`
+            const current = this.store.task(id)!
+            this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
+            this.store.event(id, 'warning', message, source.sourceId, stage)
+          } finally { output.flush() }
+          const current = this.store.task(id)!
+          this.patch(id, { done: current.done + 1, reused: current.reused + Number(reused), summary: `${PIPELINE_STAGE_LABEL[stage]} ${current.done + 1}/${sources.length}${reused ? ' · 复用已有结果' : ''}` })
+        }
+        this.finishStage(id)
+        if (this.store.task(id)!.failed > 0) {
+          const current = this.store.task(id)!
+          this.patch(id, { state: current.failed === current.total ? 'failed' : 'partial', finishedAt: new Date().toISOString(), summary: `${PIPELINE_STAGE_LABEL[stage]}结束：成功 ${current.total - current.failed}/${current.total}，失败 ${current.failed}；已完成结果保留，可重试此阶段。` })
+          return this.store.task(id)!
+        }
+      }
+      await this.checkpoint(id)
+      const current = this.store.task(id)!
+      /**
+       * `deferred > 0` 时状态是 `partial`：这一趟把能做的都做了（包括发布索引），
+       * 但还有一批分组在等归一。报 `succeeded` 会让人以为「物化完了」。
+       */
+      const suffix = deferred > 0 ? ` · 另有 ${deferred} 条 Knowledge 待处理（坐标、来源、方法完整性或 Case 版本），暂未物化` : ''
+      return this.patch(id, { state: deferred > 0 ? 'partial' : 'succeeded', finishedAt: new Date().toISOString(), summary: (task.stage === 'all' ? `全链路完成 · 抽取 ${current.results?.extract?.done ?? 0} 份（复用 ${current.results?.extract?.reused ?? 0} 份） · 物化 ${current.results?.materialize?.done ?? 0} 张 · 索引已发布` : `${PIPELINE_STAGE_LABEL[task.stage]}完成 · 处理 ${current.done} 项 · 复用 ${current.reused} 项`) + suffix })
+    } catch (cause) {
+      return this.patch(id, { state: cause instanceof TaskCancelled ? 'cancelled' : 'failed', finishedAt: new Date().toISOString(), summary: cause instanceof Error ? cause.message : String(cause) })
+    } finally { this.controls.delete(id) }
+  }
+}

package/src/pipeline/store.ts

@@ -0,0 +1,147 @@
+import type { SqliteDatabase } from '../fts.js'
+import { applyRetention } from '../retention.js'
+import type { TrialReport } from '../wiki-trial-types.js'
+import type { PipelineEvent, PipelineTask } from './types.js'
+
+export interface ExtractionCheckpoint {
+  id: string
+  sourceId: string
+  sourceVersion: number
+  sourceHash: string
+  fingerprint: string
+  report: TrialReport
+  createdAt: string
+  normalized?: { report: TrialReport; knowledgeIds: Record<string, string>; eventIds: Record<string, string>; fingerprint: string; unresolvedKnowledgeIds?: string[] }
+}
+
+/** Business checkpoints live with truth; execution history lives with extraction audits. */
+export class PipelineStore {
+  constructor(readonly truth: SqliteDatabase, private readonly audit: SqliteDatabase) {
+    truth.exec(`
+      CREATE TABLE IF NOT EXISTS pipeline_extractions (
+        id TEXT PRIMARY KEY, source_id TEXT NOT NULL, source_version INTEGER NOT NULL,
+        source_hash TEXT NOT NULL, fingerprint TEXT NOT NULL, report TEXT NOT NULL,
+        normalized TEXT, created_at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS pipeline_extract_cache ON pipeline_extractions(source_id, source_version, fingerprint);
+      CREATE TABLE IF NOT EXISTS pipeline_sources (source_id TEXT PRIMARY KEY, extracted_id TEXT, normalized_id TEXT);
+      CREATE TABLE IF NOT EXISTS pipeline_materializations (group_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, card_id TEXT NOT NULL);
+    `)
+    audit.exec(`
+      CREATE TABLE IF NOT EXISTS pipeline_tasks (id TEXT PRIMARY KEY, queued_at TEXT NOT NULL, state TEXT NOT NULL, payload TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS pipeline_tasks_time ON pipeline_tasks(queued_at DESC);
+      CREATE TABLE IF NOT EXISTS pipeline_events (
+        seq INTEGER PRIMARY KEY AUTOINCREMENT, task_id TEXT NOT NULL, kind TEXT NOT NULL,
+        text TEXT NOT NULL, source_id TEXT, at TEXT NOT NULL, stage TEXT
+      );
+      CREATE INDEX IF NOT EXISTS pipeline_events_task ON pipeline_events(task_id, seq);
+    `)
+    const eventColumns = audit.prepare('PRAGMA table_info(pipeline_events)').all() as Array<{ name: string }>
+    if (!eventColumns.some(column => column.name === 'stage')) audit.exec('ALTER TABLE pipeline_events ADD COLUMN stage TEXT')
+    // A new runtime never claims that an abandoned request is still running.
+    for (const task of (audit.prepare("SELECT payload FROM pipeline_tasks WHERE state IN ('queued','running','pausing','paused','cancelling')").all() as Array<{ payload: string }>).map(row => JSON.parse(row.payload) as PipelineTask)) {
+      if (['queued', 'running', 'pausing', 'paused', 'cancelling'].includes(task.state)) {
+        this.putTask({ ...task, state: 'interrupted', finishedAt: new Date().toISOString(), summary: '服务已重新启动；已完成的阶段保留，可重新运行此任务。' })
+      }
+    }
+  }
+
+  putTask(task: PipelineTask): void {
+    this.audit.prepare('INSERT INTO pipeline_tasks(id, queued_at, state, payload) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET state=excluded.state, payload=excluded.payload')
+      .run(task.id, task.queuedAt, task.state, JSON.stringify(task))
+    this.pruneAudit()
+  }
+
+  /**
+   * 审计库的保留上限（0070）。
+   *
+   * 实时预览每 250ms 或每 1KB 落一行，一次五分钟的抽取就能加一千行，而这里原来一条
+   * `DELETE` 都没有。上限是「水位以上保留」：升级那一刻已有的任务与事件一行不删
+   * （见 `retention.ts`）。任务的保留上限同时也是事件的：先按任务裁，再把已经不存在的
+   * 那些任务的事件（同样只在水位以上）删掉，最后兜一层总量。
+   */
+  private pruneAudit(): void {
+    applyRetention(this.audit, 'pipeline_tasks', { table: 'pipeline_tasks', key: 'rowid', keep: 200 })
+    applyRetention(this.audit, 'pipeline_events', { table: 'pipeline_events', key: 'seq', keep: 50_000, where: 'task_id NOT IN (SELECT id FROM pipeline_tasks)' })
+    applyRetention(this.audit, 'pipeline_events', { table: 'pipeline_events', key: 'seq', keep: 50_000 })
+  }
+  task(id: string): PipelineTask | undefined {
+    const row = this.audit.prepare('SELECT payload FROM pipeline_tasks WHERE id=?').get(id) as { payload: string } | undefined
+    return row ? JSON.parse(row.payload) as PipelineTask : undefined
+  }
+  tasks(limit = 20, offset = 0): { tasks: PipelineTask[]; total: number } {
+    const rows = this.audit.prepare("SELECT payload FROM pipeline_tasks ORDER BY CASE WHEN state IN ('queued','running','pausing','paused','cancelling') THEN 0 ELSE 1 END, queued_at DESC LIMIT ? OFFSET ?")
+      .all(Math.max(1, Math.min(10000, limit)), Math.max(0, offset)) as Array<{ payload: string }>
+    const total = Number((this.audit.prepare('SELECT COUNT(*) AS n FROM pipeline_tasks').get() as { n: number }).n)
+    return { tasks: rows.map(row => JSON.parse(row.payload) as PipelineTask), total }
+  }
+  event(taskId: string, kind: PipelineEvent['kind'], text: string, sourceId?: string, stage?: PipelineEvent['stage']): void {
+    this.audit.prepare('INSERT INTO pipeline_events(task_id,kind,text,source_id,at,stage) VALUES (?,?,?,?,?,?)')
+      .run(taskId, kind, text, sourceId ?? null, new Date().toISOString(), stage ?? null)
+  }
+  events(taskId: string, after = 0, limit = 100): { events: PipelineEvent[]; next: number; hasMore: boolean } {
+    const take = Number.isFinite(limit) ? Math.max(1, Math.min(500, Math.trunc(limit))) : 100
+    after = Number.isFinite(after) ? Math.max(0, Math.trunc(after)) : 0
+    const rows = this.audit.prepare('SELECT * FROM pipeline_events WHERE task_id=? AND seq>? ORDER BY seq LIMIT ?')
+      .all(taskId, after, take + 1) as Array<Record<string, unknown>>
+    const hasMore = rows.length > take
+    const events = rows.slice(0, take).map(row => ({ seq: Number(row.seq), taskId: String(row.task_id), kind: String(row.kind) as PipelineEvent['kind'], text: String(row.text), ...(row.source_id ? { sourceId: String(row.source_id) } : {}), ...(row.stage ? { stage: String(row.stage) as PipelineEvent['stage'] } : {}), at: String(row.at) }))
+    return { events, next: events.at(-1)?.seq ?? after, hasMore }
+  }
+  checkpoint(id: string): ExtractionCheckpoint | undefined {
+    const row = this.truth.prepare('SELECT * FROM pipeline_extractions WHERE id=?').get(id) as Record<string, unknown> | undefined
+    return row ? { id: String(row.id), sourceId: String(row.source_id), sourceVersion: Number(row.source_version), sourceHash: String(row.source_hash), fingerprint: String(row.fingerprint), report: JSON.parse(String(row.report)), createdAt: String(row.created_at), ...(row.normalized ? { normalized: JSON.parse(String(row.normalized)) } : {}) } : undefined
+  }
+  cached(sourceId: string, version: number, fingerprint: string): ExtractionCheckpoint | undefined {
+    const row = this.truth.prepare('SELECT id FROM pipeline_extractions WHERE source_id=? AND source_version=? AND fingerprint=? ORDER BY created_at DESC, rowid DESC LIMIT 1')
+      .get(sourceId, version, fingerprint) as { id: string } | undefined
+    return row ? this.checkpoint(row.id) : undefined
+  }
+  head(sourceId: string): { extracted?: ExtractionCheckpoint; normalized?: ExtractionCheckpoint } {
+    const row = this.truth.prepare('SELECT extracted_id,normalized_id FROM pipeline_sources WHERE source_id=?').get(sourceId) as { extracted_id?: string; normalized_id?: string } | undefined
+    return { extracted: row?.extracted_id ? this.checkpoint(row.extracted_id) : undefined, normalized: row?.normalized_id ? this.checkpoint(row.normalized_id) : undefined }
+  }
+  saveExtraction(value: ExtractionCheckpoint): void {
+    this.truth.exec('SAVEPOINT pipeline_extract')
+    try {
+    this.truth.prepare('INSERT INTO pipeline_extractions(id,source_id,source_version,source_hash,fingerprint,report,created_at) VALUES (?,?,?,?,?,?,?)')
+      .run(value.id, value.sourceId, value.sourceVersion, value.sourceHash, value.fingerprint, JSON.stringify(value.report), value.createdAt)
+    this.selectExtraction(value.sourceId, value.id)
+    this.truth.exec('RELEASE pipeline_extract')
+    } catch (cause) { this.truth.exec('ROLLBACK TO pipeline_extract; RELEASE pipeline_extract'); throw cause }
+  }
+  selectExtraction(sourceId: string, id: string): void {
+    this.truth.prepare('INSERT INTO pipeline_sources(source_id,extracted_id) VALUES (?,?) ON CONFLICT(source_id) DO UPDATE SET extracted_id=excluded.extracted_id').run(sourceId, id)
+  }
+  /**
+   * 刷新一份抽取检查点的指纹（0070）。
+   *
+   * 指纹里含「本文提到的实体清单」，而归一会往注册表里学新实体 —— 它们本来就在这份
+   * 文档里，所以学完之后指纹必然变。不刷新的话，同一份未改动的文档下次抽取一定复用不上
+   * （每次都要再调一次模型）。归一成功后把这一行改成学完之后的值，复用才成立。
+   */
+  refreshFingerprint(id: string, fingerprint: string): void {
+    this.truth.prepare('UPDATE pipeline_extractions SET fingerprint=? WHERE id=?').run(fingerprint, id)
+  }
+  saveNormalization(value: ExtractionCheckpoint, normalized: NonNullable<ExtractionCheckpoint['normalized']>): void {
+    this.truth.prepare('UPDATE pipeline_extractions SET normalized=? WHERE id=?').run(JSON.stringify(normalized), value.id)
+    this.truth.prepare('INSERT INTO pipeline_sources(source_id,normalized_id) VALUES (?,?) ON CONFLICT(source_id) DO UPDATE SET normalized_id=excluded.normalized_id').run(value.sourceId, value.id)
+  }
+  unresolvedKnowledge(): { ids: Set<string>; sources: Set<string> } {
+    const rows = this.truth.prepare('SELECT s.source_id, e.normalized FROM pipeline_sources s JOIN pipeline_extractions e ON e.id=s.normalized_id WHERE e.normalized IS NOT NULL').all() as Array<{ source_id: string; normalized: string }>
+    const ids = new Set<string>(); const sources = new Set<string>()
+    for (const row of rows) {
+      const pending = (JSON.parse(row.normalized) as NonNullable<ExtractionCheckpoint['normalized']>).unresolvedKnowledgeIds ?? []
+      if (pending.length) sources.add(row.source_id)
+      for (const id of pending) ids.add(id)
+    }
+    return { ids, sources }
+  }
+  materialization(key: string): { fingerprint: string; cardId: string } | undefined {
+    const row = this.truth.prepare('SELECT fingerprint,card_id FROM pipeline_materializations WHERE group_key=?').get(key) as { fingerprint: string; card_id: string } | undefined
+    return row ? { fingerprint: row.fingerprint, cardId: row.card_id } : undefined
+  }
+  materialized(key: string, fingerprint: string, cardId: string): void {
+    this.truth.prepare('INSERT INTO pipeline_materializations(group_key,fingerprint,card_id) VALUES (?,?,?) ON CONFLICT(group_key) DO UPDATE SET fingerprint=excluded.fingerprint,card_id=excluded.card_id').run(key, fingerprint, cardId)
+  }
+}

package/src/pipeline/types.ts

@@ -0,0 +1,93 @@
+/** Shared, browser-safe contracts for independently runnable knowledge stages. */
+export type PipelineStage = 'extract' | 'normalize' | 'derive' | 'materialize' | 'build' | 'all'
+export type PipelineState = 'queued' | 'running' | 'pausing' | 'paused' | 'cancelling' | 'succeeded' | 'partial' | 'failed' | 'cancelled' | 'interrupted'
+export type PipelineAction = 'pause' | 'resume' | 'cancel'
+
+export interface PipelineSelection {
+  sourceIds?: string[]
+  groupKeys?: string[]
+  force?: boolean
+}
+
+export interface PipelineModel {
+  provider?: string
+  model?: string
+  reasoningEffort?: string
+}
+
+export interface PipelineTask {
+  id: string
+  stage: PipelineStage
+  state: PipelineState
+  selection: PipelineSelection
+  model: PipelineModel
+  queuedAt: string
+  startedAt?: string
+  finishedAt?: string
+  currentStage?: Exclude<PipelineStage, 'all'>
+  done: number
+  total: number
+  reused: number
+  failed: number
+  summary: string
+  warnings: string[]
+  results?: Partial<Record<Exclude<PipelineStage, 'all'>, { done: number; total: number; reused: number; failed: number; summary?: string; skipped?: boolean }>>
+  legacy?: boolean
+  reportId?: string
+}
+
+export interface PipelineEvent {
+  seq: number
+  taskId: string
+  kind: 'output' | 'progress' | 'warning' | 'report'
+  text: string
+  sourceId?: string
+  stage?: Exclude<PipelineStage, 'all'>
+  at: string
+}
+
+export interface MaterializationGroup {
+  wikiKind?: 'fact' | 'way'
+  key: string
+  title: string
+  coordinate: { objectId: string; actionId: string; sceneId: string }
+  knowledgeCount: number
+  eventCount: number
+  sourceIds: string[]
+  cardId?: string
+  state: 'new' | 'changed' | 'current' | 'withdrawn' | 'blocked'
+  reason?: string
+  fingerprint: string
+}
+
+/**
+ * 物化阶段的批（宿主产生、宿主消费）。
+ *
+ * `snapshot` 对引擎是**不透明的**：它的字段类型来自服务端模块（Knowledge / EventEntity /
+ * 图片 / 相关推荐），而那些模块都依赖 `node:*`。本模块被客户端 import 取值，所以这里
+ * 不能描述它 —— 真正的形状在 `materialize-snapshot.ts`，只有服务端看得见。
+ * 引擎只负责把 `snapshot` 原样传回 `materialize()`。
+ */
+export interface MaterializationBatch {
+  groups: MaterializationGroup[]
+  /** 坐标仍有歧义 + 等待归一，两部分的合计。 */
+  blocked: number
+  blockedAmbiguous: number
+  blockedAwaitingNormalize: number
+  snapshot: unknown
+}
+
+export interface PipelineStatus {
+  sources: Array<{ id: string; title: string; version: number; extractionId?: string; normalizedId?: string; state: 'pending' | 'extracted' | 'normalized' | 'outdated' }>
+  groups: MaterializationGroup[]
+  blocked: number
+  pendingNew: number
+  pendingUpdated: number
+  pendingWithdrawn: number
+  indexPending: boolean
+}
+
+export const PIPELINE_STAGE_LABEL: Record<PipelineStage, string> = {
+  extract: '抽取 Knowledge', normalize: '归一三轴与记录', derive: 'Case 提炼 Procedure', materialize: '组织 Wiki', build: '构建检索索引', all: '全链路运行',
+}
+export const PIPELINE_ACTIVE_STATES: readonly PipelineState[] = ['queued', 'running', 'pausing', 'paused', 'cancelling']

package/src/present.ts

@@ -54,6 +54,7 @@ function short(text: string, max = 48): string {
 export interface ImportMetaSource { sourceId: string; title?: string; bytes: number }
 export interface ImportMetaFailure { index: number; title?: string; code: string; message: string }
 export interface ImportMetaCompile {
+  taskId?: string; state?: string
   cardsCreated: number; cardsUpdated: number; cardsReused?: number
   notesReused?: number; notesRecompiled?: number
   sourcesExtracted?: number; sourcesSkipped?: number; cardsRemoved?: number
@@ -101,7 +102,7 @@ export function importCallView(args: { sources?: unknown[]; compile?: boolean })
   return {
     card: 'generic',
     kind: 'edit',
-    title: `导入 ${count} 份来源${args?.compile === false ? '' : '并编译'}`,
+    title: `导入 ${count} 份来源${args?.compile === true ? '并处理知识' : ''}`,
     rawInput: titles.length > 0 ? titles.map(title => short(title)) : { 份数: count },
   }
 }
@@ -112,10 +113,15 @@ export function importResultMarkdown(meta: ImportMeta): string {
   for (const item of meta.written) lines.push(`- ${item.title ? `**${short(item.title)}**` : item.sourceId} · ${item.bytes} 字符`)
   for (const item of meta.failed) lines.push(`- 第 ${item.index + 1} 份${item.title ? `「${short(item.title)}」` : ''}失败：${item.code} ${item.message}`)
   if (meta.compile) {
-    const { cardsCreated, cardsUpdated, cardsReused, notesReused, notesRecompiled, embedding, version } = meta.compile
+    const { cardsCreated, cardsUpdated, cardsReused, notesReused, notesRecompiled, embedding, version, state, taskId } = meta.compile
+    if (state) {
+      const label = state === 'succeeded' ? '已完成' : state === 'partial' ? '部分完成' : state === 'cancelled' ? '已停止' : '未完成'
+      lines.push('', `知识处理${label}：新建 ${cardsCreated} 张、更新 ${cardsUpdated} 张。${taskId ? `任务：${taskId}。` : ''}详情请打开「知识处理」页的运行记录。`)
+      return lines.join('\n')
+    }
     lines.push('', `编译：新建 ${cardsCreated} 张、并入 ${cardsUpdated} 张、复用 ${cardsReused ?? 0} 张；召回片段复用 ${notesReused ?? 0} 条、重编 ${notesRecompiled ?? 0} 条；索引 v${version ?? '?'}${embedding ? `（向量 ${embedding}）` : ''}。`)
   } else {
-    lines.push('', '这次没有编译（`compile: false`）—— 卡片还没生成，需要时再调 `rebuild_knowledge`。')
+    lines.push('', '这次只保存来源（默认 `compile: false`）。下一步使用 `run_knowledge_pipeline` 分别抽取、归一、从 Case 提炼 Procedure、组织 Wiki，最后构建检索索引。')
   }
   return lines.join('\n')
 }
@@ -314,7 +320,7 @@ export function addSourceCallView(args: { title?: string; content?: string }): {
 }
 
 export function addSourceResultMarkdown(meta: AddSourceMeta): string {
-  return `**来源已写入** —— 「${meta.title ? short(meta.title) : meta.sourceId}」（${meta.bytes} 字符）。\n\n下一步：\`rebuild_knowledge\` 把它编译成 WikiCard。`
+  return `**来源已写入** —— 「${meta.title ? short(meta.title) : meta.sourceId}」（${meta.bytes} 字符）。\n\n下一步：\`run_knowledge_pipeline\` 分别抽取、归一、从 Case 提炼 Procedure、组织 Wiki，最后构建检索索引。`
 }
 
 export interface WriteCardsMeta {

package/src/preset-ids.ts

@@ -0,0 +1,6 @@
+/** Preset ids persisted in DSH session headers. */
+export const STANDARD_PRESET_ID = 'standard'
+
+export function isIntentEnabledPreset(presetId: unknown): boolean {
+  return presetId === STANDARD_PRESET_ID
+}