dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.14

package/src/knowledge.ts

@@ -96,6 +96,8 @@ export interface KnowledgeSource {
   currentVersion?: number
   evidenceStatus?: 'current' | 'historical' | 'unversioned' | 'missing' | 'human'
   quoteInCurrentVersion?: boolean
+  /** A completed newer extraction withdrew this document contribution. */
+  withdrawn?: boolean
   createdAt: string
 }
 
@@ -227,6 +229,7 @@ export interface KnowledgeVersion {
  * 「`object_id` 这个参数不在」与「`object_id=` 空值」。
  */
 export interface KnowledgeQuery {
+  excludeWithdrawn?: boolean
   objectId?: string
   actionId?: string
   sceneId?: string
@@ -537,6 +540,7 @@ export class KnowledgeStore {
           throw cause
         }
       }
+      if (!hasColumn(this.db, 'knowledge_sources', 'withdrawn')) this.db.exec('ALTER TABLE knowledge_sources ADD COLUMN withdrawn INTEGER NOT NULL DEFAULT 0')
       if (!hasColumn(this.db, 'knowledge', 'head_version')) {
         this.db.exec("ALTER TABLE knowledge ADD COLUMN head_version INTEGER NOT NULL DEFAULT 0")
       }
@@ -637,7 +641,7 @@ export class KnowledgeStore {
     if (existing !== null && normalizeKnowledge(existing.text) !== normalized) throw new Error('KNOWLEDGE_ID_COLLISION')
 
     if (existing === null) {
-      this.db.exec('BEGIN')
+      this.db.exec('SAVEPOINT knowledge_ingest')
       try {
         this.db.prepare(
           'INSERT INTO knowledge (id, text, normalized, object_id, action_id, scene_id, status, run_id, merged_into, head_version, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
@@ -654,10 +658,10 @@ export class KnowledgeStore {
           at: now,
         })
         this.db.prepare('UPDATE knowledge SET head_version = 1 WHERE id = ?').run(id)
-        this.db.exec('COMMIT')
+        this.db.exec('RELEASE knowledge_ingest')
         return { knowledge: this.read(id)!, merged: false, addedSources: added }
       } catch (cause) {
-        this.db.exec('ROLLBACK')
+        this.db.exec('ROLLBACK TO knowledge_ingest; RELEASE knowledge_ingest')
         throw cause
       }
     }
@@ -744,6 +748,10 @@ export class KnowledgeStore {
       const row = normalizeSource(source, now)
       const key = sourceKey(row)
       if (seen.has(key)) {
+        if (existingByKey.get(key)?.withdrawn) {
+          this.db!.prepare('UPDATE knowledge_sources SET withdrawn=0 WHERE knowledge_id=? AND document_id=? AND quote=? AND COALESCE(to_version,-1)=?')
+            .run(id, row.documentId, row.quote, row.toVersion ?? -1)
+        }
         continue
       }
       seen.add(key)
@@ -963,6 +971,7 @@ export class KnowledgeStore {
   } {
     const where: string[] = [query.includeMerged === true ? '1 = 1' : "merged_into = ''"]
     const params: Record<string, unknown> = {}
+    if (query.excludeWithdrawn) where.push('EXISTS (SELECT 1 FROM knowledge_sources active_source WHERE active_source.knowledge_id = knowledge.id AND active_source.withdrawn = 0)')
     if (query.ids !== undefined) {
       // 空数组是一个**取值**（筛出空集），不能退化成「不过滤」—— 那会让一次失败的落位
       // 查询变成「把整库的知识塞进这次召回」。
@@ -1516,6 +1525,7 @@ function rowToKnowledge(row: Record<string, unknown>, sourceRows: Array<Record<s
       author: String(source.author ?? ''),
       statement: String(source.statement ?? ''),
       ...(source.to_version === null || source.to_version === undefined ? {} : { toVersion: Number(source.to_version) }),
+      ...(Number(source.withdrawn) === 1 ? { withdrawn: true } : {}),
       createdAt: String(source.created_at),
     })),
   }

package/src/model-text.ts

@@ -101,6 +101,7 @@ export function buildModelRequest(input: {
 export async function collectModelText(
   stream: AsyncIterable<StreamChunkLike>,
   onUsage?: (usage: TrialTokenUsage) => void,
+  onDelta?: (text: string) => void,
 ): Promise<string> {
   let text = ''
   /** 流怎么结束的。用来区分「正常但空」「失败」「中止」三种情况。 */
@@ -112,6 +113,7 @@ export async function collectModelText(
     if (chunk.type === 'usage' && chunk.usage) onUsage?.(chunk.usage)
     if (chunk.type === 'text-delta' && typeof chunk.text === 'string') {
       text += chunk.text
+      onDelta?.(chunk.text)
       continue
     }
     if (chunk.type === 'finish') {
@@ -120,9 +122,9 @@ export async function collectModelText(
     }
   }
 
-  if (failure) {
-    const code = failure.code?.trim() || '未知错误码'
-    const message = failure.message?.trim() || '模型服务没有给出说明'
+  if (failure || finish === 'error' || finish === 'aborted' || finish === 'max-tokens') {
+    const code = failure?.code?.trim() || (finish === 'max-tokens' ? 'OUTPUT_TRUNCATED' : '未知错误码')
+    const message = failure?.message?.trim() || (finish === 'max-tokens' ? '输出达到长度上限，部分返回已保留，请调整模型限制后重试' : '模型服务没有给出说明')
     throw new Error(`模型调用失败（${code}）：${message}`)
   }
   if (text.length === 0) {

package/src/pipeline/engine.ts

@@ -0,0 +1,238 @@
+import { randomUUID } from 'node:crypto'
+import type { ModelCall } from '../extract-model.js'
+import type { SourceRaw, WikiTypeDef } from '../types.js'
+import type { TrialReport } from '../wiki-trial-types.js'
+import { PipelineStore, type ExtractionCheckpoint } from './store.js'
+import { PIPELINE_ACTIVE_STATES, PIPELINE_STAGE_LABEL, type MaterializationGroup, type PipelineAction, type PipelineModel, type PipelineSelection, type PipelineStage, type PipelineTask } from './types.js'
+
+export interface PipelineSource extends SourceRaw { version: number; sourceHash: string }
+export interface PipelineContext {
+  model: PipelineModel & { call?: ModelCall; why?: string }
+  note: string
+  profile?: WikiTypeDef
+  organize: boolean
+  format: 'json' | 'table'
+}
+export interface PipelineHost {
+  capture(): PipelineContext
+  serialize<T>(work: () => Promise<T>): Promise<T>
+  sources(): Promise<PipelineSource[]>
+  fingerprint(source: PipelineSource, context: PipelineContext): string
+  extract(source: PipelineSource, context: PipelineContext, output: (text: string) => void): Promise<TrialReport>
+  normalize(source: PipelineSource, checkpoint: ExtractionCheckpoint, context: PipelineContext, force: boolean, output: (text: string) => void): Promise<{ reused: boolean; warnings: string[] }>
+  groups(context: PipelineContext): Promise<{ groups: MaterializationGroup[]; blocked: number }>
+  materialize(group: MaterializationGroup, context: PipelineContext, output: (text: string) => void): Promise<void>
+  build(taskId: string, checkpoint: () => Promise<void>): Promise<string>
+  record(checkpoint: ExtractionCheckpoint): Promise<void>
+}
+
+class TaskCancelled extends Error {}
+interface Control { pause: boolean; cancel: boolean; resume?: () => void }
+
+/** One writer queue per base, shared with the legacy index builder. */
+export class KnowledgePipeline {
+  private closed = false
+  private controls = new Map<string, Control>()
+  constructor(readonly store: PipelineStore, private readonly host: PipelineHost) {}
+
+  start(stage: PipelineStage, selection: PipelineSelection = {}): { task: PipelineTask; completion: Promise<PipelineTask> } {
+    if (this.closed) throw new Error('知识处理已关闭')
+    if (!['extract', 'normalize', 'materialize', 'build', 'all'].includes(stage)) throw new Error('不支持的知识处理阶段')
+    for (const value of [selection.sourceIds, selection.groupKeys]) {
+      if (value !== undefined && (!Array.isArray(value) || value.length === 0 || value.some(id => typeof id !== 'string' || !id.trim()))) throw new Error('选择范围不能为空；不传范围表示处理当前阶段的全部待办')
+    }
+    const context = this.host.capture()
+    const { call: _call, why: _why, ...model } = context.model
+    const task: PipelineTask = {
+      id: `pipeline_${randomUUID()}`, stage, state: 'queued', selection: structuredClone(selection), model,
+      queuedAt: new Date().toISOString(), done: 0, total: 0, reused: 0, failed: 0,
+      summary: '等待运行', warnings: [],
+    }
+    this.store.putTask(task)
+    this.controls.set(task.id, { pause: false, cancel: false })
+    const completion = this.host.serialize(() => this.execute(task.id, context))
+    // Starting a background task must not create an unhandled rejection.
+    void completion.catch(() => undefined)
+    return { task, completion }
+  }
+
+  close(): void {
+    this.closed = true
+    for (const control of this.controls.values()) { control.cancel = true; control.pause = false; control.resume?.() }
+  }
+
+  control(id: string, action: PipelineAction): PipelineTask {
+    const task = this.store.task(id)
+    const control = this.controls.get(id)
+    if (!task || !control || !PIPELINE_ACTIVE_STATES.includes(task.state)) throw new Error('任务已结束；请重新运行所需阶段')
+    if (action === 'pause') {
+      control.pause = true
+      this.patch(id, { state: task.state === 'queued' ? 'paused' : 'pausing', summary: '当前文档或卡片处理完成后暂停' })
+    } else if (action === 'resume') {
+      if (!control.pause) throw new Error('任务没有暂停')
+      control.pause = false
+      this.patch(id, { state: task.startedAt ? 'running' : 'queued' })
+      control.resume?.()
+      control.resume = undefined
+    } else {
+      control.cancel = true
+      control.pause = false
+      this.patch(id, { state: 'cancelling', summary: '当前文档或卡片处理完成后停止' })
+      control.resume?.()
+    }
+    return this.store.task(id)!
+  }
+
+  private patch(id: string, patch: Partial<PipelineTask>): PipelineTask {
+    const task = { ...this.store.task(id)!, ...patch }
+    this.store.putTask(task)
+    return task
+  }
+  private finishStage(id: string): void {
+    const task = this.store.task(id)!
+    if (task.currentStage) this.patch(id, { results: { ...task.results, [task.currentStage]: { done: task.done, total: task.total, reused: task.reused, failed: task.failed } } })
+  }
+  private async checkpoint(id: string): Promise<void> {
+    const control = this.controls.get(id)
+    while (control) {
+      if (control.cancel) throw new TaskCancelled('任务已停止，已完成的文档与阶段会保留')
+      if (!control.pause) return
+      this.patch(id, { state: 'paused', summary: '已暂停，可继续运行' })
+      await new Promise<void>(resolve => { control.resume = resolve })
+    }
+  }
+  private output(id: string, sourceId?: string): { append: (text: string) => void; flush: () => void } {
+    let buffer = ''
+    let last = Date.now()
+    let bytes = 0
+    let capped = false
+    const flush = (): void => {
+      if (!buffer) return
+      this.store.event(id, 'output', buffer, sourceId)
+      buffer = ''
+      last = Date.now()
+    }
+    let failure: unknown
+    const timer = setInterval(() => { try { flush() } catch (cause) { failure = cause; clearInterval(timer) } }, 250)
+    timer.unref()
+    return {
+      append: text => {
+        if (failure) throw failure
+        bytes += Buffer.byteLength(text)
+        if (bytes > 4 * 1024 * 1024) {
+          if (!capped) { flush(); this.store.event(id, 'warning', '实时预览超过 4 MB；完整返回保存在本次抽取报告中。', sourceId); capped = true }
+          return
+        }
+        buffer += text
+        if (buffer.length >= 1024 || Date.now() - last >= 250) flush()
+      }, flush: () => { clearInterval(timer); if (failure) throw failure; flush() },
+    }
+  }
+  private async execute(id: string, context: PipelineContext): Promise<PipelineTask> {
+    const task = this.store.task(id)!
+    const stages = task.stage === 'all' ? ['extract', 'normalize', 'materialize', 'build'] as const : [task.stage]
+    try {
+      await this.checkpoint(id)
+      this.patch(id, { state: 'running', startedAt: new Date().toISOString() })
+      const allSources = await this.host.sources()
+      const sourceIds = task.selection.sourceIds === undefined ? undefined : new Set(task.selection.sourceIds)
+      if (sourceIds && [...sourceIds].some(sourceId => !allSources.some(source => source.sourceId === sourceId))) throw new Error('选择范围中有已删除或不存在的来源')
+      const sources = allSources.filter(source => !sourceIds || sourceIds.has(source.sourceId))
+      for (const stage of stages) {
+        await this.checkpoint(id)
+        this.patch(id, { currentStage: stage, done: 0, total: 0, reused: 0, failed: 0, summary: PIPELINE_STAGE_LABEL[stage] })
+        this.store.event(id, 'progress', PIPELINE_STAGE_LABEL[stage])
+        if (stage === 'build') {
+          const summary = await this.host.build(id, () => this.checkpoint(id))
+          this.patch(id, { done: 1, total: 1, summary })
+          this.finishStage(id)
+          continue
+        }
+        if (stage === 'materialize') {
+          const { groups } = await this.host.groups(context)
+          const selected = task.selection.groupKeys ? new Set(task.selection.groupKeys) : undefined
+          if (selected && [...selected].some(key => !groups.some(group => group.key === key))) throw new Error('选择的卡片分组已变化，请刷新待物化列表')
+          const affectedSources = task.stage === 'all' && sourceIds ? new Set(sources.map(source => source.sourceId)) : undefined
+          const pending = groups.filter(group => (!selected || selected.has(group.key)) && (!affectedSources || group.sourceIds.some(sourceId => affectedSources.has(sourceId))) && (task.selection.force || group.state !== 'current'))
+          this.patch(id, { total: pending.length })
+          for (const group of pending) {
+            await this.checkpoint(id)
+            const output = this.output(id)
+            try { await this.host.materialize(group, context, output.append) }
+            catch (cause) {
+              if (cause instanceof TaskCancelled) throw cause
+              const message = `${group.title}：${cause instanceof Error ? cause.message : String(cause)}`
+              const current = this.store.task(id)!
+              this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
+              this.store.event(id, 'warning', message)
+            }
+            finally { output.flush() }
+            const current = this.store.task(id)!
+            this.patch(id, { done: current.done + 1, summary: `已物化 ${current.done + 1}/${pending.length} 张：${group.title}` })
+          }
+          this.finishStage(id)
+          const current = this.store.task(id)!
+          if (current.failed) return this.patch(id, { state: current.failed === current.total ? 'failed' : 'partial', finishedAt: new Date().toISOString(), summary: `物化结束：成功 ${current.total - current.failed}/${current.total}；已完成卡片保留，可重试。` })
+          continue
+        }
+        this.patch(id, { total: sources.length })
+        for (const source of sources) {
+          await this.checkpoint(id)
+          this.store.event(id, 'progress', `${PIPELINE_STAGE_LABEL[stage]}：${source.title}`, source.sourceId)
+          const output = this.output(id, source.sourceId)
+          let reused = false
+          try {
+            if (stage === 'extract') {
+              const fingerprint = this.host.fingerprint(source, context)
+              const cached = task.selection.force ? undefined : this.store.cached(source.sourceId, source.version, fingerprint)
+              if (cached) { this.store.selectExtraction(source.sourceId, cached.id); reused = true }
+              else {
+                const report = await this.host.extract(source, context, output.append)
+                if (this.closed) throw new TaskCancelled('知识库已关闭，已完成结果保留')
+                const checkpoint: ExtractionCheckpoint = { id: `extract_${randomUUID()}`, sourceId: source.sourceId, sourceVersion: source.version, sourceHash: source.sourceHash, fingerprint, report, createdAt: new Date().toISOString() }
+                // The host checks the captured source version again before activation.
+                const current = (await this.host.sources()).find(row => row.sourceId === source.sourceId)
+                if (!current || current.version !== source.version || current.sourceHash !== source.sourceHash) {
+                  await this.host.record(checkpoint)
+                  this.store.event(id, 'report', checkpoint.id, source.sourceId)
+                  throw new Error('来源在抽取期间已变化；返回已保留在历史中，请抽取最新版本')
+                }
+                await this.host.record(checkpoint)
+                this.store.event(id, 'report', checkpoint.id, source.sourceId)
+                if (report.model.parsed !== true) throw new Error(report.model.why ?? '抽取返回未通过校验')
+                this.store.saveExtraction(checkpoint)
+              }
+            } else {
+              const checkpoint = this.store.head(source.sourceId).extracted
+              if (!checkpoint) throw new Error('尚无抽取结果，请先运行抽取')
+              if (checkpoint.sourceVersion !== source.version || checkpoint.sourceHash !== source.sourceHash) throw new Error('抽取结果依据旧版来源，请先抽取最新版本')
+              const result = await this.host.normalize(source, checkpoint, context, task.selection.force === true, output.append)
+              reused = result.reused
+              if (result.warnings.length) this.patch(id, { warnings: [...this.store.task(id)!.warnings, ...result.warnings.map(warning => `${source.title}：${warning}`)] })
+              for (const warning of result.warnings) this.store.event(id, 'warning', warning, source.sourceId)
+            }
+          } catch (cause) {
+            if (cause instanceof TaskCancelled) throw cause
+            const message = `${source.title}：${cause instanceof Error ? cause.message : String(cause)}`
+            const current = this.store.task(id)!
+            this.patch(id, { failed: current.failed + 1, warnings: [...current.warnings, message] })
+            this.store.event(id, 'warning', message, source.sourceId)
+          } finally { output.flush() }
+          const current = this.store.task(id)!
+          this.patch(id, { done: current.done + 1, reused: current.reused + Number(reused), summary: `${PIPELINE_STAGE_LABEL[stage]} ${current.done + 1}/${sources.length}${reused ? ' · 复用已有结果' : ''}` })
+        }
+        this.finishStage(id)
+        if (this.store.task(id)!.failed > 0) {
+          const current = this.store.task(id)!
+          this.patch(id, { state: current.failed === current.total ? 'failed' : 'partial', finishedAt: new Date().toISOString(), summary: `${PIPELINE_STAGE_LABEL[stage]}结束：成功 ${current.total - current.failed}/${current.total}，失败 ${current.failed}；已完成结果保留，可重试此阶段。` })
+          return this.store.task(id)!
+        }
+      }
+      await this.checkpoint(id)
+      const current = this.store.task(id)!
+      return this.patch(id, { state: 'succeeded', finishedAt: new Date().toISOString(), summary: task.stage === 'all' ? `全链路完成 · 抽取 ${current.results?.extract?.done ?? 0} 份（复用 ${current.results?.extract?.reused ?? 0} 份） · 物化 ${current.results?.materialize?.done ?? 0} 张 · 索引已发布` : `${PIPELINE_STAGE_LABEL[task.stage]}完成 · 处理 ${current.done} 项 · 复用 ${current.reused} 项` })
+    } catch (cause) {
+      return this.patch(id, { state: cause instanceof TaskCancelled ? 'cancelled' : 'failed', finishedAt: new Date().toISOString(), summary: cause instanceof Error ? cause.message : String(cause) })
+    } finally { this.controls.delete(id) }
+  }
+}

package/src/pipeline/store.ts

@@ -0,0 +1,119 @@
+import type { SqliteDatabase } from '../fts.js'
+import type { TrialReport } from '../wiki-trial-types.js'
+import type { PipelineEvent, PipelineTask } from './types.js'
+
+export interface ExtractionCheckpoint {
+  id: string
+  sourceId: string
+  sourceVersion: number
+  sourceHash: string
+  fingerprint: string
+  report: TrialReport
+  createdAt: string
+  normalized?: { report: TrialReport; knowledgeIds: Record<string, string>; eventIds: Record<string, string>; fingerprint: string; unresolvedKnowledgeIds?: string[] }
+}
+
+/** Business checkpoints live with truth; execution history lives with extraction audits. */
+export class PipelineStore {
+  constructor(readonly truth: SqliteDatabase, private readonly audit: SqliteDatabase) {
+    truth.exec(`
+      CREATE TABLE IF NOT EXISTS pipeline_extractions (
+        id TEXT PRIMARY KEY, source_id TEXT NOT NULL, source_version INTEGER NOT NULL,
+        source_hash TEXT NOT NULL, fingerprint TEXT NOT NULL, report TEXT NOT NULL,
+        normalized TEXT, created_at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS pipeline_extract_cache ON pipeline_extractions(source_id, source_version, fingerprint);
+      CREATE TABLE IF NOT EXISTS pipeline_sources (source_id TEXT PRIMARY KEY, extracted_id TEXT, normalized_id TEXT);
+      CREATE TABLE IF NOT EXISTS pipeline_materializations (group_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, card_id TEXT NOT NULL);
+    `)
+    audit.exec(`
+      CREATE TABLE IF NOT EXISTS pipeline_tasks (id TEXT PRIMARY KEY, queued_at TEXT NOT NULL, state TEXT NOT NULL, payload TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS pipeline_tasks_time ON pipeline_tasks(queued_at DESC);
+      CREATE TABLE IF NOT EXISTS pipeline_events (
+        seq INTEGER PRIMARY KEY AUTOINCREMENT, task_id TEXT NOT NULL, kind TEXT NOT NULL,
+        text TEXT NOT NULL, source_id TEXT, at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS pipeline_events_task ON pipeline_events(task_id, seq);
+    `)
+    // A new runtime never claims that an abandoned request is still running.
+    for (const task of (audit.prepare("SELECT payload FROM pipeline_tasks WHERE state IN ('queued','running','pausing','paused','cancelling')").all() as Array<{ payload: string }>).map(row => JSON.parse(row.payload) as PipelineTask)) {
+      if (['queued', 'running', 'pausing', 'paused', 'cancelling'].includes(task.state)) {
+        this.putTask({ ...task, state: 'interrupted', finishedAt: new Date().toISOString(), summary: '服务已重新启动；已完成的阶段保留，可重新运行此任务。' })
+      }
+    }
+  }
+
+  putTask(task: PipelineTask): void {
+    this.audit.prepare('INSERT INTO pipeline_tasks(id, queued_at, state, payload) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET state=excluded.state, payload=excluded.payload')
+      .run(task.id, task.queuedAt, task.state, JSON.stringify(task))
+  }
+  task(id: string): PipelineTask | undefined {
+    const row = this.audit.prepare('SELECT payload FROM pipeline_tasks WHERE id=?').get(id) as { payload: string } | undefined
+    return row ? JSON.parse(row.payload) as PipelineTask : undefined
+  }
+  tasks(limit = 20, offset = 0): { tasks: PipelineTask[]; total: number } {
+    const rows = this.audit.prepare("SELECT payload FROM pipeline_tasks ORDER BY CASE WHEN state IN ('queued','running','pausing','paused','cancelling') THEN 0 ELSE 1 END, queued_at DESC LIMIT ? OFFSET ?")
+      .all(Math.max(1, Math.min(10000, limit)), Math.max(0, offset)) as Array<{ payload: string }>
+    const total = Number((this.audit.prepare('SELECT COUNT(*) AS n FROM pipeline_tasks').get() as { n: number }).n)
+    return { tasks: rows.map(row => JSON.parse(row.payload) as PipelineTask), total }
+  }
+  event(taskId: string, kind: PipelineEvent['kind'], text: string, sourceId?: string): void {
+    this.audit.prepare('INSERT INTO pipeline_events(task_id,kind,text,source_id,at) VALUES (?,?,?,?,?)')
+      .run(taskId, kind, text, sourceId ?? null, new Date().toISOString())
+  }
+  events(taskId: string, after = 0, limit = 100): { events: PipelineEvent[]; next: number; hasMore: boolean } {
+    const take = Number.isFinite(limit) ? Math.max(1, Math.min(500, Math.trunc(limit))) : 100
+    after = Number.isFinite(after) ? Math.max(0, Math.trunc(after)) : 0
+    const rows = this.audit.prepare('SELECT * FROM pipeline_events WHERE task_id=? AND seq>? ORDER BY seq LIMIT ?')
+      .all(taskId, after, take + 1) as Array<Record<string, unknown>>
+    const hasMore = rows.length > take
+    const events = rows.slice(0, take).map(row => ({ seq: Number(row.seq), taskId: String(row.task_id), kind: String(row.kind) as PipelineEvent['kind'], text: String(row.text), ...(row.source_id ? { sourceId: String(row.source_id) } : {}), at: String(row.at) }))
+    return { events, next: events.at(-1)?.seq ?? after, hasMore }
+  }
+  checkpoint(id: string): ExtractionCheckpoint | undefined {
+    const row = this.truth.prepare('SELECT * FROM pipeline_extractions WHERE id=?').get(id) as Record<string, unknown> | undefined
+    return row ? { id: String(row.id), sourceId: String(row.source_id), sourceVersion: Number(row.source_version), sourceHash: String(row.source_hash), fingerprint: String(row.fingerprint), report: JSON.parse(String(row.report)), createdAt: String(row.created_at), ...(row.normalized ? { normalized: JSON.parse(String(row.normalized)) } : {}) } : undefined
+  }
+  cached(sourceId: string, version: number, fingerprint: string): ExtractionCheckpoint | undefined {
+    const row = this.truth.prepare('SELECT id FROM pipeline_extractions WHERE source_id=? AND source_version=? AND fingerprint=? ORDER BY created_at DESC, rowid DESC LIMIT 1')
+      .get(sourceId, version, fingerprint) as { id: string } | undefined
+    return row ? this.checkpoint(row.id) : undefined
+  }
+  head(sourceId: string): { extracted?: ExtractionCheckpoint; normalized?: ExtractionCheckpoint } {
+    const row = this.truth.prepare('SELECT extracted_id,normalized_id FROM pipeline_sources WHERE source_id=?').get(sourceId) as { extracted_id?: string; normalized_id?: string } | undefined
+    return { extracted: row?.extracted_id ? this.checkpoint(row.extracted_id) : undefined, normalized: row?.normalized_id ? this.checkpoint(row.normalized_id) : undefined }
+  }
+  saveExtraction(value: ExtractionCheckpoint): void {
+    this.truth.exec('SAVEPOINT pipeline_extract')
+    try {
+    this.truth.prepare('INSERT INTO pipeline_extractions(id,source_id,source_version,source_hash,fingerprint,report,created_at) VALUES (?,?,?,?,?,?,?)')
+      .run(value.id, value.sourceId, value.sourceVersion, value.sourceHash, value.fingerprint, JSON.stringify(value.report), value.createdAt)
+    this.selectExtraction(value.sourceId, value.id)
+    this.truth.exec('RELEASE pipeline_extract')
+    } catch (cause) { this.truth.exec('ROLLBACK TO pipeline_extract; RELEASE pipeline_extract'); throw cause }
+  }
+  selectExtraction(sourceId: string, id: string): void {
+    this.truth.prepare('INSERT INTO pipeline_sources(source_id,extracted_id) VALUES (?,?) ON CONFLICT(source_id) DO UPDATE SET extracted_id=excluded.extracted_id').run(sourceId, id)
+  }
+  saveNormalization(value: ExtractionCheckpoint, normalized: NonNullable<ExtractionCheckpoint['normalized']>): void {
+    this.truth.prepare('UPDATE pipeline_extractions SET normalized=? WHERE id=?').run(JSON.stringify(normalized), value.id)
+    this.truth.prepare('INSERT INTO pipeline_sources(source_id,normalized_id) VALUES (?,?) ON CONFLICT(source_id) DO UPDATE SET normalized_id=excluded.normalized_id').run(value.sourceId, value.id)
+  }
+  unresolvedKnowledge(): { ids: Set<string>; sources: Set<string> } {
+    const rows = this.truth.prepare('SELECT s.source_id, e.normalized FROM pipeline_sources s JOIN pipeline_extractions e ON e.id=s.normalized_id WHERE e.normalized IS NOT NULL').all() as Array<{ source_id: string; normalized: string }>
+    const ids = new Set<string>(); const sources = new Set<string>()
+    for (const row of rows) {
+      const pending = (JSON.parse(row.normalized) as NonNullable<ExtractionCheckpoint['normalized']>).unresolvedKnowledgeIds ?? []
+      if (pending.length) sources.add(row.source_id)
+      for (const id of pending) ids.add(id)
+    }
+    return { ids, sources }
+  }
+  materialization(key: string): { fingerprint: string; cardId: string } | undefined {
+    const row = this.truth.prepare('SELECT fingerprint,card_id FROM pipeline_materializations WHERE group_key=?').get(key) as { fingerprint: string; card_id: string } | undefined
+    return row ? { fingerprint: row.fingerprint, cardId: row.card_id } : undefined
+  }
+  materialized(key: string, fingerprint: string, cardId: string): void {
+    this.truth.prepare('INSERT INTO pipeline_materializations(group_key,fingerprint,card_id) VALUES (?,?,?) ON CONFLICT(group_key) DO UPDATE SET fingerprint=excluded.fingerprint,card_id=excluded.card_id').run(key, fingerprint, cardId)
+  }
+}

package/src/pipeline/types.ts

@@ -0,0 +1,74 @@
+/** Shared, browser-safe contracts for independently runnable knowledge stages. */
+export type PipelineStage = 'extract' | 'normalize' | 'materialize' | 'build' | 'all'
+export type PipelineState = 'queued' | 'running' | 'pausing' | 'paused' | 'cancelling' | 'succeeded' | 'partial' | 'failed' | 'cancelled' | 'interrupted'
+export type PipelineAction = 'pause' | 'resume' | 'cancel'
+
+export interface PipelineSelection {
+  sourceIds?: string[]
+  groupKeys?: string[]
+  force?: boolean
+}
+
+export interface PipelineModel {
+  provider?: string
+  model?: string
+  reasoningEffort?: string
+}
+
+export interface PipelineTask {
+  id: string
+  stage: PipelineStage
+  state: PipelineState
+  selection: PipelineSelection
+  model: PipelineModel
+  queuedAt: string
+  startedAt?: string
+  finishedAt?: string
+  currentStage?: Exclude<PipelineStage, 'all'>
+  done: number
+  total: number
+  reused: number
+  failed: number
+  summary: string
+  warnings: string[]
+  results?: Partial<Record<Exclude<PipelineStage, 'all'>, { done: number; total: number; reused: number; failed: number }>>
+  legacy?: boolean
+  reportId?: string
+}
+
+export interface PipelineEvent {
+  seq: number
+  taskId: string
+  kind: 'output' | 'progress' | 'warning' | 'report'
+  text: string
+  sourceId?: string
+  at: string
+}
+
+export interface MaterializationGroup {
+  key: string
+  title: string
+  coordinate: { objectId: string; actionId: string; sceneId: string }
+  knowledgeCount: number
+  eventCount: number
+  sourceIds: string[]
+  cardId?: string
+  state: 'new' | 'changed' | 'current' | 'withdrawn' | 'blocked'
+  reason?: string
+  fingerprint: string
+}
+
+export interface PipelineStatus {
+  sources: Array<{ id: string; title: string; version: number; extractionId?: string; normalizedId?: string; state: 'pending' | 'extracted' | 'normalized' | 'outdated' }>
+  groups: MaterializationGroup[]
+  blocked: number
+  pendingNew: number
+  pendingUpdated: number
+  pendingWithdrawn: number
+  indexPending: boolean
+}
+
+export const PIPELINE_STAGE_LABEL: Record<PipelineStage, string> = {
+  extract: '抽取 Knowledge / Event', normalize: '归一合并', materialize: '物化 WikiCard', build: '构建检索索引', all: '全链路运行',
+}
+export const PIPELINE_ACTIVE_STATES: readonly PipelineState[] = ['queued', 'running', 'pausing', 'paused', 'cancelling']

package/src/present.ts

@@ -54,6 +54,7 @@ function short(text: string, max = 48): string {
 export interface ImportMetaSource { sourceId: string; title?: string; bytes: number }
 export interface ImportMetaFailure { index: number; title?: string; code: string; message: string }
 export interface ImportMetaCompile {
+  taskId?: string; state?: string
   cardsCreated: number; cardsUpdated: number; cardsReused?: number
   notesReused?: number; notesRecompiled?: number
   sourcesExtracted?: number; sourcesSkipped?: number; cardsRemoved?: number
@@ -101,7 +102,7 @@ export function importCallView(args: { sources?: unknown[]; compile?: boolean })
   return {
     card: 'generic',
     kind: 'edit',
-    title: `导入 ${count} 份来源${args?.compile === false ? '' : '并编译'}`,
+    title: `导入 ${count} 份来源${args?.compile === true ? '并处理知识' : ''}`,
     rawInput: titles.length > 0 ? titles.map(title => short(title)) : { 份数: count },
   }
 }
@@ -112,10 +113,15 @@ export function importResultMarkdown(meta: ImportMeta): string {
   for (const item of meta.written) lines.push(`- ${item.title ? `**${short(item.title)}**` : item.sourceId} · ${item.bytes} 字符`)
   for (const item of meta.failed) lines.push(`- 第 ${item.index + 1} 份${item.title ? `「${short(item.title)}」` : ''}失败：${item.code} ${item.message}`)
   if (meta.compile) {
-    const { cardsCreated, cardsUpdated, cardsReused, notesReused, notesRecompiled, embedding, version } = meta.compile
+    const { cardsCreated, cardsUpdated, cardsReused, notesReused, notesRecompiled, embedding, version, state, taskId } = meta.compile
+    if (state) {
+      const label = state === 'succeeded' ? '已完成' : state === 'partial' ? '部分完成' : state === 'cancelled' ? '已停止' : '未完成'
+      lines.push('', `知识处理${label}：新建 ${cardsCreated} 张、更新 ${cardsUpdated} 张。${taskId ? `任务：${taskId}。` : ''}详情请打开知识处理与任务历史。`)
+      return lines.join('\n')
+    }
     lines.push('', `编译：新建 ${cardsCreated} 张、并入 ${cardsUpdated} 张、复用 ${cardsReused ?? 0} 张；召回片段复用 ${notesReused ?? 0} 条、重编 ${notesRecompiled ?? 0} 条；索引 v${version ?? '?'}${embedding ? `（向量 ${embedding}）` : ''}。`)
   } else {
-    lines.push('', '这次没有编译（`compile: false`）—— 卡片还没生成，需要时再调 `rebuild_knowledge`。')
+    lines.push('', '这次只保存来源（默认 `compile: false`）。下一步使用 `run_knowledge_pipeline` 分别抽取、归一、物化，最后构建检索索引。')
   }
   return lines.join('\n')
 }
@@ -314,7 +320,7 @@ export function addSourceCallView(args: { title?: string; content?: string }): {
 }
 
 export function addSourceResultMarkdown(meta: AddSourceMeta): string {
-  return `**来源已写入** —— 「${meta.title ? short(meta.title) : meta.sourceId}」（${meta.bytes} 字符）。\n\n下一步：\`rebuild_knowledge\` 把它编译成 WikiCard。`
+  return `**来源已写入** —— 「${meta.title ? short(meta.title) : meta.sourceId}」（${meta.bytes} 字符）。\n\n下一步：\`run_knowledge_pipeline\` 分别抽取、归一和物化，最后构建检索索引。`
 }
 
 export interface WriteCardsMeta {