dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.15

package/src/pipeline/store.ts

@@ -0,0 +1,147 @@
+import type { SqliteDatabase } from '../fts.js'
+import { applyRetention } from '../retention.js'
+import type { TrialReport } from '../wiki-trial-types.js'
+import type { PipelineEvent, PipelineTask } from './types.js'
+
+export interface ExtractionCheckpoint {
+  id: string
+  sourceId: string
+  sourceVersion: number
+  sourceHash: string
+  fingerprint: string
+  report: TrialReport
+  createdAt: string
+  normalized?: { report: TrialReport; knowledgeIds: Record<string, string>; eventIds: Record<string, string>; fingerprint: string; unresolvedKnowledgeIds?: string[] }
+}
+
+/** Business checkpoints live with truth; execution history lives with extraction audits. */
+export class PipelineStore {
+  constructor(readonly truth: SqliteDatabase, private readonly audit: SqliteDatabase) {
+    truth.exec(`
+      CREATE TABLE IF NOT EXISTS pipeline_extractions (
+        id TEXT PRIMARY KEY, source_id TEXT NOT NULL, source_version INTEGER NOT NULL,
+        source_hash TEXT NOT NULL, fingerprint TEXT NOT NULL, report TEXT NOT NULL,
+        normalized TEXT, created_at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS pipeline_extract_cache ON pipeline_extractions(source_id, source_version, fingerprint);
+      CREATE TABLE IF NOT EXISTS pipeline_sources (source_id TEXT PRIMARY KEY, extracted_id TEXT, normalized_id TEXT);
+      CREATE TABLE IF NOT EXISTS pipeline_materializations (group_key TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, card_id TEXT NOT NULL);
+    `)
+    audit.exec(`
+      CREATE TABLE IF NOT EXISTS pipeline_tasks (id TEXT PRIMARY KEY, queued_at TEXT NOT NULL, state TEXT NOT NULL, payload TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS pipeline_tasks_time ON pipeline_tasks(queued_at DESC);
+      CREATE TABLE IF NOT EXISTS pipeline_events (
+        seq INTEGER PRIMARY KEY AUTOINCREMENT, task_id TEXT NOT NULL, kind TEXT NOT NULL,
+        text TEXT NOT NULL, source_id TEXT, at TEXT NOT NULL, stage TEXT
+      );
+      CREATE INDEX IF NOT EXISTS pipeline_events_task ON pipeline_events(task_id, seq);
+    `)
+    const eventColumns = audit.prepare('PRAGMA table_info(pipeline_events)').all() as Array<{ name: string }>
+    if (!eventColumns.some(column => column.name === 'stage')) audit.exec('ALTER TABLE pipeline_events ADD COLUMN stage TEXT')
+    // A new runtime never claims that an abandoned request is still running.
+    for (const task of (audit.prepare("SELECT payload FROM pipeline_tasks WHERE state IN ('queued','running','pausing','paused','cancelling')").all() as Array<{ payload: string }>).map(row => JSON.parse(row.payload) as PipelineTask)) {
+      if (['queued', 'running', 'pausing', 'paused', 'cancelling'].includes(task.state)) {
+        this.putTask({ ...task, state: 'interrupted', finishedAt: new Date().toISOString(), summary: '服务已重新启动；已完成的阶段保留，可重新运行此任务。' })
+      }
+    }
+  }
+
+  putTask(task: PipelineTask): void {
+    this.audit.prepare('INSERT INTO pipeline_tasks(id, queued_at, state, payload) VALUES (?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET state=excluded.state, payload=excluded.payload')
+      .run(task.id, task.queuedAt, task.state, JSON.stringify(task))
+    this.pruneAudit()
+  }
+
+  /**
+   * 审计库的保留上限（0070）。
+   *
+   * 实时预览每 250ms 或每 1KB 落一行，一次五分钟的抽取就能加一千行，而这里原来一条
+   * `DELETE` 都没有。上限是「水位以上保留」：升级那一刻已有的任务与事件一行不删
+   * （见 `retention.ts`）。任务的保留上限同时也是事件的：先按任务裁，再把已经不存在的
+   * 那些任务的事件（同样只在水位以上）删掉，最后兜一层总量。
+   */
+  private pruneAudit(): void {
+    applyRetention(this.audit, 'pipeline_tasks', { table: 'pipeline_tasks', key: 'rowid', keep: 200 })
+    applyRetention(this.audit, 'pipeline_events', { table: 'pipeline_events', key: 'seq', keep: 50_000, where: 'task_id NOT IN (SELECT id FROM pipeline_tasks)' })
+    applyRetention(this.audit, 'pipeline_events', { table: 'pipeline_events', key: 'seq', keep: 50_000 })
+  }
+  task(id: string): PipelineTask | undefined {
+    const row = this.audit.prepare('SELECT payload FROM pipeline_tasks WHERE id=?').get(id) as { payload: string } | undefined
+    return row ? JSON.parse(row.payload) as PipelineTask : undefined
+  }
+  tasks(limit = 20, offset = 0): { tasks: PipelineTask[]; total: number } {
+    const rows = this.audit.prepare("SELECT payload FROM pipeline_tasks ORDER BY CASE WHEN state IN ('queued','running','pausing','paused','cancelling') THEN 0 ELSE 1 END, queued_at DESC LIMIT ? OFFSET ?")
+      .all(Math.max(1, Math.min(10000, limit)), Math.max(0, offset)) as Array<{ payload: string }>
+    const total = Number((this.audit.prepare('SELECT COUNT(*) AS n FROM pipeline_tasks').get() as { n: number }).n)
+    return { tasks: rows.map(row => JSON.parse(row.payload) as PipelineTask), total }
+  }
+  event(taskId: string, kind: PipelineEvent['kind'], text: string, sourceId?: string, stage?: PipelineEvent['stage']): void {
+    this.audit.prepare('INSERT INTO pipeline_events(task_id,kind,text,source_id,at,stage) VALUES (?,?,?,?,?,?)')
+      .run(taskId, kind, text, sourceId ?? null, new Date().toISOString(), stage ?? null)
+  }
+  events(taskId: string, after = 0, limit = 100): { events: PipelineEvent[]; next: number; hasMore: boolean } {
+    const take = Number.isFinite(limit) ? Math.max(1, Math.min(500, Math.trunc(limit))) : 100
+    after = Number.isFinite(after) ? Math.max(0, Math.trunc(after)) : 0
+    const rows = this.audit.prepare('SELECT * FROM pipeline_events WHERE task_id=? AND seq>? ORDER BY seq LIMIT ?')
+      .all(taskId, after, take + 1) as Array<Record<string, unknown>>
+    const hasMore = rows.length > take
+    const events = rows.slice(0, take).map(row => ({ seq: Number(row.seq), taskId: String(row.task_id), kind: String(row.kind) as PipelineEvent['kind'], text: String(row.text), ...(row.source_id ? { sourceId: String(row.source_id) } : {}), ...(row.stage ? { stage: String(row.stage) as PipelineEvent['stage'] } : {}), at: String(row.at) }))
+    return { events, next: events.at(-1)?.seq ?? after, hasMore }
+  }
+  checkpoint(id: string): ExtractionCheckpoint | undefined {
+    const row = this.truth.prepare('SELECT * FROM pipeline_extractions WHERE id=?').get(id) as Record<string, unknown> | undefined
+    return row ? { id: String(row.id), sourceId: String(row.source_id), sourceVersion: Number(row.source_version), sourceHash: String(row.source_hash), fingerprint: String(row.fingerprint), report: JSON.parse(String(row.report)), createdAt: String(row.created_at), ...(row.normalized ? { normalized: JSON.parse(String(row.normalized)) } : {}) } : undefined
+  }
+  cached(sourceId: string, version: number, fingerprint: string): ExtractionCheckpoint | undefined {
+    const row = this.truth.prepare('SELECT id FROM pipeline_extractions WHERE source_id=? AND source_version=? AND fingerprint=? ORDER BY created_at DESC, rowid DESC LIMIT 1')
+      .get(sourceId, version, fingerprint) as { id: string } | undefined
+    return row ? this.checkpoint(row.id) : undefined
+  }
+  head(sourceId: string): { extracted?: ExtractionCheckpoint; normalized?: ExtractionCheckpoint } {
+    const row = this.truth.prepare('SELECT extracted_id,normalized_id FROM pipeline_sources WHERE source_id=?').get(sourceId) as { extracted_id?: string; normalized_id?: string } | undefined
+    return { extracted: row?.extracted_id ? this.checkpoint(row.extracted_id) : undefined, normalized: row?.normalized_id ? this.checkpoint(row.normalized_id) : undefined }
+  }
+  saveExtraction(value: ExtractionCheckpoint): void {
+    this.truth.exec('SAVEPOINT pipeline_extract')
+    try {
+    this.truth.prepare('INSERT INTO pipeline_extractions(id,source_id,source_version,source_hash,fingerprint,report,created_at) VALUES (?,?,?,?,?,?,?)')
+      .run(value.id, value.sourceId, value.sourceVersion, value.sourceHash, value.fingerprint, JSON.stringify(value.report), value.createdAt)
+    this.selectExtraction(value.sourceId, value.id)
+    this.truth.exec('RELEASE pipeline_extract')
+    } catch (cause) { this.truth.exec('ROLLBACK TO pipeline_extract; RELEASE pipeline_extract'); throw cause }
+  }
+  selectExtraction(sourceId: string, id: string): void {
+    this.truth.prepare('INSERT INTO pipeline_sources(source_id,extracted_id) VALUES (?,?) ON CONFLICT(source_id) DO UPDATE SET extracted_id=excluded.extracted_id').run(sourceId, id)
+  }
+  /**
+   * 刷新一份抽取检查点的指纹（0070）。
+   *
+   * 指纹里含「本文提到的实体清单」，而归一会往注册表里学新实体 —— 它们本来就在这份
+   * 文档里，所以学完之后指纹必然变。不刷新的话，同一份未改动的文档下次抽取一定复用不上
+   * （每次都要再调一次模型）。归一成功后把这一行改成学完之后的值，复用才成立。
+   */
+  refreshFingerprint(id: string, fingerprint: string): void {
+    this.truth.prepare('UPDATE pipeline_extractions SET fingerprint=? WHERE id=?').run(fingerprint, id)
+  }
+  saveNormalization(value: ExtractionCheckpoint, normalized: NonNullable<ExtractionCheckpoint['normalized']>): void {
+    this.truth.prepare('UPDATE pipeline_extractions SET normalized=? WHERE id=?').run(JSON.stringify(normalized), value.id)
+    this.truth.prepare('INSERT INTO pipeline_sources(source_id,normalized_id) VALUES (?,?) ON CONFLICT(source_id) DO UPDATE SET normalized_id=excluded.normalized_id').run(value.sourceId, value.id)
+  }
+  unresolvedKnowledge(): { ids: Set<string>; sources: Set<string> } {
+    const rows = this.truth.prepare('SELECT s.source_id, e.normalized FROM pipeline_sources s JOIN pipeline_extractions e ON e.id=s.normalized_id WHERE e.normalized IS NOT NULL').all() as Array<{ source_id: string; normalized: string }>
+    const ids = new Set<string>(); const sources = new Set<string>()
+    for (const row of rows) {
+      const pending = (JSON.parse(row.normalized) as NonNullable<ExtractionCheckpoint['normalized']>).unresolvedKnowledgeIds ?? []
+      if (pending.length) sources.add(row.source_id)
+      for (const id of pending) ids.add(id)
+    }
+    return { ids, sources }
+  }
+  materialization(key: string): { fingerprint: string; cardId: string } | undefined {
+    const row = this.truth.prepare('SELECT fingerprint,card_id FROM pipeline_materializations WHERE group_key=?').get(key) as { fingerprint: string; card_id: string } | undefined
+    return row ? { fingerprint: row.fingerprint, cardId: row.card_id } : undefined
+  }
+  materialized(key: string, fingerprint: string, cardId: string): void {
+    this.truth.prepare('INSERT INTO pipeline_materializations(group_key,fingerprint,card_id) VALUES (?,?,?) ON CONFLICT(group_key) DO UPDATE SET fingerprint=excluded.fingerprint,card_id=excluded.card_id').run(key, fingerprint, cardId)
+  }
+}

package/src/pipeline/types.ts

@@ -0,0 +1,92 @@
+/** Shared, browser-safe contracts for independently runnable knowledge stages. */
+export type PipelineStage = 'extract' | 'normalize' | 'materialize' | 'build' | 'all'
+export type PipelineState = 'queued' | 'running' | 'pausing' | 'paused' | 'cancelling' | 'succeeded' | 'partial' | 'failed' | 'cancelled' | 'interrupted'
+export type PipelineAction = 'pause' | 'resume' | 'cancel'
+
+export interface PipelineSelection {
+  sourceIds?: string[]
+  groupKeys?: string[]
+  force?: boolean
+}
+
+export interface PipelineModel {
+  provider?: string
+  model?: string
+  reasoningEffort?: string
+}
+
+export interface PipelineTask {
+  id: string
+  stage: PipelineStage
+  state: PipelineState
+  selection: PipelineSelection
+  model: PipelineModel
+  queuedAt: string
+  startedAt?: string
+  finishedAt?: string
+  currentStage?: Exclude<PipelineStage, 'all'>
+  done: number
+  total: number
+  reused: number
+  failed: number
+  summary: string
+  warnings: string[]
+  results?: Partial<Record<Exclude<PipelineStage, 'all'>, { done: number; total: number; reused: number; failed: number; summary?: string; skipped?: boolean }>>
+  legacy?: boolean
+  reportId?: string
+}
+
+export interface PipelineEvent {
+  seq: number
+  taskId: string
+  kind: 'output' | 'progress' | 'warning' | 'report'
+  text: string
+  sourceId?: string
+  stage?: Exclude<PipelineStage, 'all'>
+  at: string
+}
+
+export interface MaterializationGroup {
+  key: string
+  title: string
+  coordinate: { objectId: string; actionId: string; sceneId: string }
+  knowledgeCount: number
+  eventCount: number
+  sourceIds: string[]
+  cardId?: string
+  state: 'new' | 'changed' | 'current' | 'withdrawn' | 'blocked'
+  reason?: string
+  fingerprint: string
+}
+
+/**
+ * 物化阶段的批（宿主产生、宿主消费）。
+ *
+ * `snapshot` 对引擎是**不透明的**：它的字段类型来自服务端模块（Knowledge / EventEntity /
+ * 图片 / 相关推荐），而那些模块都依赖 `node:*`。本模块被客户端 import 取值，所以这里
+ * 不能描述它 —— 真正的形状在 `materialize-snapshot.ts`，只有服务端看得见。
+ * 引擎只负责把 `snapshot` 原样传回 `materialize()`。
+ */
+export interface MaterializationBatch {
+  groups: MaterializationGroup[]
+  /** 坐标仍有歧义 + 等待归一，两部分的合计。 */
+  blocked: number
+  blockedAmbiguous: number
+  blockedAwaitingNormalize: number
+  snapshot: unknown
+}
+
+export interface PipelineStatus {
+  sources: Array<{ id: string; title: string; version: number; extractionId?: string; normalizedId?: string; state: 'pending' | 'extracted' | 'normalized' | 'outdated' }>
+  groups: MaterializationGroup[]
+  blocked: number
+  pendingNew: number
+  pendingUpdated: number
+  pendingWithdrawn: number
+  indexPending: boolean
+}
+
+export const PIPELINE_STAGE_LABEL: Record<PipelineStage, string> = {
+  extract: '抽取 Knowledge / Event', normalize: '归一合并', materialize: '物化 WikiCard', build: '构建检索索引', all: '全链路运行',
+}
+export const PIPELINE_ACTIVE_STATES: readonly PipelineState[] = ['queued', 'running', 'pausing', 'paused', 'cancelling']

package/src/present.ts

@@ -54,6 +54,7 @@ function short(text: string, max = 48): string {
 export interface ImportMetaSource { sourceId: string; title?: string; bytes: number }
 export interface ImportMetaFailure { index: number; title?: string; code: string; message: string }
 export interface ImportMetaCompile {
+  taskId?: string; state?: string
   cardsCreated: number; cardsUpdated: number; cardsReused?: number
   notesReused?: number; notesRecompiled?: number
   sourcesExtracted?: number; sourcesSkipped?: number; cardsRemoved?: number
@@ -101,7 +102,7 @@ export function importCallView(args: { sources?: unknown[]; compile?: boolean })
   return {
     card: 'generic',
     kind: 'edit',
-    title: `导入 ${count} 份来源${args?.compile === false ? '' : '并编译'}`,
+    title: `导入 ${count} 份来源${args?.compile === true ? '并处理知识' : ''}`,
     rawInput: titles.length > 0 ? titles.map(title => short(title)) : { 份数: count },
   }
 }
@@ -112,10 +113,15 @@ export function importResultMarkdown(meta: ImportMeta): string {
   for (const item of meta.written) lines.push(`- ${item.title ? `**${short(item.title)}**` : item.sourceId} · ${item.bytes} 字符`)
   for (const item of meta.failed) lines.push(`- 第 ${item.index + 1} 份${item.title ? `「${short(item.title)}」` : ''}失败：${item.code} ${item.message}`)
   if (meta.compile) {
-    const { cardsCreated, cardsUpdated, cardsReused, notesReused, notesRecompiled, embedding, version } = meta.compile
+    const { cardsCreated, cardsUpdated, cardsReused, notesReused, notesRecompiled, embedding, version, state, taskId } = meta.compile
+    if (state) {
+      const label = state === 'succeeded' ? '已完成' : state === 'partial' ? '部分完成' : state === 'cancelled' ? '已停止' : '未完成'
+      lines.push('', `知识处理${label}：新建 ${cardsCreated} 张、更新 ${cardsUpdated} 张。${taskId ? `任务：${taskId}。` : ''}详情请打开「知识处理」页的运行记录。`)
+      return lines.join('\n')
+    }
     lines.push('', `编译：新建 ${cardsCreated} 张、并入 ${cardsUpdated} 张、复用 ${cardsReused ?? 0} 张；召回片段复用 ${notesReused ?? 0} 条、重编 ${notesRecompiled ?? 0} 条；索引 v${version ?? '?'}${embedding ? `（向量 ${embedding}）` : ''}。`)
   } else {
-    lines.push('', '这次没有编译（`compile: false`）—— 卡片还没生成，需要时再调 `rebuild_knowledge`。')
+    lines.push('', '这次只保存来源（默认 `compile: false`）。下一步使用 `run_knowledge_pipeline` 分别抽取、归一、物化，最后构建检索索引。')
   }
   return lines.join('\n')
 }
@@ -314,7 +320,7 @@ export function addSourceCallView(args: { title?: string; content?: string }): {
 }
 
 export function addSourceResultMarkdown(meta: AddSourceMeta): string {
-  return `**来源已写入** —— 「${meta.title ? short(meta.title) : meta.sourceId}」（${meta.bytes} 字符）。\n\n下一步：\`rebuild_knowledge\` 把它编译成 WikiCard。`
+  return `**来源已写入** —— 「${meta.title ? short(meta.title) : meta.sourceId}」（${meta.bytes} 字符）。\n\n下一步：\`run_knowledge_pipeline\` 分别抽取、归一和物化，最后构建检索索引。`
 }
 
 export interface WriteCardsMeta {

package/src/retention.ts

@@ -0,0 +1,76 @@
+/**
+ * 保留上限（0070）。
+ *
+ * 文档里写着保留策略（「命名永久 + 最近 100 + 每日末版；机器版本只活到硬上限」），
+ * 而代码里对这几张表一条 `DELETE` 都没有：`revision` / `blob` / `tree` / `operations` /
+ * `pipeline_extractions` / `pipeline_events` 只增不减。这个模块把**新数据**的上限补上。
+ *
+ * ## 为什么是「水位 + 上限」，而不是直接按条数删
+ *
+ * 上限一旦生效，第一次裁剪就会把库里**已有的**历史删掉 —— 那是不可逆的，而用户的决定是
+ * 「只对新数据加上限，不动存量」。所以每张表第一次裁剪时先把当前最大键记成**水位**：
+ * 水位以下的既有行永不删除；水位以上的新行按上限保留。代价是总量变成
+ * 「升级那一刻的存量 + 上限」，而不是「上限」—— 这是「不动存量」能给出的最强的界。
+ *
+ * ## 键列的要求
+ *
+ * `key` 必须是单调递增的整数列（`rowid` 或自增 `seq`）。用时间戳当键不行：同一毫秒里
+ * 两行的顺序不确定，裁剪会随机丢一行。
+ */
+import type { SqliteDatabase } from './fts.js'
+
+export interface RetentionSpec {
+  /** 表名。内插进 SQL，所以调用方只能传常量。 */
+  table: string
+  /** 单调递增的整数键列，例如 `rowid` 或 `seq`。 */
+  key: string
+  /** 水位以上保留多少行；`0` 表示水位以上的匹配行全部删掉（用于回收无引用的 blob）。 */
+  keep: number
+  /** 额外的行条件（可含子查询）。同样只允许常量 SQL。 */
+  where?: string
+  /** 分组表达式：给了就是「每组保留 keep 行」，而不是全表保留。 */
+  groupBy?: string
+}
+
+/** 水位表的表名。它跟着所在的那个库走（审计库与 knowledge.db 各有一份）。 */
+const FLOOR_TABLE = 'retention_floor'
+
+function ensureFloors(db: SqliteDatabase): void {
+  db.exec(`CREATE TABLE IF NOT EXISTS ${FLOOR_TABLE} (name TEXT PRIMARY KEY, floor INTEGER NOT NULL, at TEXT NOT NULL)`)
+}
+
+function floorOf(db: SqliteDatabase, name: string, table: string, key: string): number {
+  const row = db.prepare(`SELECT floor FROM ${FLOOR_TABLE} WHERE name = ?`).get(name) as { floor?: unknown } | undefined
+  if (row !== undefined) return Number(row.floor ?? 0)
+  const max = Number((db.prepare(`SELECT COALESCE(MAX(${key}), 0) AS m FROM ${table}`).get() as { m?: unknown } | undefined)?.m ?? 0)
+  db.prepare(`INSERT OR IGNORE INTO ${FLOOR_TABLE}(name, floor, at) VALUES (?, ?, ?)`).run(name, max, new Date().toISOString())
+  return max
+}
+
+/**
+ * 裁掉水位以上的超限行，返回删掉的行数。
+ *
+ * 幂等且可反复调用：没有超限时最多花一次索引区间计数。
+ */
+export function applyRetention(db: SqliteDatabase, name: string, spec: RetentionSpec): number {
+  ensureFloors(db)
+  const floor = floorOf(db, name, spec.table, spec.key)
+  const filter = spec.where === undefined ? '' : ` AND (${spec.where})`
+  const above = Number((db.prepare(`SELECT COUNT(*) AS n FROM ${spec.table} WHERE ${spec.key} > ?${filter}`).get(floor) as { n?: unknown } | undefined)?.n ?? 0)
+  if (above <= spec.keep) return 0
+  if (spec.groupBy === undefined) {
+    const changes = spec.keep <= 0
+      ? db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} > ?${filter}`).run(floor)
+      : db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} > ? AND ${spec.key} < (
+           SELECT ${spec.key} FROM ${spec.table} WHERE ${spec.key} > ?${filter} ORDER BY ${spec.key} DESC LIMIT 1 OFFSET ?
+         )${filter}`).run(floor, floor, spec.keep - 1)
+    return Number((changes as { changes?: unknown })?.changes ?? 0)
+  }
+  const changes = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.key} IN (
+      SELECT ${spec.key} FROM (
+        SELECT ${spec.key}, ROW_NUMBER() OVER (PARTITION BY ${spec.groupBy} ORDER BY ${spec.key} DESC) AS rn
+        FROM ${spec.table} WHERE ${spec.key} > ?${filter}
+      ) WHERE rn > ?
+    )`).run(floor, spec.keep)
+  return Number((changes as { changes?: unknown })?.changes ?? 0)
+}