dsh-data-domain-knowledge 0.1.0-alpha.13 → 0.1.0-alpha.15

package/src/extract-table.ts

@@ -0,0 +1,126 @@
+import { decode, encode } from '@toon-format/toon'
+
+export const EXTRACTION_CONTRACT_VERSION = 'knowledge-table-v1'
+type Row = Record<string, unknown>
+const object = (value: unknown): Row => value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Row : {}
+const list = (value: unknown): Row[] => Array.isArray(value) ? value.map(object) : []
+const string = (value: unknown): string => typeof value === 'string' ? value : ''
+
+/** Strict tabular transport. The decoded result retains the existing extraction JSON model. */
+export function decodeExtractionTable(raw: string): Row {
+  const record = object(decode(raw, { strict: true }))
+  if (record.format !== EXTRACTION_CONTRACT_VERSION) throw new Error(`表格 format 必须为 ${EXTRACTION_CONTRACT_VERSION}`)
+  for (const key of ['knowledge', 'events', 'timeline', 'objects', 'actions', 'scenes', 'evidence', 'dropped']) {
+    if (!Array.isArray(record[key])) throw new Error(`表格缺少 ${key}；没有内容时写 ${key}[0]:`)
+    if ((record[key] as unknown[]).some(row => row === null || typeof row !== 'object' || Array.isArray(row))) throw new Error(`${key} 必须由固定列的记录组成`)
+  }
+  const columns: Record<string, string[]> = {
+    knowledge: ['id', 'text', 'object', 'action', 'scene', 'section', 'order', 'evidence'],
+    events: ['id', 'external_id', 'title', 'summary', 'object', 'action', 'scene', 'occurred_at', 'symptom', 'root_cause', 'root_cause_status', 'outcome', 'missing', 'evidence'],
+    timeline: ['event_id', 'order', 'type', 'description', 'result', 'at', 'evidence'],
+    objects: ['mention', 'definition', 'evidence', 'existing', 'parent'],
+    actions: ['mention', 'definition', 'evidence', 'existing', 'parent'],
+    scenes: ['mention', 'definition', 'evidence', 'existing', 'parent'],
+    evidence: ['id', 'quote', 'heading_path'], dropped: ['quote', 'why'],
+  }
+  for (const [key, fields] of Object.entries(columns)) {
+    for (const row of list(record[key])) {
+      if (fields.some(field => !(field in row))) throw new Error(`${key} 缺少固定列`)
+      for (const field of fields) {
+        if (field === 'order') {
+          if (!Number.isInteger(row[field]) || Number(row[field]) < 1) throw new Error(`${key}.order 必须为正整数`)
+        } else if (row[field] !== null && typeof row[field] !== 'string') throw new Error(`${key}.${field} 必须为字符串或 null`)
+      }
+    }
+  }
+  const evidence = new Map<string, Row>()
+  for (const row of list(record.evidence)) {
+    const id = string(row.id)
+    if (!id || evidence.has(id) || !string(row.quote)) throw new Error('出处 ID 重复、为空或缺少 quote')
+    let headings: unknown
+    try { headings = JSON.parse(string(row.heading_path) || '[]') } catch { throw new Error(`出处 ${id} 的 heading_path 不是字符串数组`) }
+    if (!Array.isArray(headings) || headings.some(value => typeof value !== 'string')) throw new Error(`出处 ${id} 的 heading_path 不是字符串数组`)
+    evidence.set(id, { quote: row.quote, heading_path: headings })
+  }
+  const refs = (value: unknown): Row[] => {
+    const ids = string(value).trim().split(/\s+/).filter(Boolean)
+    return ids.map(id => {
+      const source = evidence.get(id)
+      if (!source) throw new Error(`引用了不存在的出处 ${id}`)
+      return source
+    })
+  }
+  const assertIds = (rows: Row[], label: string): Set<string> => {
+    const ids = new Set<string>()
+    for (const row of rows) {
+      const id = string(row.id)
+      if (!id || ids.has(id)) throw new Error(`${label} 的 id 重复或为空`)
+      ids.add(id)
+    }
+    return ids
+  }
+  const knowledge = list(record.knowledge)
+  const events = list(record.events)
+  assertIds(knowledge, 'Knowledge')
+  const eventIds = assertIds(events, 'Event')
+  if (knowledge.some(row => !string(row.text).trim() || !string(row.evidence).trim())) throw new Error('Knowledge 缺少正文或出处')
+  const orders = new Set<string>()
+  for (const row of list(record.timeline)) {
+    if (!eventIds.has(string(row.event_id))) throw new Error('timeline 引用了不存在的 Event')
+    const key = `${row.event_id}:${row.order}`
+    if (orders.has(key) || !string(row.description).trim() || !['observation', 'check', 'action', 'result', 'conclusion'].includes(string(row.type))) throw new Error('timeline 顺序重复、描述为空或类型不合法')
+    orders.add(key)
+  }
+  for (const row of events) if (!string(row.title).trim() || (row.root_cause_status !== null && !['confirmed', 'suspected'].includes(string(row.root_cause_status)))) throw new Error('Event 标题为空或根因状态不合法')
+  return {
+    subject: record.subject,
+    knowledge: knowledge.map(row => ({ ...row, sources: refs(row.evidence) })),
+    events: events.map(row => {
+      let missing: unknown
+      try { missing = JSON.parse(string(row.missing) || '[]') } catch { throw new Error(`Event ${row.id} 的 missing 不是字符串数组`) }
+      if (!Array.isArray(missing) || missing.some(value => typeof value !== 'string')) throw new Error(`Event ${row.id} 的 missing 不是字符串数组`)
+      return {
+        ...row,
+        coordinate: { object_mention: row.object, action_mention: row.action, scene_mention: row.scene },
+        root_cause: row.root_cause === null || row.root_cause === '' ? null : { text: row.root_cause, status: row.root_cause_status },
+        missing,
+        sources: refs(row.evidence),
+        timeline: list(record.timeline).filter(step => step.event_id === row.id).sort((a, b) => Number(a.order) - Number(b.order)).map(step => ({ ...step, sources: refs(step.evidence) })),
+      }
+    }),
+    ...Object.fromEntries(['objects', 'actions', 'scenes'].map(key => [key, list(record[key]).map(row => ({ ...row, parent_mention: row.parent, evidence: refs(row.evidence).map(source => source.quote).join('\n') }))])),
+    dropped: record.dropped,
+  }
+}
+
+/** Used by the format benchmark and fixtures; evidence is emitted once and referenced by ID. */
+export function encodeExtractionTable(value: Row): string {
+  const evidence: Row[] = []
+  const byQuote = new Map<string, string>()
+  const references = (sources: unknown): string => list(sources).map(source => {
+    const key = JSON.stringify([source.quote, source.heading_path ?? []])
+    const existing = byQuote.get(key)
+    if (existing) return existing
+    const id = `q${evidence.length + 1}`
+    evidence.push({ id, quote: source.quote, heading_path: JSON.stringify(source.heading_path ?? []) })
+    byQuote.set(key, id)
+    return id
+  }).join(' ')
+  const timeline: Row[] = []
+  const table: Row = {
+    format: EXTRACTION_CONTRACT_VERSION,
+    subject: value.subject ?? '',
+    knowledge: list(value.knowledge).map(row => ({ id: row.id, text: row.text, object: row.object_mention ?? row.object ?? null, action: row.action_mention ?? row.action ?? null, scene: row.scene_mention ?? row.scene ?? null, section: row.section ?? '字段', order: row.order ?? 1, evidence: references(row.sources) })),
+    events: list(value.events).map(row => {
+      list(row.timeline).forEach((step, index) => timeline.push({ event_id: row.id, order: index + 1, type: step.type, description: step.description, result: step.result ?? null, at: step.at ?? null, evidence: references(step.sources) }))
+      const coordinate = object(row.coordinate)
+      const cause = object(row.root_cause)
+      return { id: row.id, external_id: row.external_id ?? null, title: row.title, summary: row.summary, object: coordinate.object_mention ?? null, action: coordinate.action_mention ?? null, scene: coordinate.scene_mention ?? null, occurred_at: row.occurred_at ?? null, symptom: row.symptom ?? null, root_cause: cause.text ?? null, root_cause_status: cause.status ?? null, outcome: row.outcome ?? null, missing: JSON.stringify(row.missing ?? []), evidence: references(row.sources) }
+    }),
+    timeline,
+    ...Object.fromEntries(['objects', 'actions', 'scenes'].map(key => [key, list(value[key]).map(row => ({ mention: row.mention, definition: row.definition ?? '', evidence: row.evidence ? references([{ quote: row.evidence, heading_path: [] }]) : '', existing: row.existing ?? null, parent: row.parent_mention ?? row.parent ?? null }))])),
+    evidence,
+    dropped: value.dropped ?? [],
+  }
+  return encode(table)
+}

package/src/extraction-runs.ts

@@ -1,5 +1,14 @@
 /**
- * 抽取轨迹（`extraction_run`）：把每一次「这份文档抽出了什么」记成一条**不可重建**的记录。
+ * 轨迹流水（`extraction_run`）：把每一次「这一段处理了什么」记成一条**不可重建**的记录。
+ *
+ * ## 三段，一张表
+ *
+ * 2026-10-18 起抽取、归一、物化是三个独立阶段，所以轨迹也分三种（`stage` 列）：
+ * 抽取（一份来源 × 一次）、归一（一份来源 × 一次）、物化（一个卡片分组 × 一次）。
+ * **共用一张表**是因为它们的读写形状完全一样：按 `(kb, stage, 时间)` 倒序翻页、
+ * 点开读一份 JSON 报告、不可重建。拆三张表只会让「这份文档跑过几次」变成三次查询。
+ *
+ * 抽取那一份的正文是整份 `TrialReport`；归一与物化的形状见 `trajectory-types.ts`。
  *
  * ## 它是什么，不是什么
  *
@@ -30,12 +39,14 @@
 import { mkdir } from 'node:fs/promises'
 import { dirname } from 'node:path'
 import { loadSqlite, type SqliteDatabase } from './fts.js'
+import { applyRetention } from './retention.js'
+import type { TrajectoryStage } from './trajectory-types.js'
 
 /** 一次抽取是谁发起的。 */
 export type ExtractionRunActor = 'panel' | 'agent'
 
 /**
- * 一条抽取轨迹的**列表投影**（不含报告本体）。
+ * 一条轨迹的**列表投影**（不含报告本体）。
  *
  * 列表要回答的是「这份文档跑过几次、什么时候跑的、模型通了没有」，所以这些字段进列；
  * 报告要等点开某一条时才读。
@@ -56,9 +67,11 @@ export interface ExtractionRunSummary {
   summary: string
   /** 报告本体在不在。列表接口不返回它。 */
   hasReport: boolean
+  /** 这条轨迹属于哪一段。旧记录一律是 `extract`（那一列有默认值）。 */
+  stage: TrajectoryStage
 }
 
-/** 一条完整的抽取轨迹：列表投影 + 报告本体。 */
+/** 一条完整的轨迹：列表投影 + 报告本体。 */
 export interface ExtractionRunRecord extends ExtractionRunSummary {
   /** 整份报告（`TrialReport` 的形状）。读的时候按 unknown 处理，由前端类型收窄。 */
   report: unknown
@@ -66,6 +79,8 @@ export interface ExtractionRunRecord extends ExtractionRunSummary {
 
 export interface ExtractionRunQuery {
   docId?: string
+  /** 只看某一段。省略 = 三段一起（面板里那份「这份文档的轨迹」就是这么读的）。 */
+  stage?: TrajectoryStage
   limit?: number
   offset?: number
 }
@@ -86,19 +101,22 @@ CREATE TABLE IF NOT EXISTS extraction_runs (
   model_ok INTEGER NOT NULL DEFAULT 0,
   model_why TEXT,
   summary TEXT NOT NULL DEFAULT '',
-  report TEXT NOT NULL DEFAULT '{}'
+  report TEXT NOT NULL DEFAULT '{}',
+  stage TEXT NOT NULL DEFAULT 'extract'
 );
 CREATE INDEX IF NOT EXISTS idx_extraction_runs_doc ON extraction_runs(kb, doc_id, created_at DESC);
 CREATE INDEX IF NOT EXISTS idx_extraction_runs_time ON extraction_runs(kb, created_at DESC);
 `
 
 /**
- * 抽取轨迹库。**一个知识库一份**（随 Runtime 创建），懒打开。
+ * 轨迹库。**一个知识库一份**（随 Runtime 创建），懒打开。
  *
  * API 与 `OperationLog` 同形（同步的 `DatabaseSync`）：本地文件、单写者，同步最简单。
  */
 export class ExtractionRunStore {
   private db: SqliteDatabase | null = null
+  /** 距离上一次裁剪又落了几条（`append` 里用）。 */
+  private appendsSincePrune = 0
 
   constructor(private readonly file: string) {}
 
@@ -111,6 +129,16 @@ export class ExtractionRunStore {
       await mkdir(dirname(this.file), { recursive: true })
       this.db = new sqlite.DatabaseSync(this.file)
       this.db.exec(SCHEMA)
+      /**
+       * 旧库补 `stage` 列（2026-10-18 加）。
+       *
+       * `CREATE TABLE IF NOT EXISTS` 不会给**已存在**的表加列，所以升级必须在真库上跑一遍迁移：
+       * 否则读旧库时那条 SELECT 直接报错，历史轨迹一条也打不开。旧记录落到默认值
+       * `extract` —— 它们本来就只有抽取这一种。
+       */
+      const columns = this.db.prepare('PRAGMA table_info(extraction_runs)').all() as Array<{ name: string }>
+      if (!columns.some(column => column.name === 'stage')) this.db.exec("ALTER TABLE extraction_runs ADD COLUMN stage TEXT NOT NULL DEFAULT 'extract'")
+      this.db.exec('CREATE INDEX IF NOT EXISTS idx_extraction_runs_stage ON extraction_runs(kb, stage, created_at DESC)')
       return true
     } catch {
       this.db = null
@@ -122,15 +150,37 @@ export class ExtractionRunStore {
     return this.db !== null
   }
 
+  /** Shared audit connection; callers must not close it. */
+  connection(): SqliteDatabase {
+    if (!this.db) throw new Error('EXTRACTION_RUNS_UNAVAILABLE')
+    return this.db
+  }
+
   /** 落一条。报告整份序列化进 `report` 列。 */
   async append(entry: ExtractionRunRecord): Promise<void> {
     if (this.db === null) throw new Error('EXTRACTION_RUNS_UNAVAILABLE')
     this.db.prepare(
-      'INSERT INTO extraction_runs (id, kb, doc_id, title, actor, created_at, model_ok, model_why, summary, report) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+      'INSERT INTO extraction_runs (id, kb, doc_id, title, actor, created_at, model_ok, model_why, summary, report, stage) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
     ).run(
       entry.id, entry.kb, entry.docId, entry.title, entry.actor, entry.createdAt,
-      entry.modelOk ? 1 : 0, entry.modelWhy ?? null, entry.summary, JSON.stringify(entry.report ?? {}),
+      entry.modelOk ? 1 : 0, entry.modelWhy ?? null, entry.summary, JSON.stringify(entry.report ?? {}), entry.stage,
     )
+    /**
+     * 这张表才是审计库里真正会长大的那张（`report` 是整份模型返回的 JSON，一条几十到几百 KB）。
+     * 上限同样只裁「水位以上」的新行 —— 升级那一刻已有的轨迹一条不删（见 `retention.ts`）。
+     * 每 25 次落一条才做一次，避免每次写入都去数一遍。
+     *
+     * **按阶段各留各的窗口**（2026-10-18）：三段的条数差一个量级 —— 300 个分组的库跑一次
+     * 全链路，物化落 300 条，抽取只落 1 条。用一条全表 FIFO 上限的话，物化会把抽取挤出去，
+     * 而抽取那份是三者里最贵、最不可重建的（它是模型的原始返回）。所以每段各自计数。
+     */
+    this.appendsSincePrune += 1
+    if (this.appendsSincePrune >= 25) {
+      this.appendsSincePrune = 0
+      for (const stage of ['extract', 'normalize', 'materialize'] as const) {
+        try { applyRetention(this.db, `extraction_runs:${stage}`, { table: 'extraction_runs', key: 'rowid', keep: 2000, where: `stage = '${stage}'` }) } catch { /* 裁剪是尽力而为 */ }
+      }
+    }
   }
 
   /** 一条完整记录（含报告）。 */
@@ -146,12 +196,13 @@ export class ExtractionRunStore {
     const where: string[] = []
     const params: Record<string, unknown> = {}
     if (query.docId !== undefined) { where.push('doc_id = :docId'); params.docId = query.docId }
+    if (query.stage !== undefined) { where.push('stage = :stage'); params.stage = query.stage }
     const clause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : ''
     const total = this.db.prepare(`SELECT COUNT(*) AS n FROM extraction_runs ${clause}`).get(params) as { n: number }
     const limit = Math.min(100, Math.max(1, query.limit ?? 20))
     const offset = Math.max(0, query.offset ?? 0)
     const rows = this.db.prepare(
-      `SELECT id, kb, doc_id, title, actor, created_at, model_ok, model_why, summary, report <> '{}' AS has_report
+      `SELECT id, kb, doc_id, title, actor, created_at, model_ok, model_why, summary, stage, report <> '{}' AS has_report
        FROM extraction_runs ${clause} ORDER BY created_at DESC, id DESC LIMIT :limit OFFSET :offset`,
     ).all({ ...params, limit, offset }) as Array<Record<string, unknown>>
     return { runs: rows.map(rowToSummary), total: total.n }
@@ -197,6 +248,8 @@ function rowToSummary(row: Record<string, unknown>): ExtractionRunSummary {
     ...(row.model_why === null || row.model_why === undefined ? {} : { modelWhy: String(row.model_why) }),
     summary: String(row.summary ?? ''),
     hasReport: row.has_report === 1 || row.has_report === true,
+    // 旧库补列之前落下的行没有这一列（或为 null）：它们只可能是抽取。
+    stage: (row.stage === 'normalize' || row.stage === 'materialize' ? row.stage : 'extract') as TrajectoryStage,
   }
 }
 
@@ -210,7 +263,13 @@ function rowToRecord(row: Record<string, unknown>): ExtractionRunRecord {
   }
 }
 
-/** 一条轨迹的 id：时间戳（36 进制）+ 随机尾巴，排序即创建序。 */
-export function extractionRunId(): string {
-  return `run_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`
+/**
+ * 一条轨迹的 id：时间戳（36 进制）+ 随机尾巴，排序即创建序。
+ *
+ * `stage` 只影响前缀，不影响排序：id 自己写着它属于哪一段，日志里一眼分得清；
+ * 而**不可依赖**它做判别 —— 判别读 `stage` 列（前缀是给人看的，列才是真源）。
+ */
+export function extractionRunId(stage?: TrajectoryStage): string {
+  const prefix = stage === undefined ? 'run' : stage
+  return `${prefix}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`
 }

package/src/fs-atomic.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto'
 import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'
 import { dirname } from 'node:path'
 
@@ -9,11 +10,27 @@ import { dirname } from 'node:path'
  * `diagnosis/` 反过来依赖 runtime 的实现细节 —— 分层就倒过来了。
  */
 
+/** 进程内的单调计数，和随机尾巴一起保证同进程同毫秒的两次写不会用同一个临时文件。 */
+let temporaryCounter = 0
+
+/**
+ * 临时文件名。
+ *
+ * 曾经是 `${path}.${pid}.${Date.now().toString(36)}.tmp` —— 同一个进程、同一个目标、
+ * 同一毫秒就会撞名。撞上之后先完成的 `rename` 把文件吃掉，后完成的那次 `writeFile`
+ * 直接 ENOENT，表现为「整次 persist 抛错、内存改了但磁盘没落」。
+ * 计数 + 随机尾巴把这一条从「取决于时序」变成「不可能」。
+ */
+function temporaryPath(path: string): string {
+  temporaryCounter = (temporaryCounter + 1) % Number.MAX_SAFE_INTEGER
+  return `${path}.${process.pid}.${temporaryCounter.toString(36)}.${randomUUID().slice(0, 8)}.tmp`
+}
+
 export async function writeTextAtomic(path: string, content: string): Promise<void> {
   // 原子写需要一个与目标同目录的临时文件，所以目标目录必须先存在。0030 起
   // SourceRaw 现在平铺在 `<段>/` 下，确保段目录存在即可。
   await mkdir(dirname(path), { recursive: true })
-  const temporary = `${path}.${process.pid}.${Date.now().toString(36)}.tmp`
+  const temporary = temporaryPath(path)
   try {
     await writeFile(temporary, content, 'utf8')
     await rename(temporary, path)
@@ -24,7 +41,7 @@ export async function writeTextAtomic(path: string, content: string): Promise<vo
 
 export async function writeBufferAtomic(path: string, content: Buffer): Promise<void> {
   await mkdir(dirname(path), { recursive: true })
-  const temporary = `${path}.${process.pid}.${Date.now().toString(36)}.tmp`
+  const temporary = temporaryPath(path)
   try {
     await writeFile(temporary, content)
     await rename(temporary, path)

package/src/images.ts

@@ -4,7 +4,7 @@ import { basename, extname, join } from 'node:path'
 import { ASSETS_DIR, PUBLISHED_IMAGE_MARKER } from './layout.js'
 import { writeBufferAtomic, writeTextAtomic } from './fs-atomic.js'
 
-/** Maximum decoded image size accepted by `images.publish`. */
+/** Maximum decoded image size accepted by `image_publish`. */
 export const MAX_PUBLISHED_IMAGE_BYTES = 20 * 1024 * 1024
 
 const IMAGE_FORMATS = [
@@ -187,7 +187,7 @@ export async function publishKnowledgeImage(
   if (!existing?.equals(bytes)) await writeBufferAtomic(join(assetDir, storedFilename), bytes)
   // Standalone hosted images have no SourceRaw or card by design. Keep that intent beside the
   // asset so the orphan sweep does not mistake it for a disposable extraction artifact.
-  await writeTextAtomic(join(assetDir, PUBLISHED_IMAGE_MARKER), 'images.publish\n')
+  await writeTextAtomic(join(assetDir, PUBLISHED_IMAGE_MARKER), 'image_publish\n')
   const relativePath = `${ASSETS_DIR}/${bucket}/${storedFilename}`
   const url = `/knowledge/assets/${encodeURIComponent(kb)}/${relativePath.split('/').map(encodeURIComponent).join('/')}`
   return {