@morningljn/mnemo 0.2.0 → 0.2.1

package/src/store.ts

@@ -12,9 +12,9 @@
 import Database from 'better-sqlite3'
 import type { Statement } from 'better-sqlite3'
 import { mkdirSync } from 'node:fs'
-import { dirname } from 'node:path'
+import { dirname, join } from 'node:path'
 import { SCHEMA } from './schema.js'
-import type { Fact, FactCategory } from './types.js'
+import type { Fact, FactCategory, DreamReport } from './types.js'
 
 // 信任评分常量
 const HELPFUL_DELTA = 0.05
@@ -136,10 +136,10 @@ export class MemoryStore {
         DROP TRIGGER IF EXISTS facts_ad;
         DROP TRIGGER IF EXISTS facts_au;
       `)
-      // 重建 FTS5 虚拟表（含 summary）
+      // 重建 FTS5 虚拟表（含 summary，使用 trigram tokenizer 支持中文子串）
       this.db.exec(`
         CREATE VIRTUAL TABLE facts_fts
-          USING fts5(content, tags, summary, content=facts, content_rowid=fact_id);
+          USING fts5(content, tags, summary, content=facts, content_rowid=fact_id, tokenize='trigram');
       `)
       // 重填充
       this.db.exec(`
@@ -743,7 +743,9 @@ export class MemoryStore {
 
       // Rate-based adjustment (需要 30+ 次检索)
       if (row.retrieval_count > 30) {
-        if (rate < 0.05) {
+        // 高频检索保护：检索 >100 次说明被持续需要，feedback 少不代表无用
+        const highFrequencyProtected = row.retrieval_count > 100
+        if (rate < 0.05 && !highFrequencyProtected) {
           const newTrust = clampTrust(row.trust_score * 0.9)
           this.db.prepare('UPDATE facts SET trust_score = ? WHERE fact_id = ?').run(newTrust, row.fact_id)
           demoted++
@@ -839,6 +841,165 @@ export class MemoryStore {
     }
   }
 
+  /** Dream 前备份数据库（使用 better-sqlite3 内置备份，安全处理 WAL 模式） */
+  async backupDatabase(): Promise<string> {
+    const dbDir = dirname(this.db.name)
+    const backupDir = join(dbDir, 'backup')
+    mkdirSync(backupDir, { recursive: true })
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)
+    const backupPath = join(backupDir, `dream-${timestamp}.db`)
+    await this.db.backup(backupPath)
+    return backupPath
+  }
+
+  /** 合并同 category 内 Jaccard > 0.6 的重叠 fact */
+  mergeOverlappingFacts(): { merged: number; details: Array<{ kept: number; removed: number; similarity: number }> } {
+    return this.db.transaction(() => {
+      const categories: FactCategory[] = ['identity', 'coding_style', 'tool_pref', 'workflow', 'general']
+      let merged = 0
+      const details: Array<{ kept: number; removed: number; similarity: number }> = []
+
+      for (const cat of categories) {
+        const rows = this.db.prepare(
+          'SELECT fact_id, content, retrieval_count FROM facts WHERE category = ? ORDER BY trust_score DESC'
+        ).all(cat) as Array<{ fact_id: number; content: string; retrieval_count: number }>
+
+        const removed = new Set<number>()
+
+        for (let i = 0; i < rows.length; i++) {
+          if (removed.has(rows[i].fact_id)) continue
+          const tokensA = this.tokenizeForDedup(rows[i].content)
+
+          for (let j = i + 1; j < rows.length; j++) {
+            if (removed.has(rows[j].fact_id)) continue
+            const tokensB = this.tokenizeForDedup(rows[j].content)
+            const sim = this.jaccardSimilarity(tokensA, tokensB)
+
+            if (sim > 0.6) {
+              const aHighFreq = rows[i].retrieval_count > 100
+              const bHighFreq = rows[j].retrieval_count > 100
+
+              if (aHighFreq && bHighFreq) continue
+
+              let keptId: number, removedId: number
+              if (bHighFreq) {
+                keptId = rows[j].fact_id
+                removedId = rows[i].fact_id
+              } else {
+                keptId = rows[i].fact_id
+                removedId = rows[j].fact_id
+              }
+
+              this.removeFact(removedId)
+              removed.add(removedId)
+              details.push({ kept: keptId, removed: removedId, similarity: Math.round(sim * 100) / 100 })
+              merged++
+              // 外层 fact 被删除时停止内层循环，避免幽灵操作
+              if (removedId === rows[i].fact_id) break
+            }
+          }
+        }
+      }
+      return { merged, details }
+    })()
+  }
+
+  /** 分类修正：按关键词规则表将误分类的 fact 挪到正确 category */
+  reclassifyFacts(): number {
+    return this.db.transaction(() => {
+      const rules: Array<{ keywords: string[]; target: FactCategory }> = [
+        { keywords: ['角色设定', '暖暖', '身份', '编程女朋友', '暖宝宝'], target: 'identity' },
+        { keywords: ['编码规范', '代码风格', 'pytest', '文件不超过', '方法不超过'], target: 'coding_style' },
+        { keywords: ['工作流', 'OpenSpec', 'writing-plans', 'subagent'], target: 'workflow' },
+        { keywords: ['偏好', 'VS Code', '编辑器', 'IDE', '快捷键'], target: 'tool_pref' },
+      ]
+
+      const rows = this.db.prepare(
+        'SELECT fact_id, content, category FROM facts'
+      ).all() as Array<{ fact_id: number; content: string; category: string }>
+
+      let reclassified = 0
+      for (const row of rows) {
+        for (const rule of rules) {
+          if (rule.target === row.category) continue
+          if (rule.keywords.some(kw => row.content.includes(kw))) {
+            this.db.prepare("UPDATE facts SET category = ?, updated_at = datetime('now', 'localtime') WHERE fact_id = ?").run(rule.target, row.fact_id)
+            reclassified++
+            break
+          }
+        }
+      }
+      return reclassified
+    })()
+  }
+
+  /** 执行完整 dream cycle：备份 → 压缩 → 合并 → 重分类 → 报告 */
+  async runDream(options?: { skipBackup?: boolean }): Promise<DreamReport> {
+    if (!options?.skipBackup) {
+      await this.backupDatabase()
+    }
+
+    const compressed = this.compressLongFacts()
+    const mergeResult = this.mergeOverlappingFacts()
+    const reclassified = this.reclassifyFacts()
+
+    const stats = this.db.prepare(`
+      SELECT COUNT(*) as total,
+             AVG(trust_score) as avg_trust,
+             AVG(length(content)) as avg_length
+      FROM facts
+    `).get() as { total: number; avg_trust: number; avg_length: number }
+
+    const categories: FactCategory[] = ['identity', 'coding_style', 'tool_pref', 'workflow', 'general']
+    const coverage: Record<string, number> = {}
+    for (const cat of categories) {
+      const row = this.db.prepare('SELECT COUNT(*) as c FROM facts WHERE category = ?').get(cat) as { c: number }
+      coverage[cat] = row.c
+    }
+
+    return {
+      merged: mergeResult.merged,
+      compressed,
+      reclassified,
+      deleted: mergeResult.merged,
+      mergeDetails: mergeResult.details,
+      health: {
+        total: stats.total,
+        avg_trust: Math.round((stats.avg_trust ?? 0) * 100) / 100,
+        avg_length: Math.round(stats.avg_length ?? 0),
+        coverage: coverage as Record<FactCategory, number>,
+      },
+    }
+  }
+
+  /** 压缩长 fact：content > 200 字且无 summary 的，自动提取前 2 句作为 summary */
+  compressLongFacts(): number {
+    const rows = this.db.prepare(
+      "SELECT fact_id, content FROM facts WHERE length(content) > 200 AND (summary IS NULL OR summary = '')"
+    ).all() as Array<{ fact_id: number; content: string }>
+
+    let compressed = 0
+    for (const row of rows) {
+      const summary = this.extractSummary(row.content)
+      if (summary) {
+        this.db.prepare('UPDATE facts SET summary = ? WHERE fact_id = ?').run(summary, row.fact_id)
+        compressed++
+      }
+    }
+    return compressed
+  }
+
+  /** 从 content 提取前 2 个完整句子（总长 ≤ 150 字） */
+  private extractSummary(content: string): string | null {
+    const sentences = content.split(/[。\n.]/).map(s => s.trim()).filter(s => s.length > 0)
+    if (sentences.length === 0) return null
+    let summary = sentences[0]
+    if (sentences.length > 1 && summary.length + sentences[1].length <= 148) {
+      summary += '。' + sentences[1]
+    }
+    return summary.length <= 150 ? summary : summary.slice(0, 147) + '...'
+  }
+
   /** 获取数据库连接（供 FactRetriever 直接使用） */
   get connection(): Database.Database {
     return this.db

package/src/types.ts

@@ -55,7 +55,7 @@ export interface RetrieverOptions {
 
 /** fact_store 工具调用参数 */
 export interface FactStoreArgs {
-  action: 'add' | 'search' | 'probe' | 'related' | 'reason' | 'contradict' | 'update' | 'remove' | 'list' | 'learn' | 'audit'
+  action: 'add' | 'search' | 'probe' | 'related' | 'reason' | 'contradict' | 'update' | 'remove' | 'list' | 'learn' | 'audit' | 'dream'
   content?: string | string[]
   query?: string
   entity?: string
@@ -82,3 +82,27 @@ export interface SecurityScanResult {
   hasPii: boolean
   injectionAttempts: string[]
 }
+
+/** Dream 整理报告 */
+export interface DreamReport {
+  merged: number
+  compressed: number
+  reclassified: number
+  deleted: number
+  mergeDetails: Array<{ kept: number; removed: number; similarity: number }>
+  health: {
+    total: number
+    avg_trust: number
+    avg_length: number
+    coverage: Record<FactCategory, number>
+  }
+}
+
+/** 精简搜索结果 */
+export interface CompactFactResult {
+  factId: number
+  display: string
+  category: FactCategory
+  trustScore: number
+  score: number
+}

package/tests/resource.test.ts

@@ -21,42 +21,44 @@ afterEach(() => {
 })
 
 describe('ResourceManager', () => {
-  it('returns empty array for empty category', () => {
-    const facts = manager.getFacts('identity')
-    expect(facts).toEqual([])
+  it('returns empty markdown for empty category', () => {
+    const text = readResource('identity')
+    expect(text).toContain('身份与行为设定')
+    expect(text).not.toContain('## 你的身份')
   })
 
-  it('returns facts ordered by trust score', () => {
+  it('formats identity facts as instructions', () => {
+    store.addFact('AI角色设定：大名暖暖，身份是用户的编程助手', 'identity')
+    const text = readResource('identity')
+    expect(text).toContain('你的身份')
+    expect(text).toContain('暖暖')
+  })
+
+  it('formats non-identity facts as reference', () => {
     store.addFact('用户偏好深色主题', 'tool_pref')
-    store.addFact('用户喜欢 VS Code', 'tool_pref')
-    const facts = manager.getFacts('tool_pref')
-    expect(facts.length).toBe(2)
-    // listFacts returns trust_score DESC, first fact added gets default trust
-    expect(facts.length).toBeGreaterThan(0)
+    const text = readResource('tool_pref')
+    expect(text).toContain('工具偏好')
+    expect(text).toContain('深色主题')
   })
 
   it('caches results', () => {
     store.addFact('测试事实', 'general')
-    manager.getFacts('general')
+    readResource('general')
+    expect(manager.cacheSize()).toBe(1)
+    readResource('general')
     expect(manager.cacheSize()).toBe(1)
-    // Second call should hit cache
-    const facts2 = manager.getFacts('general')
-    expect(facts2.length).toBe(1)
   })
 
-  it('invalidates cache on write', () => {
+  it('invalidates cache', () => {
     store.addFact('测试事实', 'general')
-    manager.getFacts('general')
+    readResource('general')
     expect(manager.cacheSize()).toBe(1)
     manager.invalidate()
     expect(manager.cacheSize()).toBe(0)
   })
-
-  it('limits to top 10 facts', () => {
-    for (let i = 0; i < 15; i++) {
-      store.addFact(`事实 ${i}`, 'general')
-    }
-    const facts = manager.getFacts('general')
-    expect(facts.length).toBe(10)
-  })
 })
+
+function readResource(category: string): string {
+  const result = (manager as any).readCategory(category)
+  return result.contents[0].text
+}

package/tests/store.test.ts

@@ -153,15 +153,31 @@ describe('logRetrieval', () => {
 })
 
 describe('runLearning', () => {
-  it('demotes high retrieval low helpful facts', () => {
+  it('demotes moderate retrieval low helpful facts', () => {
     const id = store.addFact('demote me', 'general')
-    store.connection.prepare('UPDATE facts SET retrieval_count = 100, helpful_count = 2, trust_score = 1.0 WHERE fact_id = ?').run(id)
+    store.connection.prepare('UPDATE facts SET retrieval_count = 50, helpful_count = 1, trust_score = 1.0 WHERE fact_id = ?').run(id)
     const result = store.runLearning()
     const row = store.connection.prepare('SELECT trust_score FROM facts WHERE fact_id = ?').get(id) as any
     expect(row.trust_score).toBeLessThan(1.0)
     expect(result.demoted).toBeGreaterThanOrEqual(1)
   })
 
+  it('protects high frequency facts from demotion', () => {
+    const id = store.addFact('高频角色设定', 'identity')
+    store.connection.prepare('UPDATE facts SET retrieval_count = 200, helpful_count = 2, trust_score = 0.9 WHERE fact_id = ?').run(id)
+    store.runLearning()
+    const row = store.connection.prepare('SELECT trust_score FROM facts WHERE fact_id = ?').get(id) as any
+    expect(row.trust_score).toBe(0.9)
+  })
+
+  it('still demotes moderate frequency facts with low trust', () => {
+    const id = store.addFact('中频低信任', 'general')
+    store.connection.prepare('UPDATE facts SET retrieval_count = 50, helpful_count = 0, trust_score = 0.3 WHERE fact_id = ?').run(id)
+    store.runLearning()
+    const row = store.connection.prepare('SELECT trust_score FROM facts WHERE fact_id = ?').get(id) as any
+    expect(row.trust_score).toBeLessThan(0.3)
+  })
+
   it('promotes high helpful rate facts', () => {
     const id = store.addFact('promote me', 'general')
     store.connection.prepare('UPDATE facts SET retrieval_count = 50, helpful_count = 20, trust_score = 0.5 WHERE fact_id = ?').run(id)
@@ -214,3 +230,114 @@ describe('runAudit', () => {
     expect(report.long_without_summary.length).toBeGreaterThanOrEqual(1)
   })
 })
+
+describe('dream - backup', () => {
+  it('creates backup before dream', async () => {
+    store.addFact('test fact for backup', 'general')
+    const result = await store.backupDatabase()
+    expect(result).toBeTruthy()
+    expect(result).toContain('dream-')
+    expect(result).toContain('.db')
+  })
+})
+
+describe('dream - compress', () => {
+  it('generates summary for long facts without summary', () => {
+    const longContent = '用户偏好使用 TypeScript 开发前端项目。偏好 React 框架进行组件化开发。' + '额外补充说明'.repeat(50)
+    store.addFact(longContent, 'coding_style')
+    const result = store.compressLongFacts()
+    expect(result).toBeGreaterThanOrEqual(1)
+    const row = store.connection.prepare('SELECT summary FROM facts WHERE content = ?').get(longContent) as any
+    expect(row.summary).toBeTruthy()
+    expect(row.summary.length).toBeLessThanOrEqual(150)
+    expect(row.summary).toContain('TypeScript')
+  })
+
+  it('skips facts with existing summary', () => {
+    const longContent = 'x'.repeat(300)
+    const id = store.addFact(longContent, 'general')
+    store.connection.prepare('UPDATE facts SET summary = ? WHERE fact_id = ?').run('existing summary', id)
+    const result = store.compressLongFacts()
+    expect(result).toBe(0)
+    const row = store.connection.prepare('SELECT summary FROM facts WHERE fact_id = ?').get(id) as any
+    expect(row.summary).toBe('existing summary')
+  })
+
+  it('skips short facts', () => {
+    store.addFact('short fact', 'general')
+    const result = store.compressLongFacts()
+    expect(result).toBe(0)
+  })
+})
+
+describe('dream - merge', () => {
+  it('merges overlapping facts in same category', () => {
+    store.addFact('用户偏好使用 TypeScript 编写前端代码', 'coding_style')
+    store.addFact('用户偏好使用 TypeScript 编写后端代码', 'coding_style')
+    const result = store.mergeOverlappingFacts()
+    expect(result.merged).toBeGreaterThanOrEqual(1)
+    expect(result.details.length).toBeGreaterThanOrEqual(1)
+  })
+
+  it('protects high frequency facts from deletion', () => {
+    const id1 = store.addFact('用户偏好使用 TypeScript 编写前端代码', 'coding_style')
+    const id2 = store.addFact('用户偏好使用 TypeScript 编写前端代码扩展', 'coding_style')
+    store.connection.prepare('UPDATE facts SET retrieval_count = 200 WHERE fact_id = ?').run(id1)
+    const result = store.mergeOverlappingFacts()
+    const kept = store.connection.prepare('SELECT fact_id FROM facts WHERE fact_id = ?').get(id1) as any
+    expect(kept).toBeTruthy()
+  })
+
+  it('does not merge facts across categories', () => {
+    store.addFact('用户偏好使用 TypeScript 编写前端代码', 'coding_style')
+    store.addFact('用户偏好使用 TypeScript 编写前端代码', 'general')
+    const result = store.mergeOverlappingFacts()
+    expect(result.merged).toBe(0)
+  })
+
+  it('does not merge when both facts are high frequency', () => {
+    const id1 = store.addFact('高频测试事实 AAA 长内容匹配', 'coding_style')
+    const id2 = store.addFact('高频测试事实 BBB 长内容匹配', 'coding_style')
+    store.connection.prepare('UPDATE facts SET retrieval_count = 200 WHERE fact_id IN (?, ?)').run(id1, id2)
+    const result = store.mergeOverlappingFacts()
+    // Both facts should still exist
+    const row1 = store.connection.prepare('SELECT fact_id FROM facts WHERE fact_id = ?').get(id1) as any
+    const row2 = store.connection.prepare('SELECT fact_id FROM facts WHERE fact_id = ?').get(id2) as any
+    expect(row1).toBeTruthy()
+    expect(row2).toBeTruthy()
+  })
+})
+
+describe('dream - reclassify', () => {
+  it('moves miscategorized facts by keywords', () => {
+    const id = store.addFact('编码规范：文件不超过 500 行', 'identity')
+    const result = store.reclassifyFacts()
+    expect(result).toBeGreaterThanOrEqual(1)
+    const row = store.connection.prepare('SELECT category FROM facts WHERE fact_id = ?').get(id) as any
+    expect(row.category).toBe('coding_style')
+  })
+
+  it('skips correctly categorized facts', () => {
+    store.addFact('用户偏好使用 VS Code 编辑器', 'tool_pref')
+    const result = store.reclassifyFacts()
+    expect(result).toBe(0)
+  })
+})
+
+describe('dream - runDream', () => {
+  it('runs full dream cycle and returns report', async () => {
+    // 长文 fact（触发压缩）
+    store.addFact('用户偏好使用 TypeScript 开发前端。使用 React 框架。' + 'x'.repeat(250), 'coding_style')
+    // 重叠 fact（触发合并）
+    store.addFact('用户偏好使用 TypeScript 开发前端代码', 'coding_style')
+    // 分类错误 fact（触发重分类）
+    store.addFact('编码规范：文件不超过 500 行', 'identity')
+
+    const report = await store.runDream({ skipBackup: true })
+    expect(report.compressed).toBeGreaterThanOrEqual(0)
+    expect(report.merged).toBeGreaterThanOrEqual(0)
+    expect(report.reclassified).toBeGreaterThanOrEqual(0)
+    expect(report.health.total).toBeGreaterThanOrEqual(1)
+    expect(report.health.coverage).toBeTruthy()
+  })
+})