@yun-zero/claw-memory 0.1.0

package/src/services/memory.ts

@@ -0,0 +1,338 @@
+import Database from 'better-sqlite3';
+import { v4 as uuidv4 } from 'uuid';
+import { readFileSync } from 'fs';
+import { writeFile, mkdir } from 'fs/promises';
+import { dirname, join } from 'path';
+import { MemoryRepository, CreateMemoryInput } from '../db/repository.js';
+import { EntityRepository } from '../db/entityRepository.js';
+import { calculateWeight, DEFAULT_TIME_DECAY, type SearchOptions } from './retrieval.js';
+import { SummarizerService } from './summarizer.js';
+import { generateSummaryWithLLM } from '../config/llm.js';
+import { MetadataExtractor } from './metadataExtractor.js';
+import type { Memory, SaveMemoryInput, GetContextInput, TimeBucket, WeeklyReport, IntegratedSummary } from '../types.js';
+
+export class MemoryService {
+  private db: Database.Database;
+  private memoryRepo: MemoryRepository;
+  private entityRepo: EntityRepository;
+  private summarizer: SummarizerService;
+  private extractor: MetadataExtractor;
+  private dataDir: string;
+
+  constructor(db: Database.Database, dataDir: string = './memories') {
+    this.db = db;
+    this.memoryRepo = new MemoryRepository(db);
+    this.entityRepo = new EntityRepository(db);
+    this.summarizer = new SummarizerService(db);
+    this.extractor = new MetadataExtractor();
+    this.dataDir = dataDir;
+  }
+
+  getLatestIntegratedSummary(): IntegratedSummary | null {
+    return this.memoryRepo.getLatestIntegratedSummary();
+  }
+
+  async saveMemory(input: SaveMemoryInput): Promise<Memory> {
+    const { content, metadata } = input;
+
+    // 生成文件路径
+    const date = new Date();
+    const datePath = `${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, '0')}/${String(date.getDate()).padStart(2, '0')}`;
+    const fileName = `${uuidv4()}.md`;
+    const contentPath = join(this.dataDir, datePath, fileName);
+
+    // 保存内容到文件
+    await this.saveContentToFile(contentPath, content);
+
+    // 获取已有的整体摘要
+    const existingSummary = this.getLatestIntegratedSummary();
+
+    // 调用 LLM 提取元数据（传入已有摘要）
+    const extracted = content.trim()
+      ? await this.extractor.extract(content, existingSummary || undefined)
+      : { tags: [], keywords: [], subjects: [], importance: 0.5, summary: '', integratedSummary: existingSummary || { active_areas: [], key_topics: [], recent_summary: '' } };
+
+    // 合并：LLM 提取的覆盖用户传入的
+    const mergedMetadata = {
+      tags: extracted.tags,
+      keywords: extracted.keywords,
+      subjects: extracted.subjects,
+      importance: extracted.importance,
+      summary: extracted.summary,
+      ...metadata
+    };
+
+    // 创建记忆记录
+    const clampedImportance = this.clampImportance(mergedMetadata.importance);
+    const memoryInput: CreateMemoryInput = {
+      contentPath,
+      summary: mergedMetadata.summary || undefined,
+      integratedSummary: extracted.integratedSummary,
+      importance: clampedImportance ?? 0.5,
+      tokenCount: this.estimateTokens(content)
+    };
+
+    const memory = this.memoryRepo.create(memoryInput);
+
+    // 处理实体关联
+    await this.processEntities(memory.id, mergedMetadata);
+
+    return memory;
+  }
+
+  async searchMemory(options: SearchOptions): Promise<Memory[]> {
+    const { query, timeRange, tags, limit = 10 } = options;
+
+    // 构建时间过滤条件
+    const dateFilter = this.buildDateFilter(timeRange);
+
+    // 获取所有候选记忆
+    let memories = this.memoryRepo.findAll(100);
+
+    // 应用查询过滤（如果提供了查询字符串）
+    if (query && query.trim() !== '') {
+      const searchTerm = query.toLowerCase();
+      memories = memories.filter(m => {
+        const summaryMatch = m.summary?.toLowerCase().includes(searchTerm);
+        // 读取内容文件进行匹配
+        let contentMatch = false;
+        try {
+          const content = this.readContentFromFile(m.contentPath);
+          contentMatch = content.toLowerCase().includes(searchTerm);
+        } catch {
+          // 如果读取失败，忽略内容匹配
+        }
+        return summaryMatch || contentMatch;
+      });
+    }
+
+    // 应用时间过滤
+    if (dateFilter) {
+      memories = memories.filter(m => {
+        const created = m.createdAt.toISOString().split('T')[0];
+        return created >= dateFilter.start && created <= dateFilter.end;
+      });
+    }
+
+    // 计算权重并排序
+    const weightedMemories = memories.map(memory => ({
+      memory,
+      weight: calculateWeight({
+        entityMatch: 0,
+        timeDecay: DEFAULT_TIME_DECAY,
+        memoryDate: memory.createdAt.toISOString().split('T')[0],
+        tagMatch: 0,
+        importance: memory.importance
+      })
+    }));
+
+    // 按权重排序
+    weightedMemories.sort((a, b) => b.weight - a.weight);
+
+    // 返回结果
+    return weightedMemories.slice(0, limit).map(w => w.memory);
+  }
+
+  async getContext(input: GetContextInput): Promise<string> {
+    const { query, maxTokens = 8000 } = input;
+
+    const memories = await this.searchMemory({
+      query,
+      limit: 20,
+      maxTokens
+    });
+
+    // 累积内容直到达到 token 限制
+    let totalTokens = 0;
+    const contextParts: string[] = [];
+
+    for (const memory of memories) {
+      const content = this.readContentFromFile(memory.contentPath);
+      const tokens = this.estimateTokens(content);
+
+      if (totalTokens + tokens > maxTokens) {
+        break;
+      }
+
+      contextParts.push(content);
+      totalTokens += tokens;
+
+      // 更新访问计数
+      this.memoryRepo.updateLastAccessed(memory.id);
+    }
+
+    return contextParts.join('\n\n---\n\n');
+  }
+
+  async getSummary(period: 'day' | 'week' | 'month', date?: string): Promise<TimeBucket> {
+    const targetDate = date ? new Date(date) : new Date();
+    const { startDate, endDate } = this.calculatePeriodRange(period, targetDate);
+
+    // 聚合数据
+    const report: WeeklyReport = await this.summarizer.aggregateMemories(startDate, endDate);
+
+    // 生成 LLM 总结
+    try {
+      const reportString = this.summarizer.reportToString(report);
+      report.summary = await generateSummaryWithLLM(reportString);
+    } catch (error) {
+      console.warn('Failed to generate LLM summary:', error);
+      report.summary = '总结生成失败，请配置有效的 LLM API Key。';
+    }
+
+    // 保存到 time_buckets 表
+    this.saveTimeBucket(startDate, report);
+
+    return {
+      date: startDate,
+      memoryCount: report.basic.totalMemories,
+      summary: report.summary,
+      summaryGeneratedAt: new Date(),
+      keyTopics: report.topics.keyTopics,
+      createdAt: new Date()
+    };
+  }
+
+  private calculatePeriodRange(period: 'day' | 'week' | 'month', date: Date): { startDate: string; endDate: string } {
+    const endDate = date.toISOString().split('T')[0];
+    const startDate = new Date(date);
+
+    switch (period) {
+      case 'day':
+        // 返回当天
+        break;
+      case 'week':
+        startDate.setDate(startDate.getDate() - 7);
+        break;
+      case 'month':
+        startDate.setMonth(startDate.getMonth() - 1);
+        break;
+    }
+
+    return {
+      startDate: startDate.toISOString().split('T')[0],
+      endDate
+    };
+  }
+
+  private saveTimeBucket(date: string, report: WeeklyReport): void {
+    const stmt = this.db.prepare(`
+      INSERT OR REPLACE INTO time_buckets (date, memory_count, summary, summary_generated_at, key_topics, created_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `);
+
+    stmt.run(
+      date,
+      report.basic.totalMemories,
+      report.summary,
+      new Date().toISOString(),
+      JSON.stringify(report.topics.keyTopics),
+      new Date().toISOString()
+    );
+  }
+
+  private async saveContentToFile(path: string, content: string): Promise<void> {
+    const dir = dirname(path);
+    await mkdir(dir, { recursive: true });
+    await writeFile(path, content, 'utf-8');
+  }
+
+  private readContentFromFile(path: string): string {
+    try {
+      return readFileSync(path, 'utf-8');
+    } catch {
+      return '';
+    }
+  }
+
+  private async processEntities(memoryId: string, metadata: SaveMemoryInput['metadata']): Promise<void> {
+    // 处理标签
+    if (metadata.tags) {
+      for (const tag of metadata.tags) {
+        const entity = this.entityRepo.findOrCreate({
+          name: tag,
+          type: 'tag',
+          level: tag.split('/').length - 1
+        });
+
+        this.db.prepare(`
+          INSERT OR IGNORE INTO memory_entities (memory_id, entity_id, relevance)
+          VALUES (?, ?, ?)
+        `).run(memoryId, entity.id, 1.0);
+      }
+    }
+
+    // 处理主题
+    if (metadata.subjects) {
+      for (const subject of metadata.subjects) {
+        const entity = this.entityRepo.findOrCreate({
+          name: subject,
+          type: 'subject'
+        });
+
+        this.db.prepare(`
+          INSERT OR IGNORE INTO memory_entities (memory_id, entity_id, relevance)
+          VALUES (?, ?, ?)
+        `).run(memoryId, entity.id, 0.8);
+      }
+    }
+
+    // 处理关键词
+    if (metadata.keywords) {
+      for (const keyword of metadata.keywords) {
+        const entity = this.entityRepo.findOrCreate({
+          name: keyword,
+          type: 'keyword'
+        });
+
+        this.db.prepare(`
+          INSERT OR IGNORE INTO memory_entities (memory_id, entity_id, relevance)
+          VALUES (?, ?, ?)
+        `).run(memoryId, entity.id, 0.6);
+      }
+    }
+  }
+
+  private buildDateFilter(timeRange?: SearchOptions['timeRange']): { start: string; end: string } | null {
+    if (!timeRange || timeRange === 'all') return null;
+
+    const now = new Date();
+    const end = now.toISOString().split('T')[0];
+    let start: string;
+
+    switch (timeRange) {
+      case 'today':
+        start = end;
+        break;
+      case 'week':
+        now.setDate(now.getDate() - 7);
+        start = now.toISOString().split('T')[0];
+        break;
+      case 'month':
+        now.setMonth(now.getMonth() - 1);
+        start = now.toISOString().split('T')[0];
+        break;
+      case 'year':
+        now.setFullYear(now.getFullYear() - 1);
+        start = now.toISOString().split('T')[0];
+        break;
+      default:
+        return null;
+    }
+
+    return { start, end };
+  }
+
+  private clampImportance(value: number | undefined): number | undefined {
+    if (value === undefined) {
+      return undefined;
+    }
+    return Math.max(0, Math.min(1, value));
+  }
+
+  private estimateTokens(text: string): number {
+    const chineseChars = (text.match(/[\u4e00-\u9fa5]/g) || []).length;
+    const otherChars = text.length - chineseChars;
+    return Math.ceil(chineseChars / 1.5 + otherChars / 4);
+  }
+}

package/src/services/memoryIndex.ts

@@ -0,0 +1,140 @@
+import Database from 'better-sqlite3';
+import { TodoRepository } from '../db/todoRepository.js';
+import { MemoryRepository } from '../db/repository.js';
+import { EntityRepository } from '../db/entityRepository.js';
+import type { IntegratedSummary } from '../types.js';
+
+export interface MemoryIndexOptions {
+  period: 'day' | 'week' | 'month';
+  date?: string;
+  includeTodos?: boolean;
+  includeRecent?: boolean;
+  recentLimit?: number;
+}
+
+export interface MemoryIndex {
+  period: { start: string; end: string };
+  activeAreas: {
+    tags: { name: string; count: number }[];
+    keywords: string[];
+  };
+  todos: { id: string; content: string; period: string }[];
+  recentActivity: { date: string; summary: string }[];
+  integratedSummary?: IntegratedSummary;
+}
+
+export async function getMemoryIndex(db: Database.Database, options: MemoryIndexOptions): Promise<MemoryIndex> {
+  const { startDate, endDate } = calculatePeriodRange(options.period, options.date);
+
+  // 并行获取活跃领域
+  const [tags, keywords] = await Promise.all([
+    Promise.resolve(getTopTags(db, startDate, endDate, 10)),
+    Promise.resolve(getTopKeywords(db, startDate, endDate, 10))
+  ]);
+
+  // 并行获取待办和最近动态
+  const [todos, recentActivity] = await Promise.all([
+    options.includeTodos
+      ? getTodos(db, options.period, endDate)
+      : Promise.resolve([]),
+    options.includeRecent
+      ? getRecentActivity(db, startDate, endDate, options.recentLimit || 5)
+      : Promise.resolve([])
+  ]);
+
+  // 获取缓存的整体摘要（从最新的 memory）
+  const memoryRepo = new MemoryRepository(db);
+  const rawSummary = memoryRepo.getLatestIntegratedSummary();
+  // Convert null to undefined to match the interface type
+  const integratedSummary = rawSummary === null ? undefined : rawSummary;
+
+  return {
+    period: { start: startDate, end: endDate },
+    activeAreas: { tags, keywords },
+    todos,
+    recentActivity,
+    integratedSummary
+  };
+}
+
+async function getTodos(db: Database.Database, period: string, endDate: string) {
+  const todoRepo = new TodoRepository(db);
+  const allTodos = todoRepo.findByPeriod(period, endDate);
+  return allTodos
+    .filter(t => !t.completedAt)
+    .map(t => ({ id: t.id, content: t.content, period: t.period }));
+}
+
+async function getRecentActivity(db: Database.Database, startDate: string, endDate: string, limit: number) {
+  const recentMemories = findMemoriesByDateRange(db, startDate, endDate, limit);
+  return recentMemories.map(m => ({
+    date: m.createdAt.toISOString().split('T')[0],
+    summary: m.summary || ''
+  }));
+}
+
+function calculatePeriodRange(period: string, date?: string) {
+  const endDate = date || new Date().toISOString().split('T')[0];
+  const startDate = new Date(endDate);
+
+  switch (period) {
+    case 'day': break;
+    case 'week': startDate.setDate(startDate.getDate() - 7); break;
+    case 'month': startDate.setMonth(startDate.getMonth() - 1); break;
+  }
+
+  return {
+    startDate: startDate.toISOString().split('T')[0],
+    endDate
+  };
+}
+
+function getTopTags(db: Database.Database, startDate: string, endDate: string, limit: number) {
+  const rows = db.prepare(`
+    SELECT e.name, COUNT(me.memory_id) as count
+    FROM memory_entities me
+    JOIN entities e ON me.entity_id = e.id
+    JOIN memories m ON me.memory_id = m.id
+    WHERE e.type = 'tag' AND date(m.created_at) BETWEEN ? AND ?
+    GROUP BY e.id ORDER BY count DESC LIMIT ?
+  `).all(startDate, endDate, limit) as { name: string; count: number }[];
+
+  return rows;
+}
+
+function getTopKeywords(db: Database.Database, startDate: string, endDate: string, limit: number) {
+  const rows = db.prepare(`
+    SELECT e.name FROM memory_entities me
+    JOIN entities e ON me.entity_id = e.id
+    JOIN memories m ON me.memory_id = m.id
+    WHERE e.type = 'keyword' AND date(m.created_at) BETWEEN ? AND ?
+    GROUP BY e.id ORDER BY COUNT(*) DESC LIMIT ?
+  `).all(startDate, endDate, limit) as { name: string }[];
+  return rows.map(r => r.name);
+}
+
+interface MemoryRow {
+  id: string;
+  contentPath: string;
+  summary: string | null;
+  createdAt: Date;
+  importance: number;
+}
+
+function findMemoriesByDateRange(db: Database.Database, startDate: string, endDate: string, limit: number): MemoryRow[] {
+  const rows = db.prepare(`
+    SELECT id, content_path, summary, created_at, importance
+    FROM memories
+    WHERE date(created_at) BETWEEN ? AND ?
+    ORDER BY created_at DESC
+    LIMIT ?
+  `).all(startDate, endDate, limit) as any[];
+
+  return rows.map(row => ({
+    id: row.id,
+    contentPath: row.content_path,
+    summary: row.summary,
+    createdAt: new Date(row.created_at),
+    importance: row.importance
+  }));
+}

package/src/services/metadataExtractor.ts

@@ -0,0 +1,89 @@
+import { generateSummaryWithLLM } from '../config/llm.js';
+import type { IntegratedSummary } from '../types.js';
+
+export interface ExtractedMetadata {
+  tags: string[];
+  keywords: string[];
+  subjects: string[];
+  importance: number;
+  summary: string;
+  integratedSummary: IntegratedSummary;
+}
+
+const EXTRACTION_PROMPT = `请从以下对话内容中提取结构化元数据，并整合已有的整体摘要：
+
+当前对话内容：
+{content}
+
+已有整体摘要（请在此基础上增量更新）：
+{existing_summary}
+
+请以 JSON 格式返回：
+{
+  "tags": ["一级分类/二级分类"],
+  "keywords": ["关键词1", "关键词2"],
+  "subjects": ["主题1"],
+  "importance": 0.0-1.0,
+  "summary": "当前对话的一句话摘要",
+  "integrated_summary": {
+    "active_areas": ["领域名 (出现次数)"],
+    "key_topics": ["主题1", "主题2"],
+    "recent_summary": "整体摘要自然语言描述"
+  }
+}
+
+注意：
+- tags 使用层级结构
+- integrated_summary 需整合历史信息，在已有基础上增加新领域
+- 只返回 JSON，不要其他内容`;
+
+export class MetadataExtractor {
+  async extract(content: string, existingSummary?: IntegratedSummary): Promise<ExtractedMetadata> {
+    let existingSummaryText = '无';
+    if (existingSummary) {
+      existingSummaryText = `活跃领域: ${existingSummary.active_areas.join(', ')}\n关键词: ${existingSummary.key_topics.join(', ')}\n近期摘要: ${existingSummary.recent_summary}`;
+    }
+
+    const prompt = EXTRACTION_PROMPT
+      .replace('{content}', content)
+      .replace('{existing_summary}', existingSummaryText);
+
+    try {
+      const response = await generateSummaryWithLLM(prompt);
+      return this.parseLLMResponse(response);
+    } catch (error) {
+      console.warn('LLM extraction failed:', error);
+      return this.fallbackExtract(content);
+    }
+  }
+
+  async callLLM(prompt: string): Promise<ExtractedMetadata> {
+    const response = await generateSummaryWithLLM(prompt);
+    return this.parseLLMResponse(response);
+  }
+
+  private parseLLMResponse(response: string): ExtractedMetadata {
+    try {
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        return JSON.parse(jsonMatch[0]);
+      }
+    } catch {}
+    return this.fallbackExtract('');
+  }
+
+  private fallbackExtract(content: string): ExtractedMetadata {
+    return {
+      tags: [],
+      keywords: [],
+      subjects: [],
+      importance: 0.5,
+      summary: content.substring(0, 100),
+      integratedSummary: {
+        active_areas: [],
+        key_topics: [],
+        recent_summary: ''
+      }
+    };
+  }
+}

package/src/services/retrieval.ts

@@ -0,0 +1,71 @@
+import type { Memory, Entity } from '../types.js';
+
+export interface TimeDecayConfig {
+  today: number;
+  week: number;
+  month: number;
+  year: number;
+  older: number;
+}
+
+export interface WeightInput {
+  entityMatch: number;
+  timeDecay: TimeDecayConfig;
+  memoryDate: string;
+  tagMatch: number;
+  importance: number;
+}
+
+export const DEFAULT_TIME_DECAY: TimeDecayConfig = {
+  today: 30,
+  week: 20,
+  month: 10,
+  year: 5,
+  older: 0
+};
+
+export function calculateWeight(input: WeightInput): number {
+  const { entityMatch, timeDecay, memoryDate, tagMatch, importance } = input;
+
+  // 实体匹配权重 (0-40)
+  const entityWeight = Math.min(entityMatch * 10, 40);
+
+  // 时间衰减权重 (0-30)
+  const timeWeight = getTimeWeight(memoryDate, timeDecay);
+
+  // 标签层级权重 (0-20)
+  const tagWeight = Math.min(tagMatch * 2, 20);
+
+  // 重要性权重 (0-10)
+  const importanceWeight = importance * 10;
+
+  // 总分 = 实体匹配 × 0.4 + 时间衰减 × 0.3 + 标签层级 × 0.2 + 重要性 × 0.1
+  // 归一化到 0-100
+  const total =
+    entityWeight * 0.4 +
+    timeWeight * 0.3 +
+    tagWeight * 0.2 +
+    importanceWeight * 0.1;
+
+  return Math.round(total * 10) / 10;
+}
+
+function getTimeWeight(memoryDate: string, config: TimeDecayConfig): number {
+  const today = new Date();
+  const memory = new Date(memoryDate);
+  const diffDays = Math.floor((today.getTime() - memory.getTime()) / (1000 * 60 * 60 * 24));
+
+  if (diffDays === 0) return config.today;
+  if (diffDays <= 7) return config.week;
+  if (diffDays <= 30) return config.month;
+  if (diffDays <= 365) return config.year;
+  return config.older;
+}
+
+export interface SearchOptions {
+  query: string;
+  timeRange?: 'today' | 'week' | 'month' | 'year' | 'all';
+  tags?: string[];
+  limit?: number;
+  maxTokens?: number;
+}