openclaw-mem 1.0.0

package/lib/context-builder.js

@@ -0,0 +1,401 @@
+/**
+ * OpenClaw-Mem Context Builder
+ * Generates context to inject into new sessions using progressive disclosure
+ */
+
+import database from './database.js';
+
+// Token estimation (4 chars ≈ 1 token)
+const CHARS_PER_TOKEN = 4;
+
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(String(text).length / CHARS_PER_TOKEN);
+}
+
+// Type emoji mapping
+const TYPE_EMOJI = {
+  'Edit': '📝',
+  'Write': '✏️',
+  'Read': '📖',
+  'Bash': '💻',
+  'Grep': '🔍',
+  'Glob': '📁',
+  'WebFetch': '🌐',
+  'WebSearch': '🔎',
+  'Task': '🤖',
+  'default': '🔵'
+};
+
+function getTypeEmoji(toolName) {
+  return TYPE_EMOJI[toolName] || TYPE_EMOJI.default;
+}
+
+// Format timestamp
+function formatTime(timestamp) {
+  if (!timestamp) return '';
+  const date = new Date(timestamp);
+  return date.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
+}
+
+function formatDate(timestamp) {
+  if (!timestamp) return '';
+  const date = new Date(timestamp);
+  return date.toISOString().split('T')[0];
+}
+
+// Filter out low-value observations (like recall test queries)
+function filterHighValueObservations(observations) {
+  const lowValuePatterns = [
+    '请查看 SESSION-MEMORY',
+    'SESSION-MEMORY.md 里没有',
+    '请查看 SESSION-MEMORY.md，告诉我',
+    '记忆检索当前不可用',
+    '/memory search',
+    '/memory get',
+    '之前没有记录到',
+    '没有任何关于'
+  ];
+
+  return observations.filter(obs => {
+    const summary = obs.summary || '';
+    // Always filter out observations matching low-value patterns
+    const isLowValue = lowValuePatterns.some(pattern => summary.includes(pattern));
+    if (isLowValue) return false;
+
+    // Keep observations that have actual content (not just metadata)
+    return true;
+  });
+}
+
+// Group observations by date
+function groupByDate(observations) {
+  const groups = {};
+  for (const obs of observations) {
+    const date = formatDate(obs.timestamp);
+    if (!groups[date]) {
+      groups[date] = [];
+    }
+    groups[date].push(obs);
+  }
+  return groups;
+}
+
+// Build index table (Layer 1 - compact)
+function buildIndexTable(observations) {
+  if (!observations || observations.length === 0) {
+    return '*(No recent observations)*';
+  }
+
+  const grouped = groupByDate(observations);
+  const lines = [];
+
+  for (const [date, obs] of Object.entries(grouped)) {
+    lines.push(`### ${date}`);
+    lines.push('');
+    lines.push('| ID | Time | T | Summary | Tokens |');
+    lines.push('|----|------|---|---------|--------|');
+
+    for (const o of obs) {
+      const id = `#${o.id}`;
+      const time = formatTime(o.timestamp);
+      const emoji = getTypeEmoji(o.tool_name);
+      const summary = o.summary || `${o.tool_name} operation`;
+      const truncSummary = summary.length > 50 ? summary.slice(0, 47) + '...' : summary;
+      const tokens = `~${o.tokens_read || estimateTokens(summary)}`;
+
+      lines.push(`| ${id} | ${time} | ${emoji} | ${truncSummary} | ${tokens} |`);
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// Build full details (Layer 3 - expensive)
+function buildFullDetails(observations, limit = 5) {
+  if (!observations || observations.length === 0) {
+    return '';
+  }
+
+  const toShow = observations.slice(0, limit);
+  const lines = [];
+
+  for (const o of toShow) {
+    lines.push(`#### #${o.id} - ${o.tool_name}`);
+    lines.push('');
+
+    if (o.summary) {
+      lines.push(`**Summary**: ${o.summary}`);
+      lines.push('');
+    }
+
+    // Show key facts from tool input
+    const input = o.tool_input || {};
+    const facts = [];
+
+    if (input.file_path) facts.push(`- File: \`${input.file_path}\``);
+    if (input.command) facts.push(`- Command: \`${input.command.slice(0, 100)}\``);
+    if (input.pattern) facts.push(`- Pattern: \`${input.pattern}\``);
+    if (input.query) facts.push(`- Query: ${input.query.slice(0, 100)}`);
+    if (input.url) facts.push(`- URL: ${input.url}`);
+
+    if (facts.length > 0) {
+      lines.push('**Details**:');
+      lines.push(...facts);
+      lines.push('');
+    }
+
+    lines.push('---');
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// Build token economics summary
+function buildTokenEconomics(observations) {
+  let totalDiscovery = 0;
+  let totalRead = 0;
+
+  for (const o of observations) {
+    totalDiscovery += o.tokens_discovery || 0;
+    totalRead += o.tokens_read || estimateTokens(o.summary || '');
+  }
+
+  const savings = totalDiscovery - totalRead;
+  const savingsPercent = totalDiscovery > 0 ? Math.round((savings / totalDiscovery) * 100) : 0;
+
+  if (totalDiscovery === 0) {
+    return `**Observations**: ${observations.length} | **Read cost**: ~${totalRead} tokens`;
+  }
+
+  return `**Discovery**: ${totalDiscovery} tokens | **Read**: ${totalRead} tokens | **Saved**: ${savings} (${savingsPercent}%)`;
+}
+
+// Build retrieval instructions
+function buildRetrievalInstructions() {
+  return `
+---
+
+**Need more context?** Use these commands:
+- Search: \`/memory search <query>\`
+- Get details: \`/memory get <id>\`
+- Timeline: \`/memory timeline <id>\`
+`;
+}
+
+/**
+ * Build topic summaries from database search
+ */
+function buildTopicSummaries() {
+  // Key topics to search for (based on common interests)
+  const topics = [
+    { name: 'AI记忆系统', keywords: ['短期记忆', '长期记忆', 'AI记忆', '向量数据库', 'RAG'] },
+    { name: '长寿与健身', keywords: ['长寿', '禁食', '热量限制', '力量训练', '有氧运动'] },
+    { name: '编程技术', keywords: ['Node.js', 'TypeScript', '微服务', '架构'] }
+  ];
+
+  const sections = [];
+
+  for (const topic of topics) {
+    const topicResults = [];
+    for (const kw of topic.keywords) {
+      try {
+        const results = database.searchObservations(kw, 2);
+        for (const r of results) {
+          // Avoid duplicates
+          if (!topicResults.find(t => t.id === r.id)) {
+            topicResults.push(r);
+          }
+        }
+      } catch (e) {
+        // Search might fail, continue
+      }
+    }
+
+    if (topicResults.length > 0) {
+      sections.push(`### ${topic.name}`);
+      sections.push('');
+      for (const r of topicResults.slice(0, 3)) {
+        const summary = r.summary || '';
+        if (summary.length > 30) {
+          sections.push(`- **#${r.id}**: ${summary.slice(0, 150)}${summary.length > 150 ? '...' : ''}`);
+        }
+      }
+      sections.push('');
+    }
+  }
+
+  if (sections.length > 0) {
+    return '## 历史话题讨论\n\n' + sections.join('\n');
+  }
+  return '';
+}
+
+/**
+ * Build complete context for session injection
+ */
+export function buildContext(projectPath, options = {}) {
+  const {
+    observationLimit = 50,
+    fullDetailCount = 5,
+    showTokenEconomics = true,
+    showRetrievalInstructions = true
+  } = options;
+
+  // Fetch recent observations and filter out low-value ones
+  const rawObservations = database.getRecentObservations(projectPath, observationLimit * 3); // Fetch more to compensate for filtering
+  const observations = filterHighValueObservations(rawObservations).slice(0, observationLimit);
+
+  if (observations.length === 0) {
+    return null; // No context to inject
+  }
+
+  // Fetch recent summaries
+  const summaries = database.getRecentSummaries(projectPath, 3);
+
+  // Build context parts
+  const parts = [];
+
+  // Header
+  parts.push('<openclaw-mem-context>');
+  parts.push('# Recent Activity');
+  parts.push('');
+
+  // Token economics
+  if (showTokenEconomics) {
+    parts.push(buildTokenEconomics(observations));
+    parts.push('');
+  }
+
+  // Topic summaries (from historical search)
+  const topicSummaries = buildTopicSummaries();
+  if (topicSummaries) {
+    parts.push(topicSummaries);
+    parts.push('');
+  }
+
+  // Index table (all observations, compact)
+  parts.push('## Index');
+  parts.push('');
+  parts.push(buildIndexTable(observations));
+
+  // Full details (top N)
+  if (fullDetailCount > 0) {
+    const details = buildFullDetails(observations, fullDetailCount);
+    if (details) {
+      parts.push('## Recent Details');
+      parts.push('');
+      parts.push(details);
+    }
+  }
+
+  // Session summaries
+  if (summaries.length > 0) {
+    parts.push('## Previous Sessions');
+    parts.push('');
+    for (const s of summaries) {
+      if (s.request) parts.push(`- **Goal**: ${s.request}`);
+      if (s.completed) parts.push(`- **Completed**: ${s.completed}`);
+      if (s.next_steps) parts.push(`- **Next**: ${s.next_steps}`);
+      parts.push('');
+    }
+  }
+
+  // Retrieval instructions
+  if (showRetrievalInstructions) {
+    parts.push(buildRetrievalInstructions());
+  }
+
+  parts.push('</openclaw-mem-context>');
+
+  return parts.join('\n');
+}
+
+/**
+ * Search observations and return formatted results
+ */
+export function searchContext(query, limit = 20) {
+  const results = database.searchObservations(query, limit);
+
+  if (results.length === 0) {
+    return `No observations found for query: "${query}"`;
+  }
+
+  const lines = [
+    `## Search Results for "${query}"`,
+    '',
+    '| ID | Tool | Summary | Date |',
+    '|----|------|---------|------|'
+  ];
+
+  for (const r of results) {
+    const summary = r.summary_highlight || r.summary || `${r.tool_name} operation`;
+    const truncSummary = summary.length > 60 ? summary.slice(0, 57) + '...' : summary;
+    const date = formatDate(r.timestamp);
+    lines.push(`| #${r.id} | ${r.tool_name} | ${truncSummary} | ${date} |`);
+  }
+
+  lines.push('');
+  lines.push(`*${results.length} results. Use \`/memory get <id>\` for full details.*`);
+
+  return lines.join('\n');
+}
+
+/**
+ * Get full observation details by IDs
+ */
+export function getObservationDetails(ids) {
+  const observations = database.getObservations(ids);
+
+  if (observations.length === 0) {
+    return `No observations found for IDs: ${ids.join(', ')}`;
+  }
+
+  return buildFullDetails(observations, observations.length);
+}
+
+/**
+ * Get timeline around an observation
+ */
+export function getTimeline(anchorId, depthBefore = 3, depthAfter = 2) {
+  const anchor = database.getObservation(anchorId);
+  if (!anchor) {
+    return `Observation #${anchorId} not found`;
+  }
+
+  // Get surrounding observations from same session
+  const allObs = database.getRecentObservations(null, 100);
+  const anchorIdx = allObs.findIndex(o => o.id === anchorId);
+
+  if (anchorIdx === -1) {
+    return buildFullDetails([anchor], 1);
+  }
+
+  const startIdx = Math.max(0, anchorIdx - depthAfter); // Note: list is DESC, so after = before in time
+  const endIdx = Math.min(allObs.length, anchorIdx + depthBefore + 1);
+  const timeline = allObs.slice(startIdx, endIdx).reverse();
+
+  const lines = [
+    `## Timeline around #${anchorId}`,
+    ''
+  ];
+
+  for (const o of timeline) {
+    const marker = o.id === anchorId ? '**→**' : '  ';
+    const time = formatTime(o.timestamp);
+    const emoji = getTypeEmoji(o.tool_name);
+    const summary = o.summary || `${o.tool_name} operation`;
+    lines.push(`${marker} ${time} ${emoji} #${o.id}: ${summary}`);
+  }
+
+  return lines.join('\n');
+}
+
+export default {
+  buildContext,
+  searchContext,
+  getObservationDetails,
+  getTimeline
+};

package/lib/database.js

@@ -0,0 +1,309 @@
+/**
+ * OpenClaw-Mem Database Module
+ * SQLite-based storage for observations, sessions, and summaries
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import Database from 'better-sqlite3';
+
+const DATA_DIR = path.join(os.homedir(), '.openclaw-mem');
+const DB_PATH = path.join(DATA_DIR, 'memory.db');
+
+// Ensure data directory exists
+if (!fs.existsSync(DATA_DIR)) {
+  fs.mkdirSync(DATA_DIR, { recursive: true });
+}
+
+// Initialize database
+const db = new Database(DB_PATH);
+db.pragma('journal_mode = WAL');
+
+// Create tables
+db.exec(`
+  -- Sessions table
+  CREATE TABLE IF NOT EXISTS sessions (
+    id TEXT PRIMARY KEY,
+    project_path TEXT,
+    session_key TEXT,
+    started_at TEXT DEFAULT (datetime('now')),
+    ended_at TEXT,
+    status TEXT DEFAULT 'active',
+    source TEXT
+  );
+
+  -- Observations table (tool calls)
+  CREATE TABLE IF NOT EXISTS observations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id TEXT,
+    timestamp TEXT DEFAULT (datetime('now')),
+    tool_name TEXT NOT NULL,
+    tool_input TEXT,
+    tool_response TEXT,
+    summary TEXT,
+    concepts TEXT,
+    tokens_discovery INTEGER DEFAULT 0,
+    tokens_read INTEGER DEFAULT 0,
+    FOREIGN KEY (session_id) REFERENCES sessions(id)
+  );
+
+  -- Summaries table
+  CREATE TABLE IF NOT EXISTS summaries (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id TEXT,
+    content TEXT,
+    request TEXT,
+    completed TEXT,
+    next_steps TEXT,
+    created_at TEXT DEFAULT (datetime('now')),
+    FOREIGN KEY (session_id) REFERENCES sessions(id)
+  );
+
+  -- Full-text search index
+  CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
+    tool_name,
+    summary,
+    concepts,
+    content='observations',
+    content_rowid='id'
+  );
+
+  -- Triggers for FTS sync
+  CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
+    INSERT INTO observations_fts(rowid, tool_name, summary, concepts)
+    VALUES (new.id, new.tool_name, new.summary, new.concepts);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
+    INSERT INTO observations_fts(observations_fts, rowid, tool_name, summary, concepts)
+    VALUES ('delete', old.id, old.tool_name, old.summary, old.concepts);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
+    INSERT INTO observations_fts(observations_fts, rowid, tool_name, summary, concepts)
+    VALUES ('delete', old.id, old.tool_name, old.summary, old.concepts);
+    INSERT INTO observations_fts(rowid, tool_name, summary, concepts)
+    VALUES (new.id, new.tool_name, new.summary, new.concepts);
+  END;
+
+  -- Indexes
+  CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
+  CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp DESC);
+  CREATE INDEX IF NOT EXISTS idx_observations_tool ON observations(tool_name);
+  CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_path);
+`);
+
+// Prepared statements
+const stmts = {
+  // Sessions
+  createSession: db.prepare(`
+    INSERT INTO sessions (id, project_path, session_key, source)
+    VALUES (?, ?, ?, ?)
+  `),
+
+  getSession: db.prepare(`
+    SELECT * FROM sessions WHERE id = ?
+  `),
+
+  endSession: db.prepare(`
+    UPDATE sessions SET ended_at = datetime('now'), status = 'completed'
+    WHERE id = ?
+  `),
+
+  getActiveSession: db.prepare(`
+    SELECT * FROM sessions WHERE session_key = ? AND status = 'active'
+    ORDER BY started_at DESC LIMIT 1
+  `),
+
+  // Observations
+  saveObservation: db.prepare(`
+    INSERT INTO observations (session_id, tool_name, tool_input, tool_response, summary, concepts, tokens_discovery, tokens_read)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `),
+
+  getObservation: db.prepare(`
+    SELECT * FROM observations WHERE id = ?
+  `),
+
+  getObservations: db.prepare(`
+    SELECT * FROM observations WHERE id IN (SELECT value FROM json_each(?))
+  `),
+
+  updateObservationSummary: db.prepare(`
+    UPDATE observations SET summary = ?, concepts = ?, tokens_read = ?
+    WHERE id = ?
+  `),
+
+  getRecentObservations: db.prepare(`
+    SELECT o.*, s.project_path
+    FROM observations o
+    JOIN sessions s ON o.session_id = s.id
+    WHERE s.project_path = ?
+    ORDER BY o.timestamp DESC
+    LIMIT ?
+  `),
+
+  getRecentObservationsAll: db.prepare(`
+    SELECT o.*, s.project_path
+    FROM observations o
+    JOIN sessions s ON o.session_id = s.id
+    ORDER BY o.timestamp DESC
+    LIMIT ?
+  `),
+
+  searchObservations: db.prepare(`
+    SELECT o.*, s.project_path,
+           highlight(observations_fts, 1, '<mark>', '</mark>') as summary_highlight
+    FROM observations_fts fts
+    JOIN observations o ON fts.rowid = o.id
+    JOIN sessions s ON o.session_id = s.id
+    WHERE observations_fts MATCH ?
+    ORDER BY rank
+    LIMIT ?
+  `),
+
+  // Summaries
+  saveSummary: db.prepare(`
+    INSERT INTO summaries (session_id, content, request, completed, next_steps)
+    VALUES (?, ?, ?, ?, ?)
+  `),
+
+  getRecentSummaries: db.prepare(`
+    SELECT su.*, s.project_path
+    FROM summaries su
+    JOIN sessions s ON su.session_id = s.id
+    WHERE s.project_path = ?
+    ORDER BY su.created_at DESC
+    LIMIT ?
+  `),
+
+  // Stats
+  getStats: db.prepare(`
+    SELECT
+      (SELECT COUNT(*) FROM sessions) as total_sessions,
+      (SELECT COUNT(*) FROM observations) as total_observations,
+      (SELECT COUNT(*) FROM summaries) as total_summaries,
+      (SELECT SUM(tokens_discovery) FROM observations) as total_discovery_tokens,
+      (SELECT SUM(tokens_read) FROM observations) as total_read_tokens
+  `)
+};
+
+// Database API
+export const database = {
+  // Session operations
+  createSession(id, projectPath, sessionKey, source = 'unknown') {
+    try {
+      stmts.createSession.run(id, projectPath, sessionKey, source);
+      return { success: true, id };
+    } catch (err) {
+      // Session might already exist
+      return { success: false, error: err.message };
+    }
+  },
+
+  getSession(id) {
+    return stmts.getSession.get(id);
+  },
+
+  getActiveSession(sessionKey) {
+    return stmts.getActiveSession.get(sessionKey);
+  },
+
+  endSession(id) {
+    stmts.endSession.run(id);
+  },
+
+  // Observation operations
+  saveObservation(sessionId, toolName, toolInput, toolResponse, options = {}) {
+    const {
+      summary = null,
+      concepts = null,
+      tokensDiscovery = 0,
+      tokensRead = 0
+    } = options;
+
+    const result = stmts.saveObservation.run(
+      sessionId,
+      toolName,
+      JSON.stringify(toolInput),
+      JSON.stringify(toolResponse),
+      summary,
+      concepts,
+      tokensDiscovery,
+      tokensRead
+    );
+
+    return { success: true, id: result.lastInsertRowid };
+  },
+
+  getObservation(id) {
+    const row = stmts.getObservation.get(id);
+    if (row) {
+      row.tool_input = JSON.parse(row.tool_input || '{}');
+      row.tool_response = JSON.parse(row.tool_response || '{}');
+    }
+    return row;
+  },
+
+  getObservations(ids) {
+    const rows = stmts.getObservations.all(JSON.stringify(ids));
+    return rows.map(row => ({
+      ...row,
+      tool_input: JSON.parse(row.tool_input || '{}'),
+      tool_response: JSON.parse(row.tool_response || '{}')
+    }));
+  },
+
+  updateObservationSummary(id, summary, concepts, tokensRead) {
+    stmts.updateObservationSummary.run(summary, concepts, tokensRead, id);
+  },
+
+  getRecentObservations(projectPath, limit = 50) {
+    const rows = projectPath
+      ? stmts.getRecentObservations.all(projectPath, limit)
+      : stmts.getRecentObservationsAll.all(limit);
+
+    return rows.map(row => ({
+      ...row,
+      tool_input: JSON.parse(row.tool_input || '{}'),
+      tool_response: JSON.parse(row.tool_response || '{}')
+    }));
+  },
+
+  searchObservations(query, limit = 20) {
+    try {
+      const rows = stmts.searchObservations.all(query, limit);
+      return rows.map(row => ({
+        ...row,
+        tool_input: JSON.parse(row.tool_input || '{}'),
+        tool_response: JSON.parse(row.tool_response || '{}')
+      }));
+    } catch (err) {
+      console.error('[openclaw-mem] Search error:', err.message);
+      return [];
+    }
+  },
+
+  // Summary operations
+  saveSummary(sessionId, content, request = null, completed = null, nextSteps = null) {
+    const result = stmts.saveSummary.run(sessionId, content, request, completed, nextSteps);
+    return { success: true, id: result.lastInsertRowid };
+  },
+
+  getRecentSummaries(projectPath, limit = 5) {
+    return stmts.getRecentSummaries.all(projectPath, limit);
+  },
+
+  // Stats
+  getStats() {
+    return stmts.getStats.get();
+  },
+
+  // Close database
+  close() {
+    db.close();
+  }
+};
+
+export default database;