openclaw-mem 1.0.4 → 1.2.1

package/lib/context-builder.js

@@ -1,415 +0,0 @@
-/**
- * OpenClaw-Mem Context Builder
- * Generates context to inject into new sessions using progressive disclosure
- */
-
-import database from './database.js';
-
-// Token estimation (4 chars ≈ 1 token)
-const CHARS_PER_TOKEN = 4;
-
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(String(text).length / CHARS_PER_TOKEN);
-}
-
-// Type emoji mapping
-const TYPE_EMOJI = {
-  'Edit': '📝',
-  'Write': '✏️',
-  'Read': '📖',
-  'Bash': '💻',
-  'Grep': '🔍',
-  'Glob': '📁',
-  'WebFetch': '🌐',
-  'WebSearch': '🔎',
-  'Task': '🤖',
-  'default': '🔵'
-};
-
-function getTypeEmoji(toolName) {
-  return TYPE_EMOJI[toolName] || TYPE_EMOJI.default;
-}
-
-// Format timestamp
-function formatTime(timestamp) {
-  if (!timestamp) return '';
-  const date = new Date(timestamp);
-  return date.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit' });
-}
-
-function formatDate(timestamp) {
-  if (!timestamp) return '';
-  const date = new Date(timestamp);
-  return date.toISOString().split('T')[0];
-}
-
-// Filter out low-value observations (like recall test queries)
-function filterHighValueObservations(observations) {
-  const lowValuePatterns = [
-    '请查看 SESSION-MEMORY',
-    'SESSION-MEMORY.md 里没有',
-    '请查看 SESSION-MEMORY.md，告诉我',
-    '记忆检索当前不可用',
-    '/memory search',
-    '/memory get',
-    '之前没有记录到',
-    '没有任何关于'
-  ];
-
-  return observations.filter(obs => {
-    const summary = obs.summary || '';
-    // Always filter out observations matching low-value patterns
-    const isLowValue = lowValuePatterns.some(pattern => summary.includes(pattern));
-    if (isLowValue) return false;
-
-    // Keep observations that have actual content (not just metadata)
-    return true;
-  });
-}
-
-// Group observations by date
-function groupByDate(observations) {
-  const groups = {};
-  for (const obs of observations) {
-    const date = formatDate(obs.timestamp);
-    if (!groups[date]) {
-      groups[date] = [];
-    }
-    groups[date].push(obs);
-  }
-  return groups;
-}
-
-// Build index table (Layer 1 - compact)
-function buildIndexTable(observations) {
-  if (!observations || observations.length === 0) {
-    return '*(No recent observations)*';
-  }
-
-  const grouped = groupByDate(observations);
-  const lines = [];
-
-  for (const [date, obs] of Object.entries(grouped)) {
-    lines.push(`### ${date}`);
-    lines.push('');
-    lines.push('| ID | Time | T | Summary | Tokens |');
-    lines.push('|----|------|---|---------|--------|');
-
-    for (const o of obs) {
-      const id = `#${o.id}`;
-      const time = formatTime(o.timestamp);
-      const emoji = getTypeEmoji(o.tool_name);
-      const summary = o.summary || `${o.tool_name} operation`;
-      const truncSummary = summary.length > 50 ? summary.slice(0, 47) + '...' : summary;
-      const tokens = `~${o.tokens_read || estimateTokens(summary)}`;
-
-      lines.push(`| ${id} | ${time} | ${emoji} | ${truncSummary} | ${tokens} |`);
-    }
-    lines.push('');
-  }
-
-  return lines.join('\n');
-}
-
-// Build full details (Layer 3 - expensive)
-function buildFullDetails(observations, limit = 5) {
-  if (!observations || observations.length === 0) {
-    return '';
-  }
-
-  const toShow = observations.slice(0, limit);
-  const lines = [];
-
-  for (const o of toShow) {
-    lines.push(`#### #${o.id} - ${o.tool_name}`);
-    lines.push('');
-
-    if (o.summary) {
-      lines.push(`**Summary**: ${o.summary}`);
-      lines.push('');
-    }
-
-    // Show key facts from tool input
-    const input = o.tool_input || {};
-    const facts = [];
-
-    if (input.file_path) facts.push(`- File: \`${input.file_path}\``);
-    if (input.command) facts.push(`- Command: \`${input.command.slice(0, 100)}\``);
-    if (input.pattern) facts.push(`- Pattern: \`${input.pattern}\``);
-    if (input.query) facts.push(`- Query: ${input.query.slice(0, 100)}`);
-    if (input.url) facts.push(`- URL: ${input.url}`);
-
-    if (facts.length > 0) {
-      lines.push('**Details**:');
-      lines.push(...facts);
-      lines.push('');
-    }
-
-    lines.push('---');
-    lines.push('');
-  }
-
-  return lines.join('\n');
-}
-
-// Build token economics summary
-function buildTokenEconomics(observations) {
-  let totalDiscovery = 0;
-  let totalRead = 0;
-
-  for (const o of observations) {
-    totalDiscovery += o.tokens_discovery || 0;
-    totalRead += o.tokens_read || estimateTokens(o.summary || '');
-  }
-
-  const savings = totalDiscovery - totalRead;
-  const savingsPercent = totalDiscovery > 0 ? Math.round((savings / totalDiscovery) * 100) : 0;
-
-  if (totalDiscovery === 0) {
-    return `**Observations**: ${observations.length} | **Read cost**: ~${totalRead} tokens`;
-  }
-
-  return `**Discovery**: ${totalDiscovery} tokens | **Read**: ${totalRead} tokens | **Saved**: ${savings} (${savingsPercent}%)`;
-}
-
-// Build retrieval instructions
-function buildRetrievalInstructions() {
-  return `
----
-
-**Need more context?** Use these commands:
-- Search: \`/memory search <query>\`
-- Get details: \`/memory get <id>\`
-- Timeline: \`/memory timeline <id>\`
-`;
-}
-
-/**
- * Build topic summaries from user's actual recorded concepts
- * Dynamically extracts topics from what the user has discussed
- */
-function buildTopicSummaries() {
-  // Get all recent observations to extract actual concepts
-  const recentObs = database.getRecentObservations(null, 100);
-
-  // Extract and count concepts from user's actual data
-  const conceptCounts = {};
-  for (const obs of recentObs) {
-    if (obs.concepts) {
-      const concepts = obs.concepts.split(',').map(c => c.trim()).filter(c => c.length > 1);
-      for (const concept of concepts) {
-        // Skip generic tool names
-        if (['edit', 'bash', 'read', 'grep', 'write', 'glob'].includes(concept.toLowerCase())) continue;
-        conceptCounts[concept] = (conceptCounts[concept] || 0) + 1;
-      }
-    }
-  }
-
-  // Get top concepts (mentioned at least twice)
-  const topConcepts = Object.entries(conceptCounts)
-    .filter(([_, count]) => count >= 2)
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, 10)
-    .map(([concept, _]) => concept);
-
-  if (topConcepts.length === 0) return '';
-
-  const sections = [];
-  const seenIds = new Set();
-
-  // Search for each top concept
-  for (const concept of topConcepts.slice(0, 5)) {
-    try {
-      const results = database.searchObservations(concept, 3);
-      const newResults = results.filter(r => !seenIds.has(r.id));
-
-      if (newResults.length > 0) {
-        sections.push(`### ${concept}`);
-        sections.push('');
-        for (const r of newResults.slice(0, 2)) {
-          seenIds.add(r.id);
-          const summary = r.summary || '';
-          if (summary.length > 20) {
-            sections.push(`- **#${r.id}**: ${summary.slice(0, 150)}${summary.length > 150 ? '...' : ''}`);
-          }
-        }
-        sections.push('');
-      }
-    } catch (e) {
-      // Search might fail, continue
-    }
-  }
-
-  if (sections.length > 0) {
-    return '## 历史话题讨论\n\n基于您的实际对话自动提取的主题：\n\n' + sections.join('\n');
-  }
-  return '';
-}
-
-/**
- * Build complete context for session injection
- */
-export function buildContext(projectPath, options = {}) {
-  const {
-    observationLimit = 50,
-    fullDetailCount = 5,
-    showTokenEconomics = true,
-    showRetrievalInstructions = true
-  } = options;
-
-  // Fetch recent observations and filter out low-value ones
-  const rawObservations = database.getRecentObservations(projectPath, observationLimit * 3); // Fetch more to compensate for filtering
-  const observations = filterHighValueObservations(rawObservations).slice(0, observationLimit);
-
-  if (observations.length === 0) {
-    return null; // No context to inject
-  }
-
-  // Fetch recent summaries
-  const summaries = database.getRecentSummaries(projectPath, 3);
-
-  // Build context parts
-  const parts = [];
-
-  // Header
-  parts.push('<openclaw-mem-context>');
-  parts.push('# Recent Activity');
-  parts.push('');
-
-  // Token economics
-  if (showTokenEconomics) {
-    parts.push(buildTokenEconomics(observations));
-    parts.push('');
-  }
-
-  // Topic summaries (from historical search)
-  const topicSummaries = buildTopicSummaries();
-  if (topicSummaries) {
-    parts.push(topicSummaries);
-    parts.push('');
-  }
-
-  // Index table (all observations, compact)
-  parts.push('## Index');
-  parts.push('');
-  parts.push(buildIndexTable(observations));
-
-  // Full details (top N)
-  if (fullDetailCount > 0) {
-    const details = buildFullDetails(observations, fullDetailCount);
-    if (details) {
-      parts.push('## Recent Details');
-      parts.push('');
-      parts.push(details);
-    }
-  }
-
-  // Session summaries
-  if (summaries.length > 0) {
-    parts.push('## Previous Sessions');
-    parts.push('');
-    for (const s of summaries) {
-      if (s.request) parts.push(`- **Goal**: ${s.request}`);
-      if (s.completed) parts.push(`- **Completed**: ${s.completed}`);
-      if (s.next_steps) parts.push(`- **Next**: ${s.next_steps}`);
-      parts.push('');
-    }
-  }
-
-  // Retrieval instructions
-  if (showRetrievalInstructions) {
-    parts.push(buildRetrievalInstructions());
-  }
-
-  parts.push('</openclaw-mem-context>');
-
-  return parts.join('\n');
-}
-
-/**
- * Search observations and return formatted results
- */
-export function searchContext(query, limit = 20) {
-  const results = database.searchObservations(query, limit);
-
-  if (results.length === 0) {
-    return `No observations found for query: "${query}"`;
-  }
-
-  const lines = [
-    `## Search Results for "${query}"`,
-    '',
-    '| ID | Tool | Summary | Date |',
-    '|----|------|---------|------|'
-  ];
-
-  for (const r of results) {
-    const summary = r.summary_highlight || r.summary || `${r.tool_name} operation`;
-    const truncSummary = summary.length > 60 ? summary.slice(0, 57) + '...' : summary;
-    const date = formatDate(r.timestamp);
-    lines.push(`| #${r.id} | ${r.tool_name} | ${truncSummary} | ${date} |`);
-  }
-
-  lines.push('');
-  lines.push(`*${results.length} results. Use \`/memory get <id>\` for full details.*`);
-
-  return lines.join('\n');
-}
-
-/**
- * Get full observation details by IDs
- */
-export function getObservationDetails(ids) {
-  const observations = database.getObservations(ids);
-
-  if (observations.length === 0) {
-    return `No observations found for IDs: ${ids.join(', ')}`;
-  }
-
-  return buildFullDetails(observations, observations.length);
-}
-
-/**
- * Get timeline around an observation
- */
-export function getTimeline(anchorId, depthBefore = 3, depthAfter = 2) {
-  const anchor = database.getObservation(anchorId);
-  if (!anchor) {
-    return `Observation #${anchorId} not found`;
-  }
-
-  // Get surrounding observations from same session
-  const allObs = database.getRecentObservations(null, 100);
-  const anchorIdx = allObs.findIndex(o => o.id === anchorId);
-
-  if (anchorIdx === -1) {
-    return buildFullDetails([anchor], 1);
-  }
-
-  const startIdx = Math.max(0, anchorIdx - depthAfter); // Note: list is DESC, so after = before in time
-  const endIdx = Math.min(allObs.length, anchorIdx + depthBefore + 1);
-  const timeline = allObs.slice(startIdx, endIdx).reverse();
-
-  const lines = [
-    `## Timeline around #${anchorId}`,
-    ''
-  ];
-
-  for (const o of timeline) {
-    const marker = o.id === anchorId ? '**→**' : '  ';
-    const time = formatTime(o.timestamp);
-    const emoji = getTypeEmoji(o.tool_name);
-    const summary = o.summary || `${o.tool_name} operation`;
-    lines.push(`${marker} ${time} ${emoji} #${o.id}: ${summary}`);
-  }
-
-  return lines.join('\n');
-}
-
-export default {
-  buildContext,
-  searchContext,
-  getObservationDetails,
-  getTimeline
-};

package/lib/database.js

@@ -1,309 +0,0 @@
-/**
- * OpenClaw-Mem Database Module
- * SQLite-based storage for observations, sessions, and summaries
- */
-
-import fs from 'node:fs';
-import path from 'node:path';
-import os from 'node:os';
-import Database from 'better-sqlite3';
-
-const DATA_DIR = path.join(os.homedir(), '.openclaw-mem');
-const DB_PATH = path.join(DATA_DIR, 'memory.db');
-
-// Ensure data directory exists
-if (!fs.existsSync(DATA_DIR)) {
-  fs.mkdirSync(DATA_DIR, { recursive: true });
-}
-
-// Initialize database
-const db = new Database(DB_PATH);
-db.pragma('journal_mode = WAL');
-
-// Create tables
-db.exec(`
-  -- Sessions table
-  CREATE TABLE IF NOT EXISTS sessions (
-    id TEXT PRIMARY KEY,
-    project_path TEXT,
-    session_key TEXT,
-    started_at TEXT DEFAULT (datetime('now')),
-    ended_at TEXT,
-    status TEXT DEFAULT 'active',
-    source TEXT
-  );
-
-  -- Observations table (tool calls)
-  CREATE TABLE IF NOT EXISTS observations (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    session_id TEXT,
-    timestamp TEXT DEFAULT (datetime('now')),
-    tool_name TEXT NOT NULL,
-    tool_input TEXT,
-    tool_response TEXT,
-    summary TEXT,
-    concepts TEXT,
-    tokens_discovery INTEGER DEFAULT 0,
-    tokens_read INTEGER DEFAULT 0,
-    FOREIGN KEY (session_id) REFERENCES sessions(id)
-  );
-
-  -- Summaries table
-  CREATE TABLE IF NOT EXISTS summaries (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    session_id TEXT,
-    content TEXT,
-    request TEXT,
-    completed TEXT,
-    next_steps TEXT,
-    created_at TEXT DEFAULT (datetime('now')),
-    FOREIGN KEY (session_id) REFERENCES sessions(id)
-  );
-
-  -- Full-text search index
-  CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
-    tool_name,
-    summary,
-    concepts,
-    content='observations',
-    content_rowid='id'
-  );
-
-  -- Triggers for FTS sync
-  CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
-    INSERT INTO observations_fts(rowid, tool_name, summary, concepts)
-    VALUES (new.id, new.tool_name, new.summary, new.concepts);
-  END;
-
-  CREATE TRIGGER IF NOT EXISTS observations_ad AFTER DELETE ON observations BEGIN
-    INSERT INTO observations_fts(observations_fts, rowid, tool_name, summary, concepts)
-    VALUES ('delete', old.id, old.tool_name, old.summary, old.concepts);
-  END;
-
-  CREATE TRIGGER IF NOT EXISTS observations_au AFTER UPDATE ON observations BEGIN
-    INSERT INTO observations_fts(observations_fts, rowid, tool_name, summary, concepts)
-    VALUES ('delete', old.id, old.tool_name, old.summary, old.concepts);
-    INSERT INTO observations_fts(rowid, tool_name, summary, concepts)
-    VALUES (new.id, new.tool_name, new.summary, new.concepts);
-  END;
-
-  -- Indexes
-  CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
-  CREATE INDEX IF NOT EXISTS idx_observations_timestamp ON observations(timestamp DESC);
-  CREATE INDEX IF NOT EXISTS idx_observations_tool ON observations(tool_name);
-  CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_path);
-`);
-
-// Prepared statements
-const stmts = {
-  // Sessions
-  createSession: db.prepare(`
-    INSERT INTO sessions (id, project_path, session_key, source)
-    VALUES (?, ?, ?, ?)
-  `),
-
-  getSession: db.prepare(`
-    SELECT * FROM sessions WHERE id = ?
-  `),
-
-  endSession: db.prepare(`
-    UPDATE sessions SET ended_at = datetime('now'), status = 'completed'
-    WHERE id = ?
-  `),
-
-  getActiveSession: db.prepare(`
-    SELECT * FROM sessions WHERE session_key = ? AND status = 'active'
-    ORDER BY started_at DESC LIMIT 1
-  `),
-
-  // Observations
-  saveObservation: db.prepare(`
-    INSERT INTO observations (session_id, tool_name, tool_input, tool_response, summary, concepts, tokens_discovery, tokens_read)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-  `),
-
-  getObservation: db.prepare(`
-    SELECT * FROM observations WHERE id = ?
-  `),
-
-  getObservations: db.prepare(`
-    SELECT * FROM observations WHERE id IN (SELECT value FROM json_each(?))
-  `),
-
-  updateObservationSummary: db.prepare(`
-    UPDATE observations SET summary = ?, concepts = ?, tokens_read = ?
-    WHERE id = ?
-  `),
-
-  getRecentObservations: db.prepare(`
-    SELECT o.*, s.project_path
-    FROM observations o
-    JOIN sessions s ON o.session_id = s.id
-    WHERE s.project_path = ?
-    ORDER BY o.timestamp DESC
-    LIMIT ?
-  `),
-
-  getRecentObservationsAll: db.prepare(`
-    SELECT o.*, s.project_path
-    FROM observations o
-    JOIN sessions s ON o.session_id = s.id
-    ORDER BY o.timestamp DESC
-    LIMIT ?
-  `),
-
-  searchObservations: db.prepare(`
-    SELECT o.*, s.project_path,
-           highlight(observations_fts, 1, '<mark>', '</mark>') as summary_highlight
-    FROM observations_fts fts
-    JOIN observations o ON fts.rowid = o.id
-    JOIN sessions s ON o.session_id = s.id
-    WHERE observations_fts MATCH ?
-    ORDER BY rank
-    LIMIT ?
-  `),
-
-  // Summaries
-  saveSummary: db.prepare(`
-    INSERT INTO summaries (session_id, content, request, completed, next_steps)
-    VALUES (?, ?, ?, ?, ?)
-  `),
-
-  getRecentSummaries: db.prepare(`
-    SELECT su.*, s.project_path
-    FROM summaries su
-    JOIN sessions s ON su.session_id = s.id
-    WHERE s.project_path = ?
-    ORDER BY su.created_at DESC
-    LIMIT ?
-  `),
-
-  // Stats
-  getStats: db.prepare(`
-    SELECT
-      (SELECT COUNT(*) FROM sessions) as total_sessions,
-      (SELECT COUNT(*) FROM observations) as total_observations,
-      (SELECT COUNT(*) FROM summaries) as total_summaries,
-      (SELECT SUM(tokens_discovery) FROM observations) as total_discovery_tokens,
-      (SELECT SUM(tokens_read) FROM observations) as total_read_tokens
-  `)
-};
-
-// Database API
-export const database = {
-  // Session operations
-  createSession(id, projectPath, sessionKey, source = 'unknown') {
-    try {
-      stmts.createSession.run(id, projectPath, sessionKey, source);
-      return { success: true, id };
-    } catch (err) {
-      // Session might already exist
-      return { success: false, error: err.message };
-    }
-  },
-
-  getSession(id) {
-    return stmts.getSession.get(id);
-  },
-
-  getActiveSession(sessionKey) {
-    return stmts.getActiveSession.get(sessionKey);
-  },
-
-  endSession(id) {
-    stmts.endSession.run(id);
-  },
-
-  // Observation operations
-  saveObservation(sessionId, toolName, toolInput, toolResponse, options = {}) {
-    const {
-      summary = null,
-      concepts = null,
-      tokensDiscovery = 0,
-      tokensRead = 0
-    } = options;
-
-    const result = stmts.saveObservation.run(
-      sessionId,
-      toolName,
-      JSON.stringify(toolInput),
-      JSON.stringify(toolResponse),
-      summary,
-      concepts,
-      tokensDiscovery,
-      tokensRead
-    );
-
-    return { success: true, id: result.lastInsertRowid };
-  },
-
-  getObservation(id) {
-    const row = stmts.getObservation.get(id);
-    if (row) {
-      row.tool_input = JSON.parse(row.tool_input || '{}');
-      row.tool_response = JSON.parse(row.tool_response || '{}');
-    }
-    return row;
-  },
-
-  getObservations(ids) {
-    const rows = stmts.getObservations.all(JSON.stringify(ids));
-    return rows.map(row => ({
-      ...row,
-      tool_input: JSON.parse(row.tool_input || '{}'),
-      tool_response: JSON.parse(row.tool_response || '{}')
-    }));
-  },
-
-  updateObservationSummary(id, summary, concepts, tokensRead) {
-    stmts.updateObservationSummary.run(summary, concepts, tokensRead, id);
-  },
-
-  getRecentObservations(projectPath, limit = 50) {
-    const rows = projectPath
-      ? stmts.getRecentObservations.all(projectPath, limit)
-      : stmts.getRecentObservationsAll.all(limit);
-
-    return rows.map(row => ({
-      ...row,
-      tool_input: JSON.parse(row.tool_input || '{}'),
-      tool_response: JSON.parse(row.tool_response || '{}')
-    }));
-  },
-
-  searchObservations(query, limit = 20) {
-    try {
-      const rows = stmts.searchObservations.all(query, limit);
-      return rows.map(row => ({
-        ...row,
-        tool_input: JSON.parse(row.tool_input || '{}'),
-        tool_response: JSON.parse(row.tool_response || '{}')
-      }));
-    } catch (err) {
-      console.error('[openclaw-mem] Search error:', err.message);
-      return [];
-    }
-  },
-
-  // Summary operations
-  saveSummary(sessionId, content, request = null, completed = null, nextSteps = null) {
-    const result = stmts.saveSummary.run(sessionId, content, request, completed, nextSteps);
-    return { success: true, id: result.lastInsertRowid };
-  },
-
-  getRecentSummaries(projectPath, limit = 5) {
-    return stmts.getRecentSummaries.all(projectPath, limit);
-  },
-
-  // Stats
-  getStats() {
-    return stmts.getStats.get();
-  },
-
-  // Close database
-  close() {
-    db.close();
-  }
-};
-
-export default database;