openclaw-memory-mapper 1.0.0

package/server/index.ts

@@ -0,0 +1,541 @@
+import express, { Request, Response } from 'express';
+import { readdir, readFile, stat } from 'fs/promises';
+import { existsSync } from 'fs';
+import path from 'path';
+import { homedir } from 'os';
+import { fileURLToPath } from 'url';
+import { parse as csvParse } from 'csv-parse/sync';
+
+const { join, relative, extname, basename, dirname } = path;
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const app = express();
+const PORT = process.env.PORT || 3001;
+
+// Parse JSON bodies
+app.use(express.json());
+
+// Serve static files from the built frontend in production
+// When compiled: server is at dist/server/index.js, frontend at dist/
+// When running from source: server is at server/index.ts, frontend at dist/
+const isCompiledServer = __dirname.includes('dist/server') || __dirname.includes('dist\\server');
+const distPath = isCompiledServer
+  ? join(__dirname, '..') // dist/server -> dist/
+  : join(__dirname, '..', 'dist'); // server/ -> dist/
+
+if (existsSync(join(distPath, 'index.html'))) {
+  app.use(express.static(distPath));
+}
+
+// Default memory paths (can be overridden via /api/config/sources)
+const OPENCLAW_PATH = join(homedir(), '.openclaw');
+
+// Configured source paths from client
+interface ConfiguredSource {
+  id: string;
+  path: string;
+}
+let configuredSources: ConfiguredSource[] = [];
+
+// Helper to expand ~ and validate path is within home directory
+function expandAndValidatePath(pathParam: string): { valid: boolean; expanded: string; error?: string } {
+  const expanded = pathParam.startsWith('~')
+    ? join(homedir(), pathParam.slice(1))
+    : pathParam;
+
+  const resolved = path.resolve(expanded);
+  const home = homedir();
+
+  if (!resolved.startsWith(home + path.sep) && resolved !== home) {
+    return { valid: false, expanded, error: 'Path must be within home directory' };
+  }
+
+  return { valid: true, expanded };
+}
+
+// Get base path for a source (uses configured sources if available, falls back to defaults)
+function getBasePath(sourceId: string): string | null {
+  const configured = configuredSources.find(s => s.id === sourceId);
+  if (configured) {
+    const { valid, expanded } = expandAndValidatePath(configured.path);
+    if (valid) return expanded;
+  }
+
+  // Fallback to default
+  if (sourceId === 'openclaw') return OPENCLAW_PATH;
+  return null;
+}
+
+// Security: sanitize error messages to avoid leaking sensitive info in production
+const IS_PRODUCTION = process.env.NODE_ENV === 'production';
+function sanitizeError(error: unknown): string | undefined {
+  if (!IS_PRODUCTION) {
+    return String(error);
+  }
+  // In production, only return generic message - log full error server-side
+  console.error('Server error:', error);
+  return undefined;
+}
+
+interface FileNode {
+  name: string;
+  path: string;
+  type: 'file' | 'directory';
+  size?: number;
+  modified?: string;
+  children?: FileNode[];
+}
+
+interface LogEntry {
+  timestamp: string;
+  session_id: string;
+  turn_number: string;
+  agent_name: string;
+  action_type: string;
+  target_file: string;
+  content_summary: string;
+  tokens_used: string;
+  latency_ms: string;
+  success: string;
+  error_message: string;
+}
+
+// Recursively read directory structure
+async function readDirRecursive(dirPath: string, basePath: string): Promise<FileNode[]> {
+  const entries = await readdir(dirPath, { withFileTypes: true });
+  const nodes: FileNode[] = [];
+
+  for (const entry of entries) {
+    const fullPath = join(dirPath, entry.name);
+    const relativePath = relative(basePath, fullPath);
+
+    if (entry.isDirectory()) {
+      // Skip hidden directories and node_modules
+      if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
+
+      const children = await readDirRecursive(fullPath, basePath);
+      nodes.push({
+        name: entry.name,
+        path: relativePath,
+        type: 'directory',
+        children,
+      });
+    } else {
+      const stats = await stat(fullPath);
+      nodes.push({
+        name: entry.name,
+        path: relativePath,
+        type: 'file',
+        size: stats.size,
+        modified: stats.mtime.toISOString(),
+      });
+    }
+  }
+
+  // Sort: directories first, then alphabetically
+  return nodes.sort((a, b) => {
+    if (a.type !== b.type) return a.type === 'directory' ? -1 : 1;
+    return a.name.localeCompare(b.name);
+  });
+}
+
+// Count files in directory tree
+function countFiles(nodes: FileNode[]): number {
+  let count = 0;
+  for (const node of nodes) {
+    if (node.type === 'file') count++;
+    if (node.children) count += countFiles(node.children);
+  }
+  return count;
+}
+
+// Get total size of directory tree
+function totalSize(nodes: FileNode[]): number {
+  let size = 0;
+  for (const node of nodes) {
+    if (node.type === 'file' && node.size) size += node.size;
+    if (node.children) size += totalSize(node.children);
+  }
+  return size;
+}
+
+// API: Configure source paths
+app.post('/api/config/sources', (req: Request, res: Response) => {
+  const { sources } = req.body as { sources: ConfiguredSource[] };
+
+  if (!Array.isArray(sources)) {
+    res.status(400).json({ error: 'sources must be an array' });
+    return;
+  }
+
+  // Validate all paths
+  const validated: ConfiguredSource[] = [];
+  for (const source of sources) {
+    if (!source.id || !source.path) continue;
+    const { valid, expanded, error } = expandAndValidatePath(source.path);
+    if (!valid) {
+      res.status(400).json({ error: `Invalid path for ${source.id}: ${error}` });
+      return;
+    }
+    validated.push({ id: source.id, path: expanded });
+  }
+
+  configuredSources = validated;
+  res.json({ success: true, sources: validated });
+});
+
+// API: List all files in memory directories
+app.get('/api/files', async (_req: Request, res: Response) => {
+  try {
+    const result: Record<string, { tree: FileNode[]; fileCount: number; totalSize: number; path: string } | null> = {};
+
+    // Use configured sources, or fall back to default
+    const sourcesToScan = configuredSources.length > 0
+      ? configuredSources
+      : [
+          { id: 'openclaw', path: OPENCLAW_PATH },
+        ];
+
+    for (const source of sourcesToScan) {
+      try {
+        const basePath = getBasePath(source.id) || source.path;
+        const tree = await readDirRecursive(basePath, basePath);
+        result[source.id] = {
+          tree,
+          fileCount: countFiles(tree),
+          totalSize: totalSize(tree),
+          path: basePath,
+        };
+      } catch {
+        // Directory doesn't exist or not accessible
+        result[source.id] = null;
+      }
+    }
+
+    res.json(result);
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to read files', details: sanitizeError(error) });
+  }
+});
+
+// API: Read a specific file
+app.get('/api/file', async (req: Request, res: Response) => {
+  try {
+    const { path: filePath, source } = req.query;
+
+    if (!filePath || typeof filePath !== 'string') {
+      res.status(400).json({ error: 'Missing path parameter' });
+      return;
+    }
+
+    if (!source || typeof source !== 'string') {
+      res.status(400).json({ error: 'Missing source parameter' });
+      return;
+    }
+
+    // Get base path from configured sources or defaults
+    const basePath = getBasePath(source);
+    if (!basePath) {
+      res.status(400).json({ error: `Unknown source: ${source}` });
+      return;
+    }
+
+    const resolvedBase = path.resolve(basePath);
+    const fullPath = path.resolve(basePath, filePath);
+
+    // Security: ensure path is within the source directory
+    if (!fullPath.startsWith(resolvedBase + path.sep) && fullPath !== resolvedBase) {
+      res.status(403).json({ error: 'Access denied' });
+      return;
+    }
+
+    const content = await readFile(fullPath, 'utf-8');
+    const stats = await stat(fullPath);
+    const ext = extname(filePath).toLowerCase();
+
+    res.json({
+      path: filePath,
+      name: basename(filePath),
+      content,
+      size: stats.size,
+      modified: stats.mtime.toISOString(),
+      extension: ext,
+    });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      res.status(404).json({ error: 'File not found' });
+    } else {
+      res.status(500).json({ error: 'Failed to read file', details: sanitizeError(error) });
+    }
+  }
+});
+
+// API: Get openclaw config
+app.get('/api/config', async (_req: Request, res: Response) => {
+  try {
+    const configPath = join(OPENCLAW_PATH, 'openclaw.json');
+    const content = await readFile(configPath, 'utf-8');
+    const config = JSON.parse(content);
+    res.json({ config, path: configPath });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      res.status(404).json({ error: 'Config file not found', path: join(OPENCLAW_PATH, 'openclaw.json') });
+    } else {
+      res.status(500).json({ error: 'Failed to read config', details: sanitizeError(error) });
+    }
+  }
+});
+
+// API: Get agent logs from memory-swarm/logs
+app.get('/api/logs', async (_req: Request, res: Response) => {
+  try {
+    const logsPath = join(OPENCLAW_PATH, 'memory-swarm', 'logs');
+    const files = await readdir(logsPath);
+    const csvFiles = files.filter(f => f.endsWith('.csv')).sort().reverse();
+
+    const logs: { file: string; entries: LogEntry[] }[] = [];
+
+    // Read up to 5 most recent log files
+    for (const file of csvFiles.slice(0, 5)) {
+      try {
+        const content = await readFile(join(logsPath, file), 'utf-8');
+        const records = csvParse(content, {
+          columns: true,
+          skip_empty_lines: true,
+          relax_column_count: true,
+        }) as LogEntry[];
+        logs.push({ file, entries: records });
+      } catch {
+        // Skip files that can't be parsed
+      }
+    }
+
+    res.json({ logs, path: logsPath });
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+      res.status(404).json({ error: 'Logs directory not found', path: join(OPENCLAW_PATH, 'memory-swarm', 'logs') });
+    } else {
+      res.status(500).json({ error: 'Failed to read logs', details: sanitizeError(error) });
+    }
+  }
+});
+
+// API: Get memory layer stats
+app.get('/api/stats', async (_req: Request, res: Response) => {
+  try {
+    const stats = {
+      layers: {
+        session: { path: 'memory/session', files: 0, size: 0, cloud: false },
+        daily: { path: 'memory', files: 0, size: 0, cloud: false },
+        supermemory: { path: 'supermemory', files: 0, size: 0, cloud: false },
+        qmd: { path: 'memory-swarm', files: 0, size: 0, cloud: false },
+        config: { path: '.openclaw', files: 0, size: 0, cloud: false },
+      },
+      connected: {
+        openclaw: false,
+      },
+    };
+
+    // Check openclaw directories
+    try {
+      await stat(OPENCLAW_PATH);
+      stats.connected.openclaw = true;
+
+      // Daily logs: count .md files directly in ~/.openclaw/memory/
+      try {
+        const memoryPath = join(OPENCLAW_PATH, 'memory');
+        const entries = await readdir(memoryPath, { withFileTypes: true });
+        const mdFiles = entries.filter(e => e.isFile() && e.name.endsWith('.md'));
+        let totalMdSize = 0;
+        for (const file of mdFiles) {
+          const fileStat = await stat(join(memoryPath, file.name));
+          totalMdSize += fileStat.size;
+        }
+        stats.layers.daily.files = mdFiles.length;
+        stats.layers.daily.size = totalMdSize;
+      } catch {
+        // memory directory doesn't exist
+      }
+
+      // QMD Swarm
+      try {
+        const qmdPath = join(OPENCLAW_PATH, 'memory-swarm');
+        const tree = await readDirRecursive(qmdPath, qmdPath);
+        stats.layers.qmd.files = countFiles(tree);
+        stats.layers.qmd.size = totalSize(tree);
+      } catch {
+        // memory-swarm doesn't exist
+      }
+
+      // Config files
+      const configTree = await readDirRecursive(OPENCLAW_PATH, OPENCLAW_PATH);
+      stats.layers.config.files = countFiles(configTree);
+      stats.layers.config.size = totalSize(configTree);
+    } catch {
+      // openclaw not accessible
+    }
+
+    res.json(stats);
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to get stats', details: sanitizeError(error) });
+  }
+});
+
+// Health check
+app.get('/api/health', (_req: Request, res: Response) => {
+  const openclawExists = existsSync(OPENCLAW_PATH);
+
+  res.json({
+    status: 'ok',
+    timestamp: new Date().toISOString(),
+    openclaw: openclawExists,
+    paths: {
+      openclaw: OPENCLAW_PATH,
+    },
+  });
+});
+
+// Check if a path exists
+// Security: only allows checking if directories exist, not reading contents
+// Path must be within user's home directory
+app.get('/api/check-path', (req: Request, res: Response) => {
+  const pathParam = req.query.path as string;
+  if (!pathParam) {
+    res.status(400).json({ error: 'Path parameter required' });
+    return;
+  }
+
+  // Expand ~ to home directory
+  const expandedPath = pathParam.startsWith('~')
+    ? join(homedir(), pathParam.slice(1))
+    : pathParam;
+
+  // Security: only allow checking paths within user's home directory
+  const resolvedPath = path.resolve(expandedPath);
+  const home = homedir();
+
+  if (!resolvedPath.startsWith(home + path.sep) && resolvedPath !== home) {
+    res.status(403).json({ error: 'Access denied: path must be within home directory' });
+    return;
+  }
+
+  const exists = existsSync(expandedPath);
+
+  res.json({
+    path: pathParam,
+    expandedPath,
+    exists,
+  });
+});
+
+// Layer documentation configuration
+interface LayerDocConfig {
+  title: string;
+  summary: string;
+  details: string;
+  updateFrequency: string;
+  components: string[];
+  docFiles: string[]; // Files to try reading for additional content
+}
+
+const layerDocConfigs: Record<string, LayerDocConfig> = {
+  input: {
+    title: 'User Input',
+    summary: 'Your question or message to the AI, analyzed to determine context needs.',
+    details: 'Every conversation starts here. Your input is analyzed to identify what memories and context might be relevant. The system looks for keywords, topics, and patterns that match stored information across all memory layers.',
+    updateFrequency: 'Real-time',
+    components: ['Query parsing', 'Intent detection', 'Topic extraction', 'Context triggers'],
+    docFiles: [],
+  },
+  session: {
+    title: 'Session Context',
+    summary: 'Current conversation held in working memory during your session.',
+    details: 'Maintains immediate conversation context and working memory. This includes what you\'ve discussed, recent tool usage patterns, active tasks, and context switches. Cleared on restart but persists within a session for continuity.',
+    updateFrequency: 'Real-time during session',
+    components: ['Current conversation thread', 'Recent tool usage', 'Active task tracking', 'Context switches'],
+    docFiles: ['memory/CONTEXT_BUILDER.md'],
+  },
+  daily: {
+    title: 'Daily Logs',
+    summary: 'Markdown files capturing insights and summaries from each day.',
+    details: 'Written at session end, these logs capture important decisions, preferences learned, and context from each day. They bridge sessions so the AI remembers yesterday\'s discussions and decisions without needing to reload everything.',
+    updateFrequency: 'Daily (end of session)',
+    components: ['Session summaries', 'Decisions made', 'Preferences learned', 'Important context'],
+    docFiles: ['memory/PERSISTENT_MEMORY.md', 'MEMORY.md'],
+  },
+  supermemory: {
+    title: 'SuperMemory',
+    summary: 'Cloud-based semantic search over long-term memories.',
+    details: 'A cloud service that provides semantic search over your entire memory history. When you ask about past projects, preferences, or decisions from weeks or months ago, SuperMemory finds relevant context using vector embeddings and similarity search.',
+    updateFrequency: 'Continuous (cloud sync)',
+    components: ['Vector embeddings', 'Semantic search', 'Long-term storage', 'Cross-session recall'],
+    docFiles: ['memory/ADVANCED_MEMORY_ARCHITECTURE.md'],
+  },
+  qmd: {
+    title: 'QMD Swarm',
+    summary: 'Multi-agent system that orchestrates memory retrieval and synthesis.',
+    details: 'The Query-Memory-Dispatch system coordinates retrieval from all memory layers. It runs on every query, combining results from session context, daily logs, and SuperMemory into a coherent context package that informs the response.',
+    updateFrequency: 'Every query',
+    components: ['Query analysis', 'Parallel retrieval', 'Result ranking', 'Context synthesis'],
+    docFiles: ['AGENTS.md', 'memory/ADVANCED_MEMORY_ARCHITECTURE.md'],
+  },
+  output: {
+    title: 'Response',
+    summary: 'AI\'s answer enriched with retrieved memories and full context.',
+    details: 'The final response is generated after gathering context from all memory layers. It\'s personalized based on your history, preferences, and the specific context retrieved for your query.',
+    updateFrequency: 'Per response',
+    components: ['Context integration', 'Personalization', 'Response generation', 'Memory-informed answers'],
+    docFiles: [],
+  },
+};
+
+// API: Get layer documentation
+app.get('/api/layer-docs/:layer', async (req: Request, res: Response) => {
+  const layer = req.params.layer as string;
+
+  const config = layerDocConfigs[layer];
+  if (!config) {
+    res.status(404).json({ error: 'Layer not found' });
+    return;
+  }
+
+  // Try to read additional content from doc files
+  let additionalContent = '';
+  const sourceFiles: string[] = [];
+
+  for (const docFile of config.docFiles) {
+    try {
+      const fullPath = join(OPENCLAW_PATH, docFile);
+      const content = await readFile(fullPath, 'utf-8');
+      additionalContent += `\n\n---\n\n**From ${docFile}:**\n\n${content.slice(0, 2000)}${content.length > 2000 ? '...' : ''}`;
+      sourceFiles.push(docFile);
+    } catch {
+      // File not found, skip
+    }
+  }
+
+  res.json({
+    layer,
+    ...config,
+    additionalContent: additionalContent || null,
+    sourceFiles,
+  });
+});
+
+// Catch-all: serve index.html for client-side routing (production only)
+const indexPath = join(distPath, 'index.html');
+if (existsSync(indexPath)) {
+  app.get('{*path}', (_req: Request, res: Response) => {
+    res.sendFile(indexPath);
+  });
+}
+
+// Bind to localhost only to prevent network exposure
+const HOST = '127.0.0.1';
+
+app.listen(Number(PORT), HOST, () => {
+  console.log(`OpenClaw Memory Mapper API server running on http://${HOST}:${PORT}`);
+  console.log(`  OPENCLAW_PATH: ${OPENCLAW_PATH}`);
+});

package/server/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "esModuleInterop": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "declaration": false,
+    "outDir": "./dist"
+  },
+  "include": ["./**/*.ts"]
+}