openclaw-mem 1.0.3 → 1.2.1

package/debug-logger.js

@@ -0,0 +1,280 @@
+#!/usr/bin/env node
+/**
+ * OpenClaw-Mem 调试日志工具
+ *
+ * 显示完整的 API 调用流程：
+ * 1. 用户消息
+ * 2. AI 调用的 exec 命令
+ * 3. API 收到的请求
+ * 4. API 返回的响应
+ * 5. AI 最终回复
+ */
+
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import http from 'http';
+
+const SESSIONS_DIR = path.join(os.homedir(), '.openclaw', 'agents', 'main', 'sessions');
+const LOG_FILE = '/tmp/openclaw-debug.log';
+
+// 颜色
+const c = {
+  reset: '\x1b[0m',
+  bold: '\x1b[1m',
+  dim: '\x1b[2m',
+  red: '\x1b[31m',
+  green: '\x1b[32m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  magenta: '\x1b[35m',
+  cyan: '\x1b[36m',
+  bgRed: '\x1b[41m',
+  bgGreen: '\x1b[42m',
+  bgYellow: '\x1b[43m',
+  bgBlue: '\x1b[44m',
+};
+
+function log(msg) {
+  const ts = new Date().toLocaleTimeString('zh-CN', { hour12: false });
+  const line = `[${ts}] ${msg}`;
+  console.log(line);
+  fs.appendFileSync(LOG_FILE, line.replace(/\x1b\[[0-9;]*m/g, '') + '\n');
+}
+
+function logSection(title, color = 'cyan') {
+  console.log();
+  console.log(`${c[color]}${'═'.repeat(60)}${c.reset}`);
+  console.log(`${c[color]}${c.bold}  ${title}${c.reset}`);
+  console.log(`${c[color]}${'═'.repeat(60)}${c.reset}`);
+}
+
+function logEvent(icon, label, content, color = 'white') {
+  const ts = new Date().toLocaleTimeString('zh-CN', { hour12: false });
+  console.log();
+  console.log(`${c.dim}[${ts}]${c.reset} ${icon} ${c[color]}${c.bold}${label}${c.reset}`);
+  if (content) {
+    const lines = String(content).split('\n').slice(0, 15);
+    lines.forEach(line => {
+      console.log(`  ${c.dim}│${c.reset} ${line.slice(0, 120)}`);
+    });
+    if (String(content).split('\n').length > 15) {
+      console.log(`  ${c.dim}│ ... (更多内容省略)${c.reset}`);
+    }
+  }
+  fs.appendFileSync(LOG_FILE, `[${ts}] ${icon} ${label}\n${content || ''}\n\n`);
+}
+
+// 监控 Session 文件
+let lastSessionFile = null;
+let processedLines = new Set();
+
+function findLatestSession() {
+  try {
+    const files = fs.readdirSync(SESSIONS_DIR)
+      .filter(f => f.endsWith('.jsonl'))
+      .map(f => ({
+        name: f,
+        path: path.join(SESSIONS_DIR, f),
+        mtime: fs.statSync(path.join(SESSIONS_DIR, f)).mtime
+      }))
+      .sort((a, b) => b.mtime - a.mtime);
+    return files[0]?.path || null;
+  } catch {
+    return null;
+  }
+}
+
+function watchSession() {
+  const latestSession = findLatestSession();
+
+  if (latestSession !== lastSessionFile) {
+    lastSessionFile = latestSession;
+    processedLines.clear();
+    if (latestSession) {
+      logEvent('📁', '监控会话文件', path.basename(latestSession), 'cyan');
+    }
+  }
+
+  if (!lastSessionFile) return;
+
+  try {
+    const content = fs.readFileSync(lastSessionFile, 'utf-8');
+    const lines = content.trim().split('\n');
+
+    for (const line of lines) {
+      const lineKey = line.slice(0, 150);
+      if (processedLines.has(lineKey)) continue;
+      processedLines.add(lineKey);
+
+      try {
+        const entry = JSON.parse(line);
+        if (entry.type !== 'message') continue;
+
+        const msg = entry.message;
+        if (!msg) continue;
+
+        // 用户消息
+        if (msg.role === 'user') {
+          let text = '';
+          if (Array.isArray(msg.content)) {
+            const textPart = msg.content.find(c => c.type === 'text');
+            text = textPart?.text || '';
+          } else {
+            text = msg.content || '';
+          }
+          if (text && !text.startsWith('A new session')) {
+            logEvent('👤', '用户消息', text.slice(0, 300), 'green');
+          }
+        }
+
+        // AI 响应
+        if (msg.role === 'assistant' && msg.content) {
+          // 检查工具调用
+          if (Array.isArray(msg.content)) {
+            for (const block of msg.content) {
+              if (block.type === 'toolCall') {
+                const toolName = block.name;
+                const args = block.arguments;
+
+                if (toolName === 'exec') {
+                  let argsObj;
+                  try {
+                    argsObj = typeof args === 'string' ? JSON.parse(args) : args;
+                  } catch {
+                    argsObj = { command: args };
+                  }
+
+                  logEvent('🔧', `工具调用: ${toolName}`,
+                    `命令: ${argsObj.command || JSON.stringify(argsObj)}`, 'yellow');
+
+                  // 高亮显示 curl 命令
+                  if (argsObj.command && argsObj.command.includes('curl')) {
+                    console.log(`  ${c.bgYellow}${c.bold} CURL 命令 ${c.reset}`);
+                    console.log(`  ${c.yellow}${argsObj.command}${c.reset}`);
+                  }
+                }
+              }
+
+              if (block.type === 'text' && block.text) {
+                logEvent('🤖', 'AI 响应', block.text.slice(0, 400), 'blue');
+              }
+            }
+          }
+        }
+
+        // 工具结果
+        if (msg.role === 'toolResult') {
+          const toolName = msg.toolName || 'unknown';
+          let result = '';
+          if (Array.isArray(msg.content)) {
+            const textPart = msg.content.find(c => c.type === 'text');
+            result = textPart?.text || '';
+          } else {
+            result = msg.content || '';
+          }
+
+          if (toolName === 'exec') {
+            const isEmpty = !result || result === '(no output)';
+            const color = isEmpty ? 'red' : 'green';
+            const icon = isEmpty ? '❌' : '✅';
+            logEvent(icon, `exec 返回结果`, result || '(空)', color);
+
+            if (isEmpty) {
+              console.log(`  ${c.bgRed}${c.bold} 警告: exec 返回空！curl 可能失败了 ${c.reset}`);
+            }
+          }
+        }
+
+      } catch (e) {
+        // 解析失败，忽略
+      }
+    }
+  } catch {
+    // 文件读取失败
+  }
+}
+
+// 创建代理 API 服务器来拦截请求
+const PROXY_PORT = 18791;
+const TARGET_PORT = 18790;
+
+function startProxyServer() {
+  const proxy = http.createServer((req, res) => {
+    let body = '';
+    req.on('data', chunk => body += chunk);
+    req.on('end', () => {
+      // 记录请求
+      logEvent('📥', `API 请求: ${req.method} ${req.url}`,
+        body ? `Body: ${body}` : '(无 body)', 'magenta');
+
+      // 转发到真实 API
+      const options = {
+        hostname: '127.0.0.1',
+        port: TARGET_PORT,
+        path: req.url,
+        method: req.method,
+        headers: req.headers
+      };
+
+      const proxyReq = http.request(options, (proxyRes) => {
+        let responseBody = '';
+        proxyRes.on('data', chunk => responseBody += chunk);
+        proxyRes.on('end', () => {
+          // 记录响应
+          const preview = responseBody.slice(0, 500);
+          logEvent('📤', `API 响应 (${proxyRes.statusCode})`, preview,
+            proxyRes.statusCode === 200 ? 'green' : 'red');
+
+          res.writeHead(proxyRes.statusCode, proxyRes.headers);
+          res.end(responseBody);
+        });
+      });
+
+      proxyReq.on('error', (err) => {
+        logEvent('❌', 'API 请求失败', err.message, 'red');
+        res.writeHead(500);
+        res.end('Proxy error');
+      });
+
+      if (body) proxyReq.write(body);
+      proxyReq.end();
+    });
+  });
+
+  proxy.listen(PROXY_PORT, '127.0.0.1', () => {
+    log(`${c.green}代理服务器运行在 http://127.0.0.1:${PROXY_PORT}${c.reset}`);
+    log(`${c.dim}(转发到 http://127.0.0.1:${TARGET_PORT})${c.reset}`);
+  });
+
+  return proxy;
+}
+
+// 主函数
+function main() {
+  // 清空日志文件
+  fs.writeFileSync(LOG_FILE, '');
+
+  logSection('OpenClaw-Mem 调试日志工具', 'cyan');
+  log('');
+  log(`${c.bold}监控内容:${c.reset}`);
+  log(`  • Session 文件: ${SESSIONS_DIR}`);
+  log(`  • 日志文件: ${LOG_FILE}`);
+  log('');
+  log(`${c.yellow}提示: 按 Ctrl+C 退出${c.reset}`);
+  log('');
+
+  // 开始监控
+  setInterval(watchSession, 300);
+
+  logSection('等待事件...', 'dim');
+}
+
+main();
+
+process.on('SIGINT', () => {
+  console.log();
+  log(`${c.yellow}调试日志已停止${c.reset}`);
+  log(`${c.dim}完整日志保存在: ${LOG_FILE}${c.reset}`);
+  process.exit(0);
+});

package/extractor.js

@@ -0,0 +1,211 @@
+/**
+ * OpenClaw-Mem LLM Extractor
+ * Uses the local OpenClaw Gateway model to extract concepts and metadata
+ */
+
+import { callGatewayChat } from './gateway-llm.js';
+
+// Cache for extracted concepts (to avoid repeated API calls)
+const conceptCache = new Map();
+const CACHE_MAX_SIZE = 1000;
+const CACHE_TTL = 60 * 60 * 1000; // 1 hour
+
+function getCacheKey(text) {
+  // Simple hash for cache key
+  let hash = 0;
+  const str = text.slice(0, 500); // Only hash first 500 chars
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = ((hash << 5) - hash) + char;
+    hash = hash & hash;
+  }
+  return hash.toString(16);
+}
+
+function cleanCache() {
+  if (conceptCache.size > CACHE_MAX_SIZE) {
+    const now = Date.now();
+    for (const [key, value] of conceptCache.entries()) {
+      if (now - value.timestamp > CACHE_TTL) {
+        conceptCache.delete(key);
+      }
+    }
+    // If still too large, remove oldest entries
+    if (conceptCache.size > CACHE_MAX_SIZE) {
+      const entries = [...conceptCache.entries()];
+      entries.sort((a, b) => a[1].timestamp - b[1].timestamp);
+      const toRemove = entries.slice(0, entries.length - CACHE_MAX_SIZE / 2);
+      for (const [key] of toRemove) {
+        conceptCache.delete(key);
+      }
+    }
+  }
+}
+
+/**
+ * Extract concepts/keywords from text using LLM
+ * @param {string} text - The text to extract concepts from
+ * @param {object} options - Options
+ * @returns {Promise<string[]>} - Array of extracted concepts
+ */
+export async function extractConcepts(text, options = {}) {
+  if (!text || typeof text !== 'string' || text.trim().length < 10) {
+    return [];
+  }
+
+  // Check cache first
+  const cacheKey = getCacheKey(text);
+  const cached = conceptCache.get(cacheKey);
+  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+    return cached.concepts;
+  }
+
+  try {
+    const content = await callGatewayChat([{
+      role: 'user',
+      content: `Extract 3-7 key concepts/topics from this text. Return ONLY a JSON array of strings, no explanation.
+
+Text: "${text.slice(0, 800)}"
+
+JSON array:`
+    }], { sessionKey: 'extract-concepts', temperature: 0.2, max_tokens: 200 });
+
+    if (!content) return [];
+    // Parse JSON array from response
+    let concepts = [];
+    try {
+      // Try to extract JSON array from response
+      const match = content.match(/\[[\s\S]*?\]/);
+      if (match) {
+        concepts = JSON.parse(match[0]);
+      }
+    } catch (parseErr) {
+      console.error('[openclaw-mem] Failed to parse LLM response:', parseErr.message);
+      return [];
+    }
+
+    // Validate and clean concepts
+    concepts = concepts
+      .filter(c => typeof c === 'string' && c.length > 1 && c.length < 50)
+      .map(c => c.trim().toLowerCase())
+      .slice(0, 7);
+
+    // Cache the result
+    cleanCache();
+    conceptCache.set(cacheKey, {
+      concepts,
+      timestamp: Date.now()
+    });
+
+    return concepts;
+  } catch (err) {
+    console.error('[openclaw-mem] LLM extraction error:', err.message);
+    return [];
+  }
+}
+
+/**
+ * Extract structured information from a tool call
+ * @param {object} data - Tool call data
+ * @returns {Promise<object>} - Extracted information
+ */
+export async function extractFromToolCall(data) {
+  const { tool_name, tool_input, tool_response, filesRead, filesModified } = data;
+
+  // Build context for extraction
+  const inputStr = typeof tool_input === 'string'
+    ? tool_input.slice(0, 300)
+    : JSON.stringify(tool_input).slice(0, 300);
+
+  const responseStr = typeof tool_response === 'string'
+    ? tool_response.slice(0, 300)
+    : JSON.stringify(tool_response).slice(0, 300);
+
+  try {
+    const content = await callGatewayChat([{
+      role: 'user',
+      content: `Analyze this tool call and extract structured information. Return ONLY valid JSON.
+
+Tool: ${tool_name}
+Input: ${inputStr}
+Output: ${responseStr}
+Files read: ${filesRead?.join(', ') || 'none'}
+Files modified: ${filesModified?.join(', ') || 'none'}
+
+Return JSON with these fields:
+{
+  "type": "decision|bugfix|feature|refactor|discovery|testing|setup|other",
+  "narrative": "One sentence describing what happened",
+  "facts": ["fact1", "fact2"],
+  "concepts": ["keyword1", "keyword2", "keyword3"]
+}
+
+JSON:`
+    }], { sessionKey: 'extract-toolcall', temperature: 0.2, max_tokens: 300 });
+
+    if (!content) throw new Error('empty response');
+
+    // Parse JSON from response
+    const match = content.match(/\{[\s\S]*\}/);
+    if (match) {
+      const result = JSON.parse(match[0]);
+      return {
+        type: result.type || 'other',
+        narrative: result.narrative || '',
+        facts: Array.isArray(result.facts) ? result.facts.slice(0, 5) : [],
+        concepts: Array.isArray(result.concepts) ? result.concepts.slice(0, 7) : []
+      };
+    }
+  } catch (err) {
+    console.error('[openclaw-mem] Tool extraction error:', err.message);
+  }
+
+  // Return empty result on error
+  return {
+    type: 'other',
+    narrative: '',
+    facts: [],
+    concepts: []
+  };
+}
+
+/**
+ * Batch extract concepts from multiple texts
+ * @param {string[]} texts - Array of texts to extract from
+ * @returns {Promise<Map<string, string[]>>} - Map of text to concepts
+ */
+export async function batchExtractConcepts(texts) {
+  const results = new Map();
+
+  // Filter out cached results first
+  const uncached = [];
+  for (const text of texts) {
+    const cacheKey = getCacheKey(text);
+    const cached = conceptCache.get(cacheKey);
+    if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+      results.set(text, cached.concepts);
+    } else {
+      uncached.push(text);
+    }
+  }
+
+  // Process uncached in batches
+  const BATCH_SIZE = 5;
+  for (let i = 0; i < uncached.length; i += BATCH_SIZE) {
+    const batch = uncached.slice(i, i + BATCH_SIZE);
+    const promises = batch.map(text => extractConcepts(text));
+    const batchResults = await Promise.all(promises);
+
+    for (let j = 0; j < batch.length; j++) {
+      results.set(batch[j], batchResults[j]);
+    }
+  }
+
+  return results;
+}
+
+export default {
+  extractConcepts,
+  extractFromToolCall,
+  batchExtractConcepts
+};

package/gateway-llm.js

@@ -0,0 +1,155 @@
+/**
+ * DeepSeek LLM helper
+ * Calls the DeepSeek OpenAI-compatible endpoint to summarize sessions.
+ */
+
+const SUMMARY_SESSION_PREFIX = 'mem-summary:';
+const DEFAULT_DEEPSEEK_BASE_URL = 'https://api.deepseek.com/v1';
+const DEFAULT_DEEPSEEK_MODEL = 'deepseek-chat';
+
+function getDeepSeekBaseUrl() {
+  return process.env.DEEPSEEK_BASE_URL
+    || DEFAULT_DEEPSEEK_BASE_URL;
+}
+
+function getDeepSeekApiKey() {
+  return process.env.DEEPSEEK_API_KEY || '';
+}
+
+function getDeepSeekModel() {
+  return process.env.DEEPSEEK_MODEL
+    || DEFAULT_DEEPSEEK_MODEL;
+}
+
+function truncateText(text, maxChars) {
+  if (!text) return '';
+  if (text.length <= maxChars) return text;
+  return text.slice(0, maxChars) + '…';
+}
+
+function formatTranscript(messages, maxChars = 8000) {
+  const lines = [];
+  for (const m of messages) {
+    const role = (m.role || 'unknown').toUpperCase();
+    const content = String(m.content || '').replace(/\s+/g, ' ').trim();
+    if (!content) continue;
+    lines.push(`${role}: ${content}`);
+  }
+  return truncateText(lines.join('\n'), maxChars);
+}
+
+function parseSummaryJson(text) {
+  if (!text) return null;
+  const match = text.match(/\{[\s\S]*\}/);
+  if (!match) return null;
+  try {
+    const obj = JSON.parse(match[0]);
+    return obj && typeof obj === 'object' ? obj : null;
+  } catch {
+    return null;
+  }
+}
+
+function normalizeSummaryFields(obj) {
+  if (!obj) return null;
+  const pick = (key) => {
+    const val = obj[key];
+    if (typeof val === 'string') return val.trim();
+    if (val == null) return '';
+    return String(val).trim();
+  };
+  return {
+    request: pick('request'),
+    learned: pick('learned'),
+    completed: pick('completed'),
+    next_steps: pick('next_steps')
+  };
+}
+
+async function callGatewayChat(messages, options = {}) {
+  const {
+    sessionKey = 'unknown',
+    temperature = 0.2,
+    max_tokens = 300,
+    model
+  } = options;
+  const apiKey = getDeepSeekApiKey();
+  if (!apiKey) {
+    console.log('[openclaw-mem] No DEEPSEEK_API_KEY found');
+    return null;
+  }
+  const baseUrl = getDeepSeekBaseUrl();
+  const resolvedModel = model || getDeepSeekModel();
+  const url = `${baseUrl}/chat/completions`;
+  const payload = {
+    model: resolvedModel,
+    stream: false,
+    temperature,
+    max_tokens,
+    messages
+  };
+
+  const headers = {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${apiKey}`
+  };
+
+  try {
+    console.log('[openclaw-mem] Calling DeepSeek API...');
+    const res = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(payload)
+    });
+    if (!res.ok) {
+      const errText = await res.text();
+      console.error('[openclaw-mem] DeepSeek API error:', res.status, errText);
+      return null;
+    }
+    const json = await res.json();
+    const content = json?.choices?.[0]?.message?.content || '';
+    console.log('[openclaw-mem] DeepSeek response received');
+    return content;
+  } catch (err) {
+    console.error('[openclaw-mem] DeepSeek fetch error:', err.message);
+    return null;
+  }
+}
+
+export async function summarizeSession(messages, options = {}) {
+  const { sessionKey = 'unknown' } = options;
+  const transcript = formatTranscript(messages);
+  if (!transcript) return null;
+
+  const buildPrompts = (strict = false) => {
+    const systemPrompt = [
+      '你是一个对话总结助手。请用中文总结这段对话，返回一个 JSON 对象，包含以下字段：',
+      '- request: 用户的主要问题或需求（一句话）',
+      '- learned: 用户从对话中学到了什么',
+      '- completed: 完成了什么任务或解答',
+      '- next_steps: 建议的下一步行动',
+      '只返回 JSON 对象，不要 markdown 代码块，不要其他内容。',
+      strict ? '重要：只输出纯 JSON，不要任何额外文字。' : ''
+    ].filter(Boolean).join('\n');
+    const userPrompt = '对话记录:\n' + transcript + '\n\nJSON:';
+    return [
+      { role: 'system', content: systemPrompt },
+      { role: 'user', content: userPrompt }
+    ];
+  };
+
+  // First attempt
+  let content = await callGatewayChat(buildPrompts(false), { sessionKey, temperature: 0.2, max_tokens: 300 });
+  let parsed = parseSummaryJson(content || '');
+  if (parsed) return normalizeSummaryFields(parsed);
+
+  // Retry once with stricter instruction
+  content = await callGatewayChat(buildPrompts(true), { sessionKey, temperature: 0.2, max_tokens: 300 });
+  parsed = parseSummaryJson(content || '');
+  if (parsed) return normalizeSummaryFields(parsed);
+
+  return null;
+}
+
+export const INTERNAL_SUMMARY_PREFIX = SUMMARY_SESSION_PREFIX;
+export { callGatewayChat };