jasper-recall 0.5.5 → 0.5.7

package/cli/digest-sessions.js

@@ -0,0 +1,233 @@
+#!/usr/bin/env node
+/**
+ * digest-sessions — Extract summaries from OpenClaw session logs
+ * 
+ * Usage:
+ *   npx jasper-recall digest-sessions [--all] [--recent N] [--dry-run]
+ *   digest-sessions [--all] [--recent N] [--dry-run]
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const readline = require('readline');
+
+// Config with environment overrides
+const WORKSPACE = process.env.RECALL_WORKSPACE || path.join(os.homedir(), '.openclaw', 'workspace');
+const SESSIONS_DIR = process.env.RECALL_SESSIONS_DIR || path.join(os.homedir(), '.openclaw', 'agents', 'main', 'sessions');
+const MEMORY_DIR = path.join(WORKSPACE, 'memory');
+const DIGEST_DIR = path.join(MEMORY_DIR, 'session-digests');
+const STATE_FILE = path.join(MEMORY_DIR, '.digest-state.json');
+
+// Parse args
+const args = process.argv.slice(2);
+const DRY_RUN = args.includes('--dry-run');
+const ALL = args.includes('--all');
+const recentIdx = args.indexOf('--recent');
+const RECENT = recentIdx !== -1 ? parseInt(args[recentIdx + 1], 10) : null;
+
+// Patterns to filter out from topics
+const SKIP_PATTERNS = [
+  /^\[message_id:/,
+  /^System:/,
+  /^\{/,
+  /^<session-init>/,
+  /^<session-identity>/,
+  /^<relevant-memories>/,
+  /^🔄 \*\*Fresh session/,
+  /^Read HEARTBEAT\.md/,
+  /^HEARTBEAT_OK/,
+  /^NO_REPLY/,
+  /^ANNOUNCE_SKIP/,
+  /^Agent-to-agent/,
+  /^📋 \*\*PR Review/,
+  /^🤖 Codex/,
+  /^✅ \*\*Hourly/,
+  /^The following memories/,
+  /^- \[memory\//,
+  /^###\s+(IDENTITY|SOUL|USER)\.md/,
+  /^cat ~/,
+  /^```/,
+  /^---$/,
+];
+
+function shouldSkip(line) {
+  const trimmed = line.trim();
+  if (!trimmed || trimmed.length < 5) return true;
+  return SKIP_PATTERNS.some(p => p.test(trimmed));
+}
+
+async function readState() {
+  try {
+    if (fs.existsSync(STATE_FILE)) {
+      return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
+    }
+  } catch {}
+  return { processed: [], lastRun: 0 };
+}
+
+function saveState(state) {
+  state.lastRun = Date.now();
+  fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+}
+
+async function parseSession(sessionFile) {
+  const topics = [];
+  const toolCounts = {};
+  let messageCount = 0;
+
+  const rl = readline.createInterface({
+    input: fs.createReadStream(sessionFile),
+    crlfDelay: Infinity
+  });
+
+  for await (const line of rl) {
+    try {
+      const entry = JSON.parse(line);
+      messageCount++;
+      
+      // Extract user messages for topics
+      if (entry.message?.role === 'user') {
+        let content = entry.message.content;
+        
+        // Handle array content (multi-part messages)
+        if (Array.isArray(content)) {
+          content = content
+            .filter(p => p.type === 'text')
+            .map(p => p.text)
+            .join(' ');
+        }
+        
+        if (typeof content === 'string') {
+          // Split into lines and filter
+          const lines = content.split('\n');
+          for (const l of lines) {
+            if (!shouldSkip(l) && topics.length < 20) {
+              topics.push(l.trim().slice(0, 200));
+            }
+          }
+        }
+      }
+      
+      // Count tool usage
+      if (entry.message?.role === 'assistant' && Array.isArray(entry.message.content)) {
+        for (const part of entry.message.content) {
+          if (part.type === 'toolCall' || part.type === 'tool_use') {
+            const name = part.name || part.toolName || 'unknown';
+            toolCounts[name] = (toolCounts[name] || 0) + 1;
+          }
+        }
+      }
+    } catch {
+      // Skip malformed lines
+    }
+  }
+
+  // Sort tools by usage
+  const tools = Object.entries(toolCounts)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 5)
+    .map(([name, count]) => `${name} (${count}x)`)
+    .join(', ');
+
+  return { topics: topics.slice(0, 10), tools: tools || 'none', messageCount };
+}
+
+async function main() {
+  // Ensure directories exist
+  fs.mkdirSync(DIGEST_DIR, { recursive: true });
+
+  // Check sessions dir
+  if (!fs.existsSync(SESSIONS_DIR)) {
+    console.log(`⚠ Sessions directory not found: ${SESSIONS_DIR}`);
+    process.exit(0);
+  }
+
+  // Get session files
+  const sessionFiles = fs.readdirSync(SESSIONS_DIR)
+    .filter(f => f.endsWith('.jsonl'))
+    .map(f => f.replace('.jsonl', ''));
+
+  if (sessionFiles.length === 0) {
+    console.log('No session files found.');
+    process.exit(0);
+  }
+
+  // Load state
+  const state = await readState();
+  const processed = new Set(state.processed);
+
+  // Filter to new sessions (unless --all)
+  let toProcess = ALL 
+    ? sessionFiles 
+    : sessionFiles.filter(s => !processed.has(s));
+
+  // Apply --recent limit
+  if (RECENT && RECENT > 0) {
+    toProcess = toProcess.slice(-RECENT);
+  }
+
+  if (toProcess.length === 0) {
+    console.log('✓ No new sessions to digest.');
+    process.exit(0);
+  }
+
+  console.log('🦊 Jasper Recall — Session Digester');
+  console.log('='.repeat(40));
+  console.log(`Sessions to process: ${toProcess.length}\n`);
+
+  for (const sessionId of toProcess) {
+    const sessionFile = path.join(SESSIONS_DIR, `${sessionId}.jsonl`);
+    if (!fs.existsSync(sessionFile)) continue;
+
+    const stats = fs.statSync(sessionFile);
+    const size = (stats.size / 1024).toFixed(0) + 'K';
+    const date = stats.mtime.toISOString().split('T')[0];
+
+    console.log(`Processing: ${sessionId.slice(0, 8)}... (${size})`);
+
+    try {
+      const { topics, tools, messageCount } = await parseSession(sessionFile);
+      
+      const digestFile = path.join(DIGEST_DIR, `${sessionId.slice(0, 8)}-${date}.md`);
+      
+      const topicsFormatted = topics.length > 0
+        ? topics.map(t => `- ${t}`).join('\n')
+        : '- (no topics extracted)';
+
+      const content = `# Session ${sessionId.slice(0, 8)} — ${date}
+
+**Size:** ${size} | **Messages:** ${messageCount}
+**Tools:** ${tools}
+
+## Topics
+
+${topicsFormatted}
+
+---
+*Full session: ${sessionFile}*
+`;
+
+      if (!DRY_RUN) {
+        fs.writeFileSync(digestFile, content);
+        state.processed.push(sessionId);
+        console.log(`  ✓ Created: ${path.basename(digestFile)}`);
+      } else {
+        console.log(`  [dry-run] Would create: ${path.basename(digestFile)}`);
+      }
+    } catch (err) {
+      console.log(`  ✗ Error: ${err.message}`);
+    }
+  }
+
+  if (!DRY_RUN) {
+    saveState(state);
+  }
+
+  console.log(`\n✓ Digests saved to: ${DIGEST_DIR}`);
+}
+
+main().catch(err => {
+  console.error('Error:', err.message);
+  process.exit(1);
+});

package/cli/index-digests.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+/**
+ * index-digests — Index memory files into ChromaDB
+ * Wrapper for the Python script
+ */
+
+const { execSync, spawn } = require('child_process');
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+const VENV_PATH = path.join(os.homedir(), '.openclaw', 'rag-env');
+const PYTHON = path.join(VENV_PATH, 'bin', 'python');
+
+// Find the Python script - check multiple locations
+const SCRIPT_LOCATIONS = [
+  path.join(__dirname, '..', 'scripts', 'index-digests.py'),
+  path.join(os.homedir(), '.local', 'share', 'jasper-recall', 'scripts', 'index-digests.py'),
+];
+
+let scriptPath = null;
+for (const loc of SCRIPT_LOCATIONS) {
+  if (fs.existsSync(loc)) {
+    scriptPath = loc;
+    break;
+  }
+}
+
+if (!scriptPath) {
+  console.error('❌ index-digests.py not found. Run: npx jasper-recall setup');
+  process.exit(1);
+}
+
+if (!fs.existsSync(PYTHON)) {
+  console.error('❌ Python venv not found. Run: npx jasper-recall setup');
+  process.exit(1);
+}
+
+// Run the Python script
+const child = spawn(PYTHON, [scriptPath, ...process.argv.slice(2)], {
+  stdio: 'inherit',
+  env: { ...process.env }
+});
+
+child.on('exit', (code) => {
+  process.exit(code || 0);
+});

package/cli/recall.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+/**
+ * recall — Semantic search over indexed memory
+ * Wrapper for the Python script
+ */
+
+const { spawn } = require('child_process');
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+const VENV_PATH = path.join(os.homedir(), '.openclaw', 'rag-env');
+const PYTHON = path.join(VENV_PATH, 'bin', 'python');
+
+// Find the Python script - check multiple locations
+const SCRIPT_LOCATIONS = [
+  path.join(__dirname, '..', 'scripts', 'recall.py'),
+  path.join(os.homedir(), '.local', 'share', 'jasper-recall', 'scripts', 'recall.py'),
+];
+
+let scriptPath = null;
+for (const loc of SCRIPT_LOCATIONS) {
+  if (fs.existsSync(loc)) {
+    scriptPath = loc;
+    break;
+  }
+}
+
+if (!scriptPath) {
+  console.error('❌ recall.py not found. Run: npx jasper-recall setup');
+  process.exit(1);
+}
+
+if (!fs.existsSync(PYTHON)) {
+  console.error('❌ Python venv not found. Run: npx jasper-recall setup');
+  process.exit(1);
+}
+
+// Run the Python script
+const child = spawn(PYTHON, [scriptPath, ...process.argv.slice(2)], {
+  stdio: 'inherit',
+  env: { ...process.env }
+});
+
+child.on('exit', (code) => {
+  process.exit(code || 0);
+});

package/extensions/jasper-recall/index.ts

@@ -11,6 +11,7 @@
  */
 
 import { execSync } from 'child_process';
+import { readFileSync, existsSync } from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 
@@ -131,18 +132,33 @@ export default function register(api: PluginApi) {
       try {
         let prependParts: string[] = [];
         
-        // If fresh session, remind agent to read identity files
+        // If fresh session, inject identity files directly into context
         if (isFreshSession) {
-          api.logger.info('[jasper-recall] Fresh session detected - adding identity reminder');
-          prependParts.push(`<session-init>
-🔄 **Fresh session detected.** Before responding, read your identity files:
-\`\`\`bash
-cat ~/.openclaw/workspace/IDENTITY.md
-cat ~/.openclaw/workspace/SOUL.md  
-cat ~/.openclaw/workspace/USER.md
-\`\`\`
-This ensures you maintain your persona and context after session reset.
-</session-init>`);
+          api.logger.info('[jasper-recall] Fresh session detected - injecting identity context');
+          
+          const workspace = path.join(os.homedir(), '.openclaw', 'workspace');
+          const identityFiles = ['IDENTITY.md', 'SOUL.md', 'USER.md'];
+          const identityParts: string[] = [];
+          
+          for (const file of identityFiles) {
+            const filePath = path.join(workspace, file);
+            if (existsSync(filePath)) {
+              try {
+                const content = readFileSync(filePath, 'utf8');
+                identityParts.push(`### ${file}\n${content}`);
+              } catch (err: any) {
+                api.logger.warn(`[jasper-recall] Failed to read ${file}: ${err.message}`);
+              }
+            }
+          }
+          
+          if (identityParts.length > 0) {
+            prependParts.push(`<session-identity>
+🔄 **Fresh session.** Your identity files:
+
+${identityParts.join('\n\n---\n\n')}
+</session-identity>`);
+          }
         }
         
         const results = runRecall(event.prompt, {
@@ -180,18 +196,33 @@ ${memoryContext}
       } catch (err: any) {
         api.logger.warn(`[jasper-recall] Auto-recall failed: ${err.message}`);
         
-        // Still inject identity reminder on fresh session even if recall fails
+        // Still inject identity context on fresh session even if recall fails
         if (isFreshSession) {
-          return {
-            prependContext: `<session-init>
-🔄 **Fresh session detected.** Before responding, read your identity files:
-\`\`\`bash
-cat ~/.openclaw/workspace/IDENTITY.md
-cat ~/.openclaw/workspace/SOUL.md
-cat ~/.openclaw/workspace/USER.md
-\`\`\`
-</session-init>`,
-          };
+          const workspace = path.join(os.homedir(), '.openclaw', 'workspace');
+          const identityFiles = ['IDENTITY.md', 'SOUL.md', 'USER.md'];
+          const identityParts: string[] = [];
+          
+          for (const file of identityFiles) {
+            const filePath = path.join(workspace, file);
+            if (existsSync(filePath)) {
+              try {
+                const content = readFileSync(filePath, 'utf8');
+                identityParts.push(`### ${file}\n${content}`);
+              } catch {
+                // Skip unreadable files
+              }
+            }
+          }
+          
+          if (identityParts.length > 0) {
+            return {
+              prependContext: `<session-identity>
+🔄 **Fresh session.** Your identity files:
+
+${identityParts.join('\n\n---\n\n')}
+</session-identity>`,
+            };
+          }
         }
       }
     });
@@ -295,6 +326,38 @@ cat ~/.openclaw/workspace/USER.md
     },
   });
 
+  // ============================================================================
+  // Command: /digest-sessions
+  // ============================================================================
+
+  api.registerCommand({
+    name: 'digest-sessions',
+    description: 'Extract summaries from session logs into memory',
+    acceptsArgs: true,
+    requireAuth: true,
+    handler: async (ctx: { args?: string }) => {
+      try {
+        const args = ctx.args?.trim() || '';
+        const digestPath = path.join(BIN_PATH, 'digest-sessions');
+        
+        // Check if digest-sessions exists in PATH, otherwise use npx
+        let cmd: string;
+        try {
+          execSync(`which ${digestPath}`, { encoding: 'utf8' });
+          cmd = `${digestPath} ${args}`;
+        } catch {
+          // Fall back to npx
+          cmd = `npx jasper-recall digest-sessions ${args}`;
+        }
+        
+        const output = execSync(cmd, { encoding: 'utf8', timeout: 300000 });
+        return { text: `🗂️ **Session Digests**\n\n${output}` };
+      } catch (err: any) {
+        return { text: `❌ Digest failed: ${err.message}` };
+      }
+    },
+  });
+
   // ============================================================================
   // Command: /jasper-recall setup
   // ============================================================================

package/package.json

@@ -1,10 +1,13 @@
 {
   "name": "jasper-recall",
-  "version": "0.5.5",
+  "version": "0.5.7",
   "description": "Local RAG system for AI agent memory using ChromaDB and sentence-transformers",
   "main": "src/index.js",
   "bin": {
-    "jasper-recall": "./cli/jasper-recall.js"
+    "jasper-recall": "./cli/jasper-recall.js",
+    "digest-sessions": "./cli/digest-sessions.js",
+    "index-digests": "./cli/index-digests.js",
+    "recall": "./cli/recall.js"
   },
   "openclaw": {
     "extensions": [