memshell 0.2.1 → 0.4.0

package/bin/memshell.js

@@ -1,89 +1,300 @@
 #!/usr/bin/env node
 'use strict';
 
+const fs = require('fs');
+const path = require('path');
 const mem = require('../src/index');
+const { LocalStore } = require('../src/index');
 
 const args = process.argv.slice(2);
 const cmd = args[0];
-const rest = args.slice(1).join(' ');
 
 const HELP = `
-  mem.sh — persistent memory for AI agents
+  \x1b[1mmem.sh\x1b[0m — persistent memory for AI agents
 
-  Usage:
-    mem set <text>          Store a memory
-    mem recall <query>      Semantic recall
-    mem list                List all memories
-    mem forget <id>         Delete a memory by ID
-    mem clear               Wipe all memories
-    mem serve [--port N]    Start API server
+  \x1b[36mCore Commands:\x1b[0m
+    memshell set <text>              Store a memory
+    memshell recall <query>          Semantic recall
+    memshell list                    List all memories
+    memshell forget <id>             Delete a memory by ID
+    memshell clear                   Wipe all memories
+    memshell important <id>          Boost memory importance
 
-  Options:
+  \x1b[36mAuto-Ingest:\x1b[0m
+    memshell ingest <file>           Extract facts from a file
+    memshell ingest --stdin          Extract facts from piped text
+    memshell ingest --watch <dir>    Watch a directory for new files
+
+  \x1b[36mIntegrations:\x1b[0m
+    memshell connect openclaw        Watch OpenClaw session transcripts
+    memshell daemon                  Run continuous ingestion daemon
+
+  \x1b[36mManagement:\x1b[0m
+    memshell config set <key> <val>  Set config value
+    memshell config get [key]        Show config
+    memshell stats                   Show memory statistics
+    memshell export                  Export all memories as JSON
+    memshell import <file.json>      Import memories from JSON
+    memshell serve [--port N]        Start API server
+
+  \x1b[36mOptions:\x1b[0m
     --agent <name>          Agent namespace
     --api <url>             Use remote API instead of local
     --key <key>             API key for remote server
+    --tags <t1,t2>          Tags (comma-separated)
+    --top <N>               Return top N results only
+    --embeddings            Enable OpenAI embeddings (needs OPENAI_API_KEY)
 
-  Examples:
-    mem set "user prefers dark mode"
-    mem recall "what theme?"
-    mem list
-    mem forget 3
+  \x1b[36mExamples:\x1b[0m
+    memshell set "user prefers dark mode" --tags preferences,ui
+    memshell recall "what theme?" --tags preferences --top 3
+    echo "User likes vim and dark mode" | memshell ingest --stdin
+    memshell connect openclaw
+    memshell config set apiKey sk-...
 `;
 
 // Parse flags
 function flag(name) {
   const i = args.indexOf('--' + name);
   if (i === -1) return null;
-  return args[i + 1] || true;
+  if (i + 1 < args.length && !args[i + 1].startsWith('--')) return args[i + 1];
+  return true;
+}
+
+function hasFlag(name) {
+  return args.includes('--' + name);
+}
+
+// Smarter text extraction: skip flag values
+function getText() {
+  const skip = new Set(['--agent', '--api', '--key', '--tags', '--top', '--port', '--watch']);
+  const parts = [];
+  let i = 1;
+  while (i < args.length) {
+    if (skip.has(args[i])) { i += 2; continue; }
+    if (args[i] === '--embeddings' || args[i] === '--stdin' || args[i] === '--force') { i++; continue; }
+    if (args[i].startsWith('--')) { i++; continue; }
+    parts.push(args[i]);
+    i++;
+  }
+  return parts.join(' ').replace(/^["']|["']$/g, '');
+}
+
+function readStdin() {
+  return new Promise((resolve) => {
+    let data = '';
+    process.stdin.setEncoding('utf8');
+    process.stdin.on('data', chunk => data += chunk);
+    process.stdin.on('end', () => resolve(data));
+    // If nothing after 100ms and stdin is a TTY, resolve empty
+    if (process.stdin.isTTY) resolve('');
+  });
 }
 
 async function main() {
   const agent = flag('agent') || 'default';
   const api = flag('api');
   const key = flag('key');
+  const tags = flag('tags') || '';
+  const top = flag('top') ? parseInt(flag('top')) : null;
+  const useEmbeddings = hasFlag('embeddings');
+
+  const configOpts = { agent };
+  if (api) { configOpts.api = api; configOpts.key = key; }
+  if (useEmbeddings) { configOpts.openaiKey = process.env.OPENAI_API_KEY; }
 
-  if (api) mem.configure({ api, key, agent });
-  else mem.configure({ agent });
+  mem.configure(configOpts);
 
-  const opts = { agent };
+  const opts = { agent, tags, top };
 
   switch (cmd) {
     case 'set': case 's': case 'save': case 'remember': {
-      const text = args.slice(1).filter(a => !a.startsWith('--')).join(' ').replace(/^["']|["']$/g, '');
-      if (!text) return console.log('Usage: mem set <text>');
-      const r = await mem.set(text, opts);
-      console.log(`✓ Stored (id: ${r.id})`);
+      const text = getText();
+      if (!text) return console.log('Usage: memshell set <text>');
+      const r = await mem.set(text, { ...opts, tags });
+      console.log(`\x1b[32m+\x1b[0m Stored (id: \x1b[1m${r.id}\x1b[0m)${tags ? ` [tags: ${tags}]` : ''}`);
       break;
     }
     case 'recall': case 'r': case 'search': case 'q': {
-      const query = args.slice(1).filter(a => !a.startsWith('--')).join(' ').replace(/^["']|["']$/g, '');
-      if (!query) return console.log('Usage: mem recall <query>');
+      const query = getText();
+      if (!query) return console.log('Usage: memshell recall <query>');
       const results = await mem.recall(query, opts);
-      if (!results.length) return console.log('No memories found.');
+      if (!results.length) return console.log('\x1b[33mNo memories found.\x1b[0m');
       for (const r of results) {
-        console.log(`  [${r.id}] ${r.text} (score: ${r.score})`);
+        const tagStr = r.tags ? ` \x1b[35m[${r.tags}]\x1b[0m` : '';
+        const srcStr = r.source && r.source !== 'manual' ? ` \x1b[2m(src: ${r.source})\x1b[0m` : '';
+        const recallStr = r.recall_count ? ` \x1b[2m(recalled ${r.recall_count}x)\x1b[0m` : '';
+        console.log(`  \x1b[36m[${r.id}]\x1b[0m ${r.text} \x1b[33m(score: ${r.score})\x1b[0m${tagStr}${srcStr}${recallStr}`);
       }
       break;
     }
     case 'list': case 'ls': case 'l': {
       const all = await mem.list(opts);
-      if (!all.length) return console.log('No memories stored.');
+      if (!all.length) return console.log('\x1b[33mNo memories stored.\x1b[0m');
       for (const r of all) {
-        console.log(`  [${r.id}] ${r.text}  (${r.created_at})`);
+        const tagStr = r.tags ? ` \x1b[35m[${r.tags}]\x1b[0m` : '';
+        const imp = r.importance !== 1.0 ? ` \x1b[33m*${r.importance.toFixed(1)}\x1b[0m` : '';
+        const srcStr = r.source && r.source !== 'manual' ? ` \x1b[2m[${r.source}]\x1b[0m` : '';
+        console.log(`  \x1b[36m[${r.id}]\x1b[0m ${r.text}${tagStr}${imp}${srcStr}  \x1b[2m(${r.created_at})\x1b[0m`);
       }
-      console.log(`\n  ${all.length} memor${all.length === 1 ? 'y' : 'ies'}`);
+      console.log(`\n  \x1b[1m${all.length}\x1b[0m memor${all.length === 1 ? 'y' : 'ies'}`);
       break;
     }
     case 'forget': case 'delete': case 'rm': {
       const id = args[1];
-      if (!id) return console.log('Usage: mem forget <id>');
+      if (!id) return console.log('Usage: memshell forget <id>');
       await mem.forget(id);
-      console.log(`✓ Forgotten (id: ${id})`);
+      console.log(`\x1b[32m+\x1b[0m Forgotten (id: ${id})`);
       break;
     }
     case 'clear': case 'wipe': case 'reset': {
       await mem.clear(opts);
-      console.log('✓ All memories cleared');
+      console.log('\x1b[32m+\x1b[0m All memories cleared');
+      break;
+    }
+    case 'important': case 'boost': {
+      const id = args[1];
+      if (!id) return console.log('Usage: memshell important <id>');
+      const r = await mem.important(Number(id));
+      if (!r) return console.log('\x1b[31mMemory not found.\x1b[0m');
+      console.log(`\x1b[32m+\x1b[0m Boosted memory ${r.id} -> importance: \x1b[1m${r.importance.toFixed(1)}\x1b[0m`);
+      break;
+    }
+    case 'stats': {
+      const s = await mem.stats(opts);
+      console.log(`\n  \x1b[1mMemory Stats\x1b[0m`);
+      console.log(`  Total:          \x1b[36m${s.total}\x1b[0m`);
+      console.log(`  Oldest:         ${s.oldest || 'n/a'}`);
+      console.log(`  Newest:         ${s.newest || 'n/a'}`);
+      console.log(`  Avg importance: \x1b[33m${s.avg_importance}\x1b[0m\n`);
+      break;
+    }
+    case 'export': {
+      const data = await mem.exportAll(opts);
+      console.log(JSON.stringify(data, null, 2));
+      break;
+    }
+    case 'import': {
+      const file = args[1];
+      if (!file) return console.log('Usage: memshell import <file.json>');
+      const raw = fs.readFileSync(path.resolve(file), 'utf8');
+      const data = JSON.parse(raw);
+      const r = await mem.importAll(Array.isArray(data) ? data : data.memories || []);
+      console.log(`\x1b[32m+\x1b[0m Imported ${r.imported} memories`);
+      break;
+    }
+    case 'ingest': {
+      const { ingestFile, ingest: ingestText } = require('../src/ingest');
+      const store = new LocalStore(undefined, useEmbeddings ? { openaiKey: process.env.OPENAI_API_KEY } : {});
+      await store.init();
+
+      if (hasFlag('stdin')) {
+        const text = await readStdin();
+        if (!text.trim()) return console.log('No input received via stdin.');
+        console.log('  Extracting facts from stdin...');
+        const result = await ingestText(text, store, { agent });
+        console.log(`\x1b[32m+\x1b[0m Extracted: ${result.extracted}, Stored: ${result.stored}, Duplicates: ${result.duplicates}`);
+      } else if (hasFlag('watch')) {
+        const dir = flag('watch');
+        if (!dir || dir === true) return console.log('Usage: memshell ingest --watch <directory>');
+        const { watchDirectory } = require('../src/ingest');
+        console.log('  Starting directory watcher (Ctrl+C to stop)...');
+        watchDirectory(dir, store, { agent });
+        // Keep process alive
+        process.on('SIGINT', () => { console.log('\n  Stopped.'); process.exit(0); });
+      } else {
+        const file = getText();
+        if (!file) return console.log('Usage: memshell ingest <file> | --stdin | --watch <dir>');
+        console.log(`  Ingesting: ${file}`);
+        const result = await ingestFile(file, store, { agent, force: hasFlag('force') });
+        if (result.skipped) {
+          console.log(`  Skipped: ${result.file} (already processed, use --force to re-ingest)`);
+        } else {
+          console.log(`\x1b[32m+\x1b[0m Extracted: ${result.extracted}, Stored: ${result.stored}, Duplicates: ${result.duplicates}`);
+        }
+      }
+      break;
+    }
+    case 'connect': {
+      const target = args[1];
+      if (target !== 'openclaw') return console.log('Usage: memshell connect openclaw');
+
+      const { watchOpenClaw, defaultOpenClawPath, setConfigValue } = require('../src/ingest');
+      const store = new LocalStore(undefined, useEmbeddings ? { openaiKey: process.env.OPENAI_API_KEY } : {});
+      await store.init();
+
+      const sessionsPath = args[2] || defaultOpenClawPath();
+      setConfigValue('watch.openclaw', sessionsPath);
+      console.log(`  OpenClaw integration configured.`);
+      console.log(`  Sessions path: ${sessionsPath}`);
+      console.log('  Watching for new transcripts (Ctrl+C to stop)...\n');
+      watchOpenClaw(sessionsPath, store, { agent });
+      process.on('SIGINT', () => { console.log('\n  Stopped.'); process.exit(0); });
+      break;
+    }
+    case 'daemon': {
+      const { loadConfig, watchDirectory, watchOpenClaw } = require('../src/ingest');
+      const store = new LocalStore(undefined, useEmbeddings ? { openaiKey: process.env.OPENAI_API_KEY } : {});
+      await store.init();
+
+      const config = loadConfig();
+      const watchers = config.watch || {};
+      let activeWatchers = 0;
+
+      console.log('  \x1b[1mmem.sh daemon\x1b[0m starting...\n');
+
+      if (watchers.openclaw) {
+        watchOpenClaw(watchers.openclaw, store, { agent });
+        activeWatchers++;
+      }
+
+      // Support array of dir watchers
+      if (Array.isArray(watchers.dirs)) {
+        for (const dir of watchers.dirs) {
+          watchDirectory(typeof dir === 'string' ? dir : dir.path, store, { agent });
+          activeWatchers++;
+        }
+      } else if (watchers.dir) {
+        watchDirectory(watchers.dir, store, { agent });
+        activeWatchers++;
+      }
+
+      if (activeWatchers === 0) {
+        console.log('  No watchers configured. Use:');
+        console.log('    memshell config set watch.openclaw ~/.openclaw/agents/main/sessions/');
+        console.log('    memshell config set watch.dir /path/to/watch');
+        process.exit(1);
+      }
+
+      console.log(`\n  ${activeWatchers} watcher(s) active. Ctrl+C to stop.\n`);
+      process.on('SIGINT', () => { console.log('\n  Daemon stopped.'); process.exit(0); });
+      break;
+    }
+    case 'config': {
+      const { loadConfig, setConfigValue } = require('../src/ingest');
+      const subCmd = args[1];
+
+      if (subCmd === 'set') {
+        const configKey = args[2];
+        const configVal = args.slice(3).join(' ');
+        if (!configKey || !configVal) return console.log('Usage: memshell config set <key> <value>');
+        const result = setConfigValue(configKey, configVal);
+        console.log(`\x1b[32m+\x1b[0m Set ${configKey} = ${configVal}`);
+      } else if (subCmd === 'get') {
+        const config = loadConfig();
+        const configKey = args[2];
+        if (configKey) {
+          const parts = configKey.split('.');
+          let val = config;
+          for (const p of parts) val = val?.[p];
+          console.log(val !== undefined ? JSON.stringify(val, null, 2) : 'Not set');
+        } else {
+          console.log(JSON.stringify(config, null, 2));
+        }
+      } else {
+        const config = loadConfig();
+        console.log(JSON.stringify(config, null, 2));
+      }
       break;
     }
     case 'serve': case 'server': {
@@ -91,6 +302,7 @@ async function main() {
       const authKey = flag('key') || process.env.MEM_KEY || '';
       process.env.MEM_PORT = port;
       if (authKey) process.env.MEM_KEY = authKey;
+      if (useEmbeddings) process.env.MEM_USE_EMBEDDINGS = '1';
       require('../server');
       break;
     }
@@ -99,4 +311,4 @@ async function main() {
   }
 }
 
-main().catch(e => { console.error('Error:', e.message); process.exit(1); });
+main().catch(e => { console.error('\x1b[31mError:\x1b[0m', e.message); process.exit(1); });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memshell",
-  "version": "0.2.1",
+  "version": "0.4.0",
   "description": "Persistent memory for AI agents. Like localStorage but for LLMs.",
   "main": "src/index.js",
   "bin": {
@@ -25,7 +25,8 @@
   "author": "justedv",
   "license": "MIT",
   "dependencies": {
-    "express": "^4.18.2"
+    "express": "^4.18.2",
+    "sql.js": "^1.11.0"
   },
   "repository": {
     "type": "git",

package/server.js

@@ -6,12 +6,20 @@ const { LocalStore } = require('./src/index');
 const app = express();
 app.use(express.json());
 
-const store = new LocalStore();
+const storeOpts = {};
+if (process.env.MEM_USE_EMBEDDINGS && process.env.OPENAI_API_KEY) {
+  storeOpts.openaiKey = process.env.OPENAI_API_KEY;
+}
+const store = new LocalStore(undefined, storeOpts);
 const PORT = process.env.MEM_PORT || 3456;
 const AUTH_KEY = process.env.MEM_KEY || '';
 
+// Ensure store is initialized
+let initPromise = store.init();
+
 // Auth middleware
-app.use('/mem', (req, res, next) => {
+app.use('/mem', async (req, res, next) => {
+  await initPromise;
   if (AUTH_KEY && req.headers['x-mem-key'] !== AUTH_KEY) {
     return res.status(401).json({ error: 'Invalid API key' });
   }
@@ -19,44 +27,86 @@ app.use('/mem', (req, res, next) => {
   next();
 });
 
+// Ingest raw text
+app.post('/mem/ingest', async (req, res) => {
+  const { text, source } = req.body;
+  if (!text) return res.status(400).json({ error: 'text is required' });
+  try {
+    const { ingest } = require('./src/ingest');
+    const result = await ingest(text, store, { agent: req.agent, source: source || 'api' });
+    res.json(result);
+  } catch (e) {
+    res.status(500).json({ error: e.message });
+  }
+});
+
 // Store a memory
-app.post('/mem', (req, res) => {
-  const { text, metadata } = req.body;
+app.post('/mem', async (req, res) => {
+  const { text, tags, importance, metadata } = req.body;
   if (!text) return res.status(400).json({ error: 'text is required' });
-  const result = store.set(text, { agent: req.agent, metadata });
+  const result = await store.set(text, { agent: req.agent, tags: tags || '', importance, metadata });
   res.json(result);
 });
 
 // Semantic recall
-app.get('/mem/recall', (req, res) => {
+app.get('/mem/recall', async (req, res) => {
   const q = req.query.q;
   if (!q) return res.status(400).json({ error: 'q parameter is required' });
   const limit = parseInt(req.query.limit) || 10;
-  const results = store.recall(q, { agent: req.agent, limit });
+  const tags = req.query.tags || '';
+  const top = req.query.top ? parseInt(req.query.top) : null;
+  const results = await store.recall(q, { agent: req.agent, limit, tags, top });
   res.json(results);
 });
 
 // List all
-app.get('/mem/list', (req, res) => {
-  const results = store.list({ agent: req.agent });
+app.get('/mem/list', async (req, res) => {
+  const results = await store.list({ agent: req.agent });
   res.json(results);
 });
 
+// Stats
+app.get('/mem/stats', async (req, res) => {
+  const stats = await store.stats({ agent: req.agent });
+  res.json(stats);
+});
+
+// Export
+app.get('/mem/export', async (req, res) => {
+  const data = await store.exportAll({ agent: req.query.agent });
+  res.json(data);
+});
+
+// Import
+app.post('/mem/import', async (req, res) => {
+  const memories = Array.isArray(req.body) ? req.body : req.body.memories || [];
+  const result = await store.importAll(memories);
+  res.json(result);
+});
+
+// Boost importance
+app.post('/mem/:id/important', async (req, res) => {
+  const result = await store.important(req.params.id);
+  if (!result) return res.status(404).json({ error: 'Memory not found' });
+  res.json(result);
+});
+
 // Delete by id
-app.delete('/mem/:id', (req, res) => {
-  store.forget(req.params.id);
+app.delete('/mem/:id', async (req, res) => {
+  await store.forget(req.params.id);
   res.json({ ok: true, id: req.params.id });
 });
 
 // Clear all for agent
-app.delete('/mem', (req, res) => {
-  store.clear({ agent: req.agent });
+app.delete('/mem', async (req, res) => {
+  await store.clear({ agent: req.agent });
   res.json({ ok: true });
 });
 
 app.listen(PORT, () => {
   console.log(`\n  🧠 mem.sh server running on http://localhost:${PORT}`);
   console.log(`  Auth: ${AUTH_KEY ? 'enabled' : 'disabled'}`);
+  console.log(`  Embeddings: ${storeOpts.openaiKey ? 'OpenAI' : 'TF-IDF'}`);
   console.log();
 });