claude-mneme 2.10.3 → 3.0.0

package/client/mneme-client.mjs

@@ -0,0 +1,223 @@
+/**
+ * Mneme Client
+ *
+ * Thin HTTP client for hooks to communicate with the Mneme server.
+ * Handles auto-start of server if not running.
+ */
+
+import { request } from 'http';
+import { existsSync, readFileSync, unlinkSync } from 'fs';
+import { spawn } from 'child_process';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { homedir } from 'os';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const MEMORY_BASE = join(homedir(), '.claude-mneme');
+const PID_FILE = join(MEMORY_BASE, '.server.pid');
+const SERVER_SCRIPT = join(__dirname, '../server/mneme-server.mjs');
+
+/**
+ * Check if server is running and responsive
+ */
+async function pingServer(host, port) {
+  return new Promise((resolve) => {
+    const req = request({
+      host,
+      port,
+      path: '/health',
+      method: 'GET',
+      timeout: 1000
+    }, (res) => {
+      resolve(res.statusCode === 200);
+    });
+
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve(false);
+    });
+
+    req.end();
+  });
+}
+
+/**
+ * Ensure server is running, start if needed
+ * Returns { host, port }
+ */
+export async function ensureServer() {
+  // Check if server is already running
+  if (existsSync(PID_FILE)) {
+    try {
+      const { pid, host, port } = JSON.parse(readFileSync(PID_FILE, 'utf-8'));
+
+      // Verify process is alive
+      try {
+        process.kill(pid, 0); // Signal 0 = check existence
+      } catch {
+        // Process dead, clean up stale PID file
+        unlinkSync(PID_FILE);
+        return ensureServer(); // Retry
+      }
+
+      // Verify server is responsive
+      if (await pingServer(host, port)) {
+        return { host, port };
+      }
+
+      // Server not responsive, clean up and retry
+      unlinkSync(PID_FILE);
+      return ensureServer();
+    } catch (e) {
+      // Invalid PID file, clean up
+      unlinkSync(PID_FILE);
+      return ensureServer();
+    }
+  }
+
+  // Start server (detached process)
+  const child = spawn('node', [SERVER_SCRIPT], {
+    detached: true,
+    stdio: 'ignore'
+  });
+  child.unref();
+
+  // Wait for server to be ready (max 3 seconds)
+  for (let i = 0; i < 30; i++) {
+    await new Promise(r => setTimeout(r, 100));
+
+    if (existsSync(PID_FILE)) {
+      try {
+        const { host, port } = JSON.parse(readFileSync(PID_FILE, 'utf-8'));
+        if (await pingServer(host, port)) {
+          return { host, port };
+        }
+      } catch {
+        // Not ready yet, continue waiting
+      }
+    }
+  }
+
+  throw new Error('Server failed to start within 3 seconds');
+}
+
+/**
+ * HTTP client for the Mneme server
+ */
+export class MnemeClient {
+  constructor(host, port) {
+    this.host = host;
+    this.port = port;
+    this.timeout = 2000; // 2 second timeout
+  }
+
+  /**
+   * Make HTTP request to server
+   */
+  async request(method, path, body = null) {
+    return new Promise((resolve, reject) => {
+      const options = {
+        host: this.host,
+        port: this.port,
+        path,
+        method,
+        headers: body ? { 'Content-Type': 'application/json' } : {},
+        timeout: this.timeout
+      };
+
+      const req = request(options, (res) => {
+        let data = '';
+        res.on('data', chunk => data += chunk);
+        res.on('end', () => {
+          try {
+            const parsed = JSON.parse(data);
+            if (res.statusCode >= 400) {
+              reject(new Error(parsed.error || `HTTP ${res.statusCode}`));
+            } else {
+              resolve(parsed);
+            }
+          } catch (e) {
+            reject(new Error('Invalid JSON response'));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error('Request timeout'));
+      });
+
+      if (body) {
+        req.write(JSON.stringify(body));
+      }
+
+      req.end();
+    });
+  }
+
+  /**
+   * POST request helper
+   */
+  async post(path, body) {
+    return this.request('POST', path, body);
+  }
+
+  /**
+   * GET request helper
+   */
+  async get(path) {
+    return this.request('GET', path);
+  }
+
+  // Session Management
+
+  async registerSession(sessionId, cwd) {
+    return this.post('/session/register', { sessionId, cwd });
+  }
+
+  async unregisterSession(sessionId) {
+    return this.post('/session/unregister', { sessionId });
+  }
+
+  // Health Check
+
+  async health() {
+    return this.get('/health');
+  }
+
+  // Log Operations
+
+  async appendLog(project, entry) {
+    return this.post('/log/append', { project, entry });
+  }
+
+  async flushLog(project = null) {
+    return this.post('/log/flush', { project });
+  }
+
+  // Summarization Operations
+
+  async triggerSummarize(project, force = false) {
+    return this.post('/summarize/trigger', { project, force });
+  }
+
+  async getSummarizeStatus(project) {
+    return this.post('/summarize/status', { project });
+  }
+
+  async getSummary(project) {
+    return this.post('/summary/get', { project });
+  }
+}
+
+/**
+ * Get a connected client (auto-starts server if needed)
+ */
+export async function getClient() {
+  const { host, port } = await ensureServer();
+  return new MnemeClient(host, port);
+}

package/package.json

@@ -1,11 +1,14 @@
 {
   "name": "claude-mneme",
-  "version": "2.10.3",
+  "version": "3.0.0",
   "license": "MIT",
   "type": "module",
   "scripts": {
     "install-deps": "npm install",
-    "test": "node --test scripts/utils.test.mjs"
+    "test": "node --test scripts/utils.test.mjs",
+    "test:server": "node server/test-server.mjs",
+    "test:log": "node server/test-log-service.mjs",
+    "test:summarization": "node server/test-summarization.mjs"
   },
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.2.31"

package/scripts/utils.mjs

@@ -1842,26 +1842,27 @@ export function scoreEntriesByRelevance(entries, cwd, config) {
 }
 
 /**
- * Append a log entry using the buffered write system.
- * Writes to .pending.jsonl for batching, then flushes if throttle allows.
+ * Append a log entry via the Mneme server (batched, deduplicated).
  * Also extracts and indexes entities from the entry.
  */
 export function appendLogEntry(entry, cwd = process.cwd()) {
-  const paths = ensureMemoryDirs(cwd);
-  const pendingPath = paths.log.replace('.jsonl', '.pending.jsonl');
   const config = loadConfig();
+  const project = getProjectRoot(cwd);
+
+  // Send to server (batched, deduplicated)
+  import('../client/mneme-client.mjs')
+    .then(({ getClient }) => getClient())
+    .then(client => client.appendLog(project, entry))
+    .catch(err => {
+      // Server unavailable, fail silently (non-critical)
+      logError(err, 'appendLogEntry');
+    });
 
-  // Always write to pending file (fast, append-only)
-  appendFileSync(pendingPath, JSON.stringify(entry) + '\n');
-
-  // Extract and index entities from the entry
+  // Extract and index entities from the entry (TODO: move to server in Phase 4)
   updateEntityIndex(entry, cwd, config);
 
-  // Invalidate cache since data has changed
+  // Invalidate cache since data has changed (TODO: move to server in Phase 3)
   invalidateCache(cwd);
-
-  // Throttled flush - only flush every 5 seconds
-  flushPendingLog(cwd, 5000);
 }
 
 /**
@@ -1938,67 +1939,20 @@ export function flushPendingLog(cwd = process.cwd(), throttleMs = 0) {
 }
 
 /**
- * Check if summarization is needed and spawn it in background if so
+ * Trigger summarization via server (throttled, queued)
  * Call this after appending to the log
  */
 export function maybeSummarize(cwd = process.cwd()) {
-  const paths = ensureMemoryDirs(cwd);
-  const config = loadConfig();
-
-  // Quick check: does log exist and have enough entries?
-  if (!existsSync(paths.log)) {
-    return;
-  }
-
-  try {
-    const logContent = readFileSync(paths.log, 'utf-8').trim();
-    if (!logContent) return;
-
-    const entryCount = logContent.split('\n').filter(l => l).length;
-
-    if (entryCount < config.maxLogEntriesBeforeSummarize) {
-      return;
-    }
-
-    // Acquire lock atomically using O_EXCL (fails if file already exists)
-    const lockFile = paths.log + '.lock';
-    try {
-      // If lock exists, check if it's stale
-      if (existsSync(lockFile)) {
-        const lockContent = readFileSync(lockFile, 'utf-8').trim();
-        const lockTime = parseInt(lockContent, 10);
-        if (lockTime && Date.now() - lockTime < 5 * 60 * 1000) {
-          return; // Lock is fresh, summarization already running
-        }
-        // Stale lock — remove it so we can try to acquire
-        try { unlinkSync(lockFile); } catch {}
-      }
-
-      // Atomic create: O_CREAT | O_EXCL | O_WRONLY fails if another process created it first
-      const fd = openSync(lockFile, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY);
-      const timestamp = Buffer.from(Date.now().toString());
-      writeSync(fd, timestamp);
-      closeSync(fd);
-    } catch {
-      // Another process won the race — let it handle summarization
-      return;
-    }
-
-    // Spawn summarize.mjs in background
-    const __filename = fileURLToPath(import.meta.url);
-    const __dirname = dirname(__filename);
-    const summarizeScript = join(__dirname, 'summarize.mjs');
-
-    const child = spawn('node', [summarizeScript, cwd], {
-      detached: true,
-      stdio: 'ignore',
-      cwd: cwd
+  const project = getProjectRoot(cwd);
+
+  // Trigger via server (server handles throttling, entry count check, etc.)
+  import('../client/mneme-client.mjs')
+    .then(({ getClient }) => getClient())
+    .then(client => client.triggerSummarize(project, false))
+    .catch(err => {
+      // Server unavailable or throttled, fail silently (non-critical)
+      logError(err, 'maybeSummarize');
     });
-
-    child.unref();
-  } catch (e) {
-    logError(e, 'maybeSummarize');
-  }
 }
 
 // ============================================================================

package/server/batch-queue.mjs

@@ -0,0 +1,81 @@
+/**
+ * BatchQueue
+ *
+ * Collects items and flushes them in batches based on:
+ * - Size threshold (max items)
+ * - Time threshold (max wait time)
+ */
+
+export class BatchQueue {
+  constructor({ maxSize = 100, maxWaitMs = 1000, processor }) {
+    this.maxSize = maxSize;
+    this.maxWaitMs = maxWaitMs;
+    this.processor = processor; // async function(batch)
+    this.batch = [];
+    this.timer = null;
+    this.flushing = false;
+  }
+
+  /**
+   * Add item to queue
+   */
+  add(item) {
+    this.batch.push(item);
+
+    // Flush immediately if batch is full
+    if (this.batch.length >= this.maxSize) {
+      this.flush();
+      return;
+    }
+
+    // Schedule flush if not already scheduled
+    if (!this.timer) {
+      this.timer = setTimeout(() => this.flush(), this.maxWaitMs);
+    }
+  }
+
+  /**
+   * Flush current batch
+   */
+  async flush() {
+    if (this.flushing || this.batch.length === 0) return;
+
+    // Clear timer
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+
+    // Take current batch
+    const items = this.batch;
+    this.batch = [];
+    this.flushing = true;
+
+    try {
+      await this.processor(items);
+    } catch (err) {
+      // Log error but don't throw (non-critical, best-effort)
+      console.error('[batch-queue] Flush error:', err.message);
+    } finally {
+      this.flushing = false;
+    }
+  }
+
+  /**
+   * Get current queue depth
+   */
+  depth() {
+    return this.batch.length;
+  }
+
+  /**
+   * Shutdown queue (flush and clear timer)
+   */
+  async shutdown() {
+    await this.flush();
+    if (this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+  }
+}

package/server/deduplicator.mjs

@@ -0,0 +1,80 @@
+/**
+ * Deduplicator
+ *
+ * Detects duplicate entries within a sliding time window.
+ * Uses content-based hashing (ignores timestamp).
+ */
+
+export class Deduplicator {
+  constructor({ windowMs = 5000 } = {}) {
+    this.windowMs = windowMs;
+    this.recentHashes = new Map(); // hash -> timestamp
+    this.cleanupInterval = setInterval(() => this.cleanup(), 60000); // Clean every minute
+  }
+
+  /**
+   * Check if entry is a duplicate
+   * Returns true if duplicate, false otherwise
+   */
+  isDuplicate(entry) {
+    const hash = this.hashEntry(entry);
+    const lastSeen = this.recentHashes.get(hash);
+    const now = Date.now();
+
+    if (lastSeen && now - lastSeen < this.windowMs) {
+      // Duplicate within window, update timestamp
+      this.recentHashes.set(hash, now);
+      return true;
+    }
+
+    // Not a duplicate, record it
+    this.recentHashes.set(hash, now);
+    return false;
+  }
+
+  /**
+   * Hash entry based on type and content (ignore timestamp)
+   */
+  hashEntry(entry) {
+    const { type, content, action, outcome, subject } = entry;
+
+    // For task entries, include action/outcome/subject
+    if (type === 'task') {
+      return `task:${action || ''}:${outcome || ''}:${subject || ''}`;
+    }
+
+    // For other entries, use type + content
+    return `${type}:${content || ''}`;
+  }
+
+  /**
+   * Clean up old entries (older than window + 1 minute buffer)
+   */
+  cleanup() {
+    const now = Date.now();
+    const expiry = this.windowMs + 60000; // Window + 1 minute buffer
+
+    for (const [hash, timestamp] of this.recentHashes) {
+      if (now - timestamp > expiry) {
+        this.recentHashes.delete(hash);
+      }
+    }
+  }
+
+  /**
+   * Get current cache size
+   */
+  size() {
+    return this.recentHashes.size;
+  }
+
+  /**
+   * Shutdown deduplicator
+   */
+  shutdown() {
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = null;
+    }
+  }
+}

package/server/log-service.mjs

@@ -0,0 +1,153 @@
+/**
+ * LogService
+ *
+ * Handles log entry batching, deduplication, and file writes.
+ * Centralizes all file I/O to eliminate lock contention.
+ */
+
+import { existsSync, appendFileSync, mkdirSync } from 'fs';
+import { join } from 'path';
+import { createHash } from 'crypto';
+import { homedir } from 'os';
+import { BatchQueue } from './batch-queue.mjs';
+import { Deduplicator } from './deduplicator.mjs';
+
+export class LogService {
+  constructor(config, logger) {
+    this.config = config;
+    this.logger = logger;
+    this.deduplicator = new Deduplicator({ windowMs: 5000 });
+    this.stats = {
+      entriesReceived: 0,
+      entriesDeduplicated: 0,
+      entriesWritten: 0,
+      batchesFlushed: 0,
+      writeErrors: 0
+    };
+
+    // Create batch queue with processor
+    this.queue = new BatchQueue({
+      maxSize: config.batching?.log?.maxSize || 100,
+      maxWaitMs: config.batching?.log?.maxWaitMs || 1000,
+      processor: (batch) => this.processBatch(batch)
+    });
+  }
+
+  /**
+   * Append log entry (queues for batched write)
+   */
+  append(project, entry) {
+    this.stats.entriesReceived++;
+
+    // Deduplicate
+    if (this.deduplicator.isDuplicate(entry)) {
+      this.stats.entriesDeduplicated++;
+      return { ok: true, deduplicated: true, queued: false };
+    }
+
+    // Queue for batched write
+    this.queue.add({ project, entry });
+
+    return { ok: true, deduplicated: false, queued: true };
+  }
+
+  /**
+   * Force immediate flush
+   */
+  async flush(project = null) {
+    await this.queue.flush();
+    return { ok: true, entriesFlushed: this.stats.entriesWritten };
+  }
+
+  /**
+   * Process a batch of entries (group by project and write)
+   */
+  async processBatch(items) {
+    // Group by project
+    const byProject = new Map();
+    for (const { project, entry } of items) {
+      if (!byProject.has(project)) {
+        byProject.set(project, []);
+      }
+      byProject.get(project).push(entry);
+    }
+
+    // Write to each project's log file
+    for (const [project, entries] of byProject) {
+      try {
+        this.writeToLog(project, entries);
+        this.stats.entriesWritten += entries.length;
+      } catch (err) {
+        this.stats.writeErrors++;
+        this.logger.error('log-write-failed', {
+          project,
+          entryCount: entries.length,
+          error: err.message
+        });
+      }
+    }
+
+    this.stats.batchesFlushed++;
+
+    this.logger.debug('log-batch-flushed', {
+      projects: byProject.size,
+      totalEntries: items.length,
+      batchNum: this.stats.batchesFlushed
+    });
+  }
+
+  /**
+   * Write entries to project log file
+   */
+  writeToLog(project, entries) {
+    const projectDir = this.getProjectMemoryDir(project);
+    const logFile = join(projectDir, 'log.jsonl');
+
+    // Ensure directory exists
+    if (!existsSync(projectDir)) {
+      mkdirSync(projectDir, { recursive: true });
+    }
+
+    // Prepare log lines
+    const lines = entries.map(e => JSON.stringify(e)).join('\n') + '\n';
+
+    // Single atomic write (no locking needed — we're the only writer)
+    appendFileSync(logFile, lines);
+  }
+
+  /**
+   * Get project memory directory path
+   */
+  getProjectMemoryDir(project) {
+    // Hash project path for directory name (same as current implementation)
+    const hash = createHash('sha256').update(project).digest('hex').slice(0, 16);
+    const memoryBase = homedir() + '/.claude-mneme';
+    return join(memoryBase, hash);
+  }
+
+  /**
+   * Get queue depth
+   */
+  queueDepth() {
+    return this.queue.depth();
+  }
+
+  /**
+   * Get stats
+   */
+  getStats() {
+    return {
+      ...this.stats,
+      queueDepth: this.queue.depth(),
+      deduplicatorSize: this.deduplicator.size()
+    };
+  }
+
+  /**
+   * Shutdown service
+   */
+  async shutdown() {
+    await this.queue.shutdown();
+    this.deduplicator.shutdown();
+  }
+}