baby-daemon 1.0.0

package/src/vectorStore.js

@@ -0,0 +1,410 @@
+import path from 'path';
+import fs from 'fs';
+import { fileURLToPath } from 'url';
+import { ai } from './config.js';
+
+// Resolve __dirname equivalent in ES Modules
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DB_PATH = path.join(PROJECT_ROOT, '.lancedb');
+const CACHE_FILE = path.join(PROJECT_ROOT, '.embeddings_cache.json');
+const MEMORY_FILE = path.join(PROJECT_ROOT, 'memory.jsonl');
+
+// Dynamically import LanceDB to handle native compilation failures gracefully
+let lancedb = null;
+let isLanceDbAvailable = false;
+
+try {
+  // Use dynamic import so it parses at runtime inside the try block
+  lancedb = await import('@lancedb/lancedb');
+  isLanceDbAvailable = true;
+} catch (error) {
+  console.warn(
+    '\n  ⚠️  Warning: Failed to load `@lancedb/lancedb` native bindings.\n' +
+    '  Baby Daemon will automatically run in fallback mode using pure JS MiniSearch.\n' +
+    `  Error details: ${error.message}\n`
+  );
+}
+
+// ─────────────────────────────────────────────────────────
+// EMBEDDING CACHE READ/WRITE
+// ─────────────────────────────────────────────────────────
+
+function loadCache() {
+  try {
+    if (fs.existsSync(CACHE_FILE)) {
+      return JSON.parse(fs.readFileSync(CACHE_FILE, 'utf-8'));
+    }
+  } catch (error) {
+    console.error('  ⚠️  Failed to read embedding cache:', error.message);
+  }
+  return {};
+}
+
+function saveCache(cache) {
+  try {
+    fs.writeFileSync(CACHE_FILE, JSON.stringify(cache, null, 2), 'utf-8');
+  } catch (error) {
+    console.error('  ⚠️  Failed to save embedding cache:', error.message);
+  }
+}
+
+// ─────────────────────────────────────────────────────────
+// GET BATCH EMBEDDINGS (WITH LOCAL CACHING)
+// ─────────────────────────────────────────────────────────
+
+/**
+ * getEmbeddingsForMemories(memories)
+ *
+ * @param {Array} memories - List of raw memory objects
+ * @returns {Promise<Array>} - List of memories enriched with their embedding vectors
+ */
+export async function getEmbeddingsForMemories(memories) {
+  if (memories.length === 0) return [];
+
+  const cache = loadCache();
+  const missingHashes = [];
+  const textsToEmbed = [];
+
+  // Determine which memories need embeddings generated
+  for (const mem of memories) {
+    if (!cache[mem.hash]) {
+      missingHashes.push(mem.hash);
+      // Construct a rich string representation for embedding context
+      // Including the type helps match queries like "what bug" or "what decision"
+      const richText = `[${mem.type.toUpperCase()}] ${mem.content} (Files: ${mem.related_files.join(', ')})`;
+      textsToEmbed.push(richText);
+    }
+  }
+
+  // Fetch embeddings from Gemini API for any cache misses
+  if (textsToEmbed.length > 0) {
+    try {
+      const response = await ai.models.embedContent({
+        model: 'gemini-embedding-2',
+        contents: textsToEmbed,
+      });
+
+      if (response && response.embeddings) {
+        for (let i = 0; i < response.embeddings.length; i++) {
+          const vector = response.embeddings[i].values;
+          const hash = missingHashes[i];
+          cache[hash] = vector;
+        }
+        saveCache(cache);
+      } else {
+        throw new Error('Invalid response structure from Gemini Embedding API');
+      }
+    } catch (error) {
+      console.error('  ✗ Error calling Gemini Embedding API:', error.message);
+      throw error;
+    }
+  }
+
+  // Map all memories to their vectorized version
+  return memories.map((mem) => ({
+    id: mem.id,
+    vector: cache[mem.hash],
+    type: mem.type,
+    content: mem.content,
+    original_text: mem.original_text,
+    confidence: mem.confidence,
+    status: mem.status || 'active',
+    related_files: mem.related_files || [],
+    tags: mem.tags || [],
+    chat_file: mem.source?.chat_file || 'unknown',
+    timestamp: mem.timestamp || new Date().toISOString(),
+    hash: mem.hash,
+  }));
+}
+
+// ─────────────────────────────────────────────────────────
+// SYNCHRONIZE MEMORIES WITH LANCEDB
+// ─────────────────────────────────────────────────────────
+
+/**
+ * syncMemoriesToVectorStore(fileName, memories)
+ * Updates LanceDB by replacing all memories associated with fileName.
+ *
+ * @param {string} fileName - Source file name
+ * @param {Array} memories - List of raw memory objects
+ */
+export async function syncMemoriesToVectorStore(fileName, memories) {
+  if (!isLanceDbAvailable) {
+    return; // Fallback mode silently bypasses LanceDB
+  }
+
+  try {
+    const db = await lancedb.connect(DB_PATH);
+    const tableNames = await db.tableNames();
+    let table;
+
+    // Get vectorized memories (using cache / calling API)
+    const vectorizedMemories = await getEmbeddingsForMemories(memories);
+
+    if (tableNames.includes('memories')) {
+      table = await db.openTable('memories');
+      
+      // Delete old records for this file (in-place soft delete)
+      // SQL-like syntax required by LanceDB
+      await table.delete(`chat_file = '${fileName}'`);
+
+      // Add new ones
+      if (vectorizedMemories.length > 0) {
+        await table.add(vectorizedMemories);
+      }
+    } else {
+      // Create table if it doesn't exist
+      if (vectorizedMemories.length > 0) {
+        table = await db.createTable('memories', vectorizedMemories);
+      }
+    }
+  } catch (error) {
+    console.error(`  ⚠️ LanceDB sync failed for ${fileName}:`, error.message);
+  }
+}
+
+// ─────────────────────────────────────────────────────────
+// SEARCH MEMORIES (SEMANTIC + KEYWORD FALLBACK)
+// ─────────────────────────────────────────────────────────
+
+/**
+ * searchMemories(queryText, filters)
+ * Performs hybrid semantic search with date and field filters, falling back to MiniSearch if needed.
+ *
+ * @param {string} queryText - Query string
+ * @param {Object} filters - Filter arguments (since, type, file, limit)
+ */
+export async function searchMemories(queryText, filters = {}) {
+  const { since, type, file, limit = 10 } = filters;
+
+  // ── LAYER 1: LanceDB Vector Search (Semantic) ─────────────────────
+  if (isLanceDbAvailable) {
+    try {
+      const db = await lancedb.connect(DB_PATH);
+      const tableNames = await db.tableNames();
+
+      if (tableNames.includes('memories')) {
+        const table = await db.openTable('memories');
+
+        // Generate embedding for the query
+        const queryEmbeddingRes = await ai.models.embedContent({
+          model: 'gemini-embedding-2',
+          contents: queryText,
+        });
+        const queryVector = queryEmbeddingRes.embeddings[0].values;
+
+        // Perform vector search using cosine distance
+        const results = await table
+          .vectorSearch(queryVector)
+          .distanceType('cosine')
+          .limit(50) // Retrieve more to allow client-side filtering
+          .toArray();
+
+        // Map and score: cosine similarity is (1 - cosine distance)
+        const scoredResults = results.map(item => {
+          const toJSArray = (val) => {
+            if (!val) return [];
+            if (Array.isArray(val)) return val;
+            if (typeof val.toArray === 'function') return val.toArray();
+            return Array.from(val);
+          };
+          return {
+            ...item,
+            related_files: toJSArray(item.related_files),
+            tags: toJSArray(item.tags),
+            score: 1 - (item._distance ?? 1),
+          };
+        });
+
+        // Filter by threshold (score >= 0.70)
+        let filtered = scoredResults.filter(item => item.score >= 0.70);
+
+        // Apply metadata filters
+        if (filtered.length > 0) {
+          filtered = applyFilters(filtered, { since, type, file });
+          if (filtered.length > 0) {
+            return {
+              method: 'semantic',
+              results: filtered.slice(0, limit),
+            };
+          }
+        }
+      }
+    } catch (error) {
+      console.warn('  ⚠️ Vector search failed, falling back to full-text:', error.message);
+    }
+  }
+
+  // ── LAYER 2: MiniSearch Keyword Fallback ─────────────────────────
+  console.log('  [Search] Falling back to keyword-based search...');
+  return {
+    method: 'keyword',
+    results: await searchMiniSearch(queryText, { since, type, file, limit }),
+  };
+}
+
+// ─────────────────────────────────────────────────────────
+// METADATA FILTERING HELPER
+// ─────────────────────────────────────────────────────────
+
+function applyFilters(results, { since, type, file }) {
+  let filtered = [...results];
+
+  // Filter by date (since)
+  if (since) {
+    const cutoffDate = new Date(since);
+    if (!isNaN(cutoffDate.getTime())) {
+      filtered = filtered.filter(item => new Date(item.timestamp) >= cutoffDate);
+    }
+  }
+
+  // Filter by type
+  if (type) {
+    const targetType = type.trim().toLowerCase();
+    filtered = filtered.filter(item => item.type.toLowerCase() === targetType);
+  }
+
+  // Filter by file
+  if (file) {
+    const targetFile = file.trim().toLowerCase();
+    filtered = filtered.filter(item => {
+      const matchSource = item.chat_file.toLowerCase().includes(targetFile);
+      const matchRelated = item.related_files.some(f => f.toLowerCase().includes(targetFile));
+      return matchSource || matchRelated;
+    });
+  }
+
+  return filtered;
+}
+
+// ─────────────────────────────────────────────────────────
+// MINISEARCH FALLBACK RUNNER
+// ─────────────────────────────────────────────────────────
+
+async function searchMiniSearch(queryText, { since, type, file, limit }) {
+  if (!fs.existsSync(MEMORY_FILE)) {
+    return [];
+  }
+
+  try {
+    const MiniSearch = (await import('minisearch')).default;
+
+    // Load and parse flat memory file
+    const content = fs.readFileSync(MEMORY_FILE, 'utf-8');
+    const documents = content
+      .split('\n')
+      .filter(line => line.trim())
+      .map(line => {
+        try {
+          const parsed = JSON.parse(line);
+          return {
+            id: parsed.id,
+            type: parsed.type,
+            content: parsed.content,
+            original_text: parsed.original_text,
+            confidence: parsed.confidence,
+            status: parsed.status,
+            related_files: parsed.related_files,
+            tags: parsed.tags,
+            chat_file: parsed.source?.chat_file || 'unknown',
+            timestamp: parsed.timestamp,
+          };
+        } catch {
+          return null;
+        }
+      })
+      .filter(Boolean);
+
+    // Initialize MiniSearch engine
+    const miniSearch = new MiniSearch({
+      fields: ['content', 'original_text', 'tags', 'related_files', 'type'],
+      storeFields: [
+        'id',
+        'type',
+        'content',
+        'original_text',
+        'confidence',
+        'status',
+        'related_files',
+        'tags',
+        'chat_file',
+        'timestamp',
+      ],
+      searchOptions: {
+        fuzzy: 0.2,
+        prefix: true,
+      },
+    });
+
+    miniSearch.addAll(documents);
+
+    // Search and score
+    const results = miniSearch.search(queryText);
+
+    // Apply metadata filters
+    return applyFilters(results, { since, type, file }).slice(0, limit);
+  } catch (error) {
+    console.error('  ✗ Keyword search fallback failed:', error.message);
+    return [];
+  }
+}
+
+// ─────────────────────────────────────────────────────────
+// ARCHIVE MEMORIES
+// ─────────────────────────────────────────────────────────
+
+/**
+ * archiveMemories(ageDays)
+ * Moves memories older than ageDays from 'memories' to 'archived_memories' table.
+ *
+ * @param {Object} options - Age options
+ * @returns {Promise<Object>} - Archival statistics
+ */
+export async function archiveMemories({ ageDays = 30 } = {}) {
+  if (!isLanceDbAvailable) {
+    throw new Error('LanceDB is not available on this platform.');
+  }
+
+  try {
+    const db = await lancedb.connect(DB_PATH);
+    const tableNames = await db.tableNames();
+
+    if (!tableNames.includes('memories')) {
+      return { archivedCount: 0, msg: 'No memories table exists yet.' };
+    }
+
+    const table = await db.openTable('memories');
+    const allMemories = await table.query().toArray();
+
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - ageDays);
+
+    const toArchive = allMemories.filter(m => new Date(m.timestamp) < cutoff);
+
+    if (toArchive.length === 0) {
+      return { archivedCount: 0, msg: `No memories older than ${ageDays} days found.` };
+    }
+
+    let archiveTable;
+    if (tableNames.includes('archived_memories')) {
+      archiveTable = await db.openTable('archived_memories');
+      await archiveTable.add(toArchive);
+    } else {
+      archiveTable = await db.createTable('archived_memories', toArchive);
+    }
+
+    // Delete from main memories table
+    const ids = toArchive.map(m => `'${m.id}'`).join(',');
+    await table.delete(`id IN (${ids})`);
+
+    return {
+      archivedCount: toArchive.length,
+      msg: `Archived ${toArchive.length} memories (older than ${ageDays} days) successfully.`,
+    };
+  } catch (error) {
+    console.error('  ✗ Archival failed:', error.message);
+    throw error;
+  }
+}
+

package/src/watcher.js

@@ -0,0 +1,263 @@
+/**
+ * watcher.js
+ * ──────────
+ * WHAT THIS FILE DOES:
+ *   Sets up chokidar to watch a folder and calls our pipeline when a
+ *   file is added or changed. For Phase 1, the "pipeline" is just:
+ *     → check idempotency → print filename → mark as processed
+ *
+ * CONCEPT: chokidar (watch library)
+ *   The OS can tell programs "hey, a file changed!" via events.
+ *   Node's built-in fs.watch does this, but it's unreliable:
+ *     - Fires twice on some OSes
+ *     - Doesn't work well across network drives or Docker
+ *     - Misses some rename/delete events
+ *   chokidar wraps fs.watch AND fs.watchFile and normalizes all of that.
+ *   It gives you a clean, reliable event system.
+ *   Think of it like: "fs.watch, but fixed."
+ *
+ * CONCEPT: Event-driven programming
+ *   Instead of your program constantly asking "did anything change?"
+ *   (called polling), the OS notifies your program when something happens.
+ *   chokidar exposes this as events: 'add', 'change', 'unlink' (delete), etc.
+ *   You "listen" for events with .on('eventName', callback).
+ *   This is the same pattern as addEventListener in the browser.
+ */
+
+import chokidar from 'chokidar';
+import fs from 'fs';
+import path from 'path';
+import readline from 'readline/promises';
+import { isAlreadyProcessed, markAsProcessed } from './idempotency.js';
+import { summarizeChatLog } from './summarizer.js';
+import { saveMemoriesForFile } from './memoryStore.js';
+import { syncMemoriesToVectorStore } from './vectorStore.js';
+
+// ─────────────────────────────────────────────────────────
+// MAIN EXPORT: startWatcher(watchPath)
+// ─────────────────────────────────────────────────────────
+
+/**
+ * startWatcher(watchPath)
+ * Starts the file watcher on the given folder path.
+ *
+ * @param {string} watchPath - The folder to watch (passed from CLI)
+ */
+export function startWatcher(watchPath, options = {}) {
+  const requireApproval = options.requireApproval ?? false;
+
+  // Verify the folder actually exists before we start
+  if (!fs.existsSync(watchPath)) {
+    console.error(`\n  ✗ Folder not found: ${watchPath}`);
+    console.error(`  Create the folder first, then run memory-watch again.\n`);
+    process.exit(1); // Exit with error code 1 (non-zero = something went wrong)
+  }
+
+  const resolvedPath = path.resolve(watchPath);
+  // path.resolve converts relative paths like "./logs" to absolute ones like "C:/proj101/logs"
+  // Always work with absolute paths to avoid confusion
+
+  console.log(`\n  🧠 Baby Daemon — Phase 1 (File Watcher)\n`);
+  console.log(`  Watching : ${resolvedPath}`);
+  console.log(`  Tracking : processed_keys.json\n`);
+  console.log(`  ─────────────────────────────────────────`);
+
+  // ─────────────────────────────────────────────────────────
+  // CHOKIDAR SETUP
+  // ─────────────────────────────────────────────────────────
+
+  /**
+   * CONCEPT: chokidar.watch(path, options)
+   *
+   * Options explained:
+   *
+   * persistent: true
+   *   Keep the process alive even if there's nothing else running.
+   *   Without this, Node might exit immediately after setup.
+   *
+   * ignoreInitial: true
+   *   When you first start watching a folder, chokidar fires 'add' for
+   *   EVERY existing file. We don't want that — we only care about NEW changes.
+   *   Setting this to true suppresses those initial 'add' events.
+   *
+   * awaitWriteFinish
+   *   This is crucial. When a program saves a large file, it doesn't
+   *   write everything at once — the OS writes in chunks.
+   *   If we react immediately, we might read an incomplete file.
+   *   awaitWriteFinish tells chokidar: "Wait until the file size stops
+   *   changing for 500ms before firing the event."
+   *   stabilityThreshold: 500ms of no changes = "write is done"
+   *   pollInterval: check every 100ms during that wait period
+   *
+   * usePolling: false
+   *   Polling = constantly checking "did this file change?" every N ms.
+   *   Event-based = OS tells us immediately. Event-based is better (less CPU).
+   *   usePolling: false means "use OS events, not polling".
+   *   (Set to true if watching network drives or Docker volumes)
+   */
+  const watcher = chokidar.watch(resolvedPath, {
+    persistent: true,
+    ignoreInitial: true,
+    awaitWriteFinish: {
+      stabilityThreshold: 500,
+      pollInterval: 100,
+    },
+    usePolling: false,
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // EVENT LISTENERS
+  // ─────────────────────────────────────────────────────────
+
+  /**
+   * CONCEPT: .on('event', callback)
+   *   chokidar is an EventEmitter — a Node.js pattern where an object
+   *   can emit named events, and you register functions to handle them.
+   *   watcher.on('add', fn)    → fn runs when a NEW file appears
+   *   watcher.on('change', fn) → fn runs when an existing file changes
+   *
+   *   The callback receives:
+   *     filePath: absolute path of the changed file
+   *     stats:    file statistics object (size, modified time, etc.)
+   *               Only available because we'll use it for idempotency
+   */
+
+  // Handle new files added to the folder
+  watcher.on('add', (filePath, stats) => {
+    handleFileEvent('NEW FILE', filePath, stats, requireApproval);
+  });
+
+  // Handle existing files that get modified
+  watcher.on('change', (filePath, stats) => {
+    handleFileEvent('MODIFIED', filePath, stats, requireApproval);
+  });
+
+  // If something goes wrong with the watcher itself
+  watcher.on('error', (error) => {
+    console.error(`\n  ✗ Watcher error: ${error.message}\n`);
+  });
+
+  // Fires once chokidar has finished its initial scan and is ready
+  watcher.on('ready', () => {
+    console.log(`  ✓ Watcher is live. Waiting for file changes...\n`);
+  });
+
+  // ─────────────────────────────────────────────────────────
+  // GRACEFUL SHUTDOWN
+  // ─────────────────────────────────────────────────────────
+
+  /**
+   * CONCEPT: process signals (SIGINT, SIGTERM)
+   *   When you press Ctrl+C in the terminal, the OS sends a signal called
+   *   SIGINT (Signal Interrupt) to your process.
+   *   By default, Node exits immediately. But we want to close the watcher
+   *   cleanly first (release file handles, etc.).
+   *   process.on('SIGINT', fn) lets us intercept that signal and run cleanup.
+   */
+  process.on('SIGINT', async () => {
+    console.log('\n\n  Shutting down watcher...');
+    await watcher.close(); // Tell chokidar to stop watching
+    console.log('  ✓ Watcher stopped. Goodbye.\n');
+    process.exit(0); // Exit with code 0 = clean exit
+  });
+}
+
+// ─────────────────────────────────────────────────────────
+// HANDLE A SINGLE FILE EVENT
+// ─────────────────────────────────────────────────────────
+
+/**
+ * handleFileEvent(eventType, filePath, stats)
+ *
+ * This is our Phase 1 "pipeline" — just idempotency check + print.
+ * In Phase 2, this is where we'll add the LLM summarization call.
+ *
+ * @param {string} eventType - 'NEW FILE' or 'MODIFIED'
+ * @param {string} filePath  - Absolute path to the changed file
+ * @param {object} stats     - fs.Stats object from chokidar
+ *
+ * CONCEPT: fs.Stats object
+ *   When chokidar detects a change, it can pass you an fs.Stats object.
+ *   This contains metadata about the file:
+ *     stats.size    → file size in bytes
+ *     stats.mtimeMs → last modified time in milliseconds since Unix epoch
+ *                     (Jan 1, 1970 00:00:00 UTC)
+ *   We use mtimeMs as part of our idempotency key.
+ *
+ * CONCEPT: Optional chaining (stats?.mtimeMs)
+ *   chokidar might not always provide stats (e.g., on some OS events).
+ *   stats?.mtimeMs means: "if stats exists, get mtimeMs; if not, return undefined"
+ *   Without ?. we'd crash if stats is undefined.
+ *   Fallback: Date.now() gives us current time in ms as a substitute.
+ */
+async function handleFileEvent(eventType, filePath, stats, requireApproval) {
+  const mtimeMs = stats?.mtimeMs ?? Date.now();
+
+  const relativePath = path.basename(filePath);
+
+  // ── IDEMPOTENCY CHECK ──────────────────────────────────
+  if (isAlreadyProcessed(filePath, mtimeMs)) {
+    console.log(`  [SKIP]  ${relativePath}  (already processed)`);
+    return;
+  }
+
+  // ── NEW / CHANGED FILE — PROCESS IT ───────────────────
+  const timestamp = new Date().toLocaleTimeString();
+  console.log(`\n  [${eventType}]  ${relativePath}`);
+  console.log(`  Time   : ${timestamp}`);
+  console.log(`  Path   : ${filePath}`);
+  console.log(`  Size   : ${stats?.size ?? 'unknown'} bytes`);
+  console.log(`  ─────────────────────────────────────────`);
+
+  try {
+    console.log(`  [LLM]  Extracting memories via Gemini...`);
+    const content = await fs.promises.readFile(filePath, 'utf-8');
+    const memories = await summarizeChatLog(content, relativePath);
+
+    let approvedMemories = [];
+
+    // Interactive approval mode logic
+    if (requireApproval && memories.length > 0) {
+      console.log(`\n  ⚠️  Approval Mode: Please review the following ${memories.length} candidate memories extracted from ${relativePath}:`);
+      
+      const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+
+      for (let i = 0; i < memories.length; i++) {
+        const mem = memories[i];
+        console.log(`\n  ┌── [Candidate #${i + 1}/${memories.length}] ─────────────────────────────`);
+        console.log(`  │  Type       : ${mem.type.toUpperCase()}`);
+        console.log(`  │  Confidence : ${mem.confidence.toFixed(2)}`);
+        console.log(`  │  Content    : "${mem.content}"`);
+        console.log(`  │  Related    : ${mem.related_files.join(', ') || 'none'}`);
+        console.log(`  │  Evidence   : "${mem.original_text.replace(/\r?\n/g, ' ')}"`);
+        console.log(`  └─────────────────────────────────────────────────────────`);
+
+        const answer = await rl.question('  Approve this memory? (Y/n): ');
+        if (answer.trim().toLowerCase() !== 'n') {
+          approvedMemories.push(mem);
+          console.log('  ✅ Approved.');
+        } else {
+          console.log('  ❌ Rejected.');
+        }
+      }
+      rl.close();
+      console.log('');
+    } else {
+      approvedMemories = memories;
+    }
+
+    console.log(`  [DB]   Saving ${approvedMemories.length} memories to memory.jsonl...`);
+    const newCount = saveMemoriesForFile(relativePath, approvedMemories);
+
+    console.log(`  [DB]   Syncing ${approvedMemories.length} memories to LanceDB...`);
+    await syncMemoriesToVectorStore(relativePath, approvedMemories);
+
+    // Only record idempotency fingerprint if we successfully processed the file
+    markAsProcessed(filePath, mtimeMs);
+    console.log(`  ✓ Done. Saved ${approvedMemories.length} memories (${newCount} total in file).`);
+    console.log(`  ✓ Recorded version fingerprint. Ready.\n`);
+  } catch (error) {
+    console.error(`  ✗ Failed to process ${relativePath}:`, error.message);
+    console.log(`  ⚠️  Idempotency fingerprint NOT recorded. Will retry next time it changes.\n`);
+  }
+}