@hbarefoot/engram 1.2.0 → 1.4.0

package/src/import/parsers/shell.js

@@ -26,14 +26,41 @@ const SECRET_COMMAND_PATTERNS = [
 
 /**
  * Detect if shell history exists
+ * @param {Object} [options] - Detection options
+ * @param {string[]} [options.paths] - Additional directories to scan for shell history
+ * @returns {{ found: boolean, path: string|null, paths: string[] }}
  */
-export function detect() {
+export function detect(options = {}) {
+  const foundPaths = [];
+  const seen = new Set();
+
   for (const loc of HISTORY_LOCATIONS) {
-    if (fs.existsSync(loc)) {
-      return { found: true, path: loc };
+    const resolved = path.resolve(loc);
+    if (!seen.has(resolved) && fs.existsSync(resolved)) {
+      seen.add(resolved);
+      foundPaths.push(resolved);
+    }
+  }
+
+  // Check additional paths for shell history files
+  if (options.paths && Array.isArray(options.paths)) {
+    for (const dir of options.paths) {
+      for (const name of ['.zsh_history', '.bash_history', '.local/share/fish/fish_history']) {
+        const loc = path.join(dir, name);
+        const resolved = path.resolve(loc);
+        if (!seen.has(resolved) && fs.existsSync(resolved)) {
+          seen.add(resolved);
+          foundPaths.push(resolved);
+        }
+      }
     }
   }
-  return { found: false, path: null };
+
+  return {
+    found: foundPaths.length > 0,
+    path: foundPaths[0] || null,
+    paths: foundPaths
+  };
 }
 
 /**
@@ -42,43 +69,46 @@ export function detect() {
 export async function parse(options = {}) {
   const result = { source: 'shell', memories: [], skipped: [], warnings: [] };
 
-  const filePath = options.filePath || (() => {
-    const detected = detect();
-    return detected.path;
-  })();
+  // Determine which files to parse
+  let filesToParse;
+  if (options.filePath) {
+    filesToParse = [options.filePath];
+  } else {
+    const detected = detect(options);
+    filesToParse = detected.paths;
+  }
 
-  if (!filePath || !fs.existsSync(filePath)) {
+  if (filesToParse.length === 0) {
     result.warnings.push('No shell history found');
     return result;
   }
 
-  const isZsh = filePath.includes('zsh');
-  const isFish = filePath.includes('fish');
-  const raw = fs.readFileSync(filePath, 'utf-8');
-
-  // Extract commands from history
-  const commands = extractCommands(raw, { isZsh, isFish });
-
-  // Count command frequency
+  // Merge commands from all history files
   const frequency = {};
   const baseCommands = {};
 
-  for (const cmd of commands) {
-    // Skip commands with potential secrets
-    if (containsSecret(cmd)) {
-      result.skipped.push({ content: cmd.substring(0, 50), reason: 'Potential secret in command' });
-      continue;
-    }
+  for (const filePath of filesToParse) {
+    if (!fs.existsSync(filePath)) continue;
 
-    // Get base command (first word)
-    const base = cmd.split(/\s+/)[0];
-    if (!base || base.length < 2) continue;
+    const isZsh = filePath.includes('zsh');
+    const isFish = filePath.includes('fish');
+    const raw = fs.readFileSync(filePath, 'utf-8');
+    const commands = extractCommands(raw, { isZsh, isFish });
 
-    baseCommands[base] = (baseCommands[base] || 0) + 1;
+    for (const cmd of commands) {
+      if (containsSecret(cmd)) {
+        result.skipped.push({ content: cmd.substring(0, 50), reason: 'Potential secret in command' });
+        continue;
+      }
 
-    // Track full commands for pattern detection (normalize args)
-    const normalized = normalizeCommand(cmd);
-    frequency[normalized] = (frequency[normalized] || 0) + 1;
+      const base = cmd.split(/\s+/)[0];
+      if (!base || base.length < 2) continue;
+
+      baseCommands[base] = (baseCommands[base] || 0) + 1;
+
+      const normalized = normalizeCommand(cmd);
+      frequency[normalized] = (frequency[normalized] || 0) + 1;
+    }
   }
 
   // Top base commands (tools the user uses most)

package/src/import/parsers/ssh.js

@@ -6,11 +6,36 @@ const SSH_CONFIG_PATH = path.join(os.homedir(), '.ssh/config');
 
 /**
  * Detect if SSH config exists
+ * @param {Object} [options] - Detection options
+ * @param {string[]} [options.paths] - Additional directories to scan for SSH config
+ * @returns {{ found: boolean, path: string|null, paths: string[] }}
  */
-export function detect() {
+export function detect(options = {}) {
+  const foundPaths = [];
+  const seen = new Set();
+
+  const resolved = path.resolve(SSH_CONFIG_PATH);
+  if (fs.existsSync(resolved)) {
+    seen.add(resolved);
+    foundPaths.push(resolved);
+  }
+
+  // Check additional paths for SSH config
+  if (options.paths && Array.isArray(options.paths)) {
+    for (const dir of options.paths) {
+      const loc = path.join(dir, '.ssh/config');
+      const r = path.resolve(loc);
+      if (!seen.has(r) && fs.existsSync(r)) {
+        seen.add(r);
+        foundPaths.push(r);
+      }
+    }
+  }
+
   return {
-    found: fs.existsSync(SSH_CONFIG_PATH),
-    path: fs.existsSync(SSH_CONFIG_PATH) ? SSH_CONFIG_PATH : null
+    found: foundPaths.length > 0,
+    path: foundPaths[0] || null,
+    paths: foundPaths
   };
 }
 
@@ -21,23 +46,45 @@ export function detect() {
 export async function parse(options = {}) {
   const result = { source: 'ssh', memories: [], skipped: [], warnings: [] };
 
-  const filePath = options.filePath || SSH_CONFIG_PATH;
+  // Determine which files to parse
+  let filesToParse;
+  if (options.filePath) {
+    filesToParse = [options.filePath];
+  } else {
+    const detected = detect(options);
+    filesToParse = detected.paths;
+  }
 
-  if (!fs.existsSync(filePath)) {
+  if (filesToParse.length === 0) {
     result.warnings.push('No ~/.ssh/config found');
     return result;
   }
 
-  const content = fs.readFileSync(filePath, 'utf-8');
-  const hosts = parseSSHConfig(content);
+  // Merge hosts from all config files, dedup by host name
+  const allHosts = [];
+  const seenHostNames = new Set();
+
+  for (const filePath of filesToParse) {
+    if (!fs.existsSync(filePath)) continue;
+
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const hosts = parseSSHConfig(content);
+
+    for (const host of hosts) {
+      if (!seenHostNames.has(host.name)) {
+        seenHostNames.add(host.name);
+        allHosts.push(host);
+      }
+    }
+  }
 
-  if (hosts.length === 0) {
+  if (allHosts.length === 0) {
     result.warnings.push('No SSH hosts found in config');
     return result;
   }
 
   // Filter out wildcard-only hosts
-  const namedHosts = hosts.filter(h => h.name !== '*' && !h.name.includes('*'));
+  const namedHosts = allHosts.filter(h => h.name !== '*' && !h.name.includes('*'));
 
   if (namedHosts.length === 0) {
     result.warnings.push('Only wildcard SSH hosts found');
@@ -70,7 +117,7 @@ export async function parse(options = {}) {
   }
 
   // Warn about skipped sensitive fields
-  const skippedCount = hosts.reduce((sum, h) => sum + (h.identityFile ? 1 : 0), 0);
+  const skippedCount = allHosts.reduce((sum, h) => sum + (h.identityFile ? 1 : 0), 0);
   if (skippedCount > 0) {
     result.warnings.push(`Skipped ${skippedCount} IdentityFile entries (security)`);
   }

package/src/import/wizard.js

@@ -1,4 +1,5 @@
 import { createInterface } from 'readline';
+import os from 'os';
 import { detectSources, scanSources, commitMemories } from './index.js';
 import { initDatabase } from '../memory/store.js';
 import { loadConfig, getDatabasePath } from '../config/index.js';
@@ -16,6 +17,7 @@ import chalk from 'chalk';
  * @param {boolean} [options.dryRun] - Preview without committing
  * @param {string} [options.namespace] - Override namespace
  * @param {string} [options.config] - Config file path
+ * @param {string[]} [options.paths] - Additional directories to scan
  */
 export async function runWizard(options = {}) {
   const config = loadConfig(options.config);
@@ -42,7 +44,9 @@ export async function runWizard(options = {}) {
     // Step 1: Detect available sources
     const spin = spinner('Scanning for sources...');
     spin.start();
-    const sources = await detectSources({ cwd: process.cwd() });
+    const scanOpts = { cwd: process.cwd() };
+    if (options.paths) scanOpts.paths = options.paths;
+    const sources = await detectSources(scanOpts);
     const foundSources = sources.filter(s => s.detected.found);
     const notFoundSources = sources.filter(s => !s.detected.found);
 
@@ -57,13 +61,16 @@ export async function runWizard(options = {}) {
     spin.succeed(`Found ${foundSources.length} source${foundSources.length === 1 ? '' : 's'}`);
     console.log('');
 
-    const srcTable = createTable({ head: ['#', 'Source', 'Description', 'Path'] });
+    const srcTable = createTable({ head: ['#', 'Source', 'Description', 'Path(s)'] });
     foundSources.forEach((s, i) => {
+      const pathsDisplay = s.detected.paths && s.detected.paths.length > 1
+        ? s.detected.paths.map(p => p.replace(os.homedir(), '~')).join(', ')
+        : (s.detected.path || '-').replace(os.homedir(), '~');
       srcTable.push([
         String(i + 1),
         chalk.bold(s.label),
         s.description,
-        chalk.dim(s.detected.path || '-')
+        chalk.dim(pathsDisplay)
       ]);
     });
     console.log(srcTable.toString());
@@ -99,7 +106,7 @@ export async function runWizard(options = {}) {
     console.log('');
     const scanSpin = spinner('Scanning sources...');
     scanSpin.start();
-    const scanResult = await scanSources(selectedIds, { cwd: process.cwd() });
+    const scanResult = await scanSources(selectedIds, scanOpts);
     scanSpin.succeed(`Found ${scanResult.memories.length} memories from ${selectedIds.length} source${selectedIds.length === 1 ? '' : 's'}`);
 
     if (scanResult.skipped.length > 0) {
@@ -205,7 +212,9 @@ async function runNonInteractive(config, options) {
 
   const scanSpin = spinner(`Scanning ${options.source}...`);
   scanSpin.start();
-  const scanResult = await scanSources([options.source], { cwd: process.cwd() });
+  const nonInteractiveOpts = { cwd: process.cwd() };
+  if (options.paths) nonInteractiveOpts.paths = options.paths;
+  const scanResult = await scanSources([options.source], nonInteractiveOpts);
   scanSpin.succeed(`Found ${scanResult.memories.length} memories`);
 
   if (scanResult.warnings.length > 0) {

package/src/memory/analytics.js

@@ -0,0 +1,259 @@
+import { getStats, getMemoriesWithEmbeddings } from './store.js';
+import { cosineSimilarity } from '../embed/index.js';
+import * as logger from '../utils/logger.js';
+
+/**
+ * Get analytics overview: totals, date-range counts, average confidence, health inputs
+ * @param {Database} db - SQLite database instance
+ * @returns {Object} Overview analytics
+ */
+export function getOverview(db) {
+  const stats = getStats(db);
+  const now = Date.now();
+  const sevenDaysAgo = now - 7 * 24 * 60 * 60 * 1000;
+  const thirtyDaysAgo = now - 30 * 24 * 60 * 60 * 1000;
+
+  const createdLast7Days = db.prepare(
+    'SELECT COUNT(*) as count FROM memories WHERE created_at >= ?'
+  ).get(sevenDaysAgo).count;
+
+  const createdLast30Days = db.prepare(
+    'SELECT COUNT(*) as count FROM memories WHERE created_at >= ?'
+  ).get(thirtyDaysAgo).count;
+
+  const avgConfidence = db.prepare(
+    'SELECT AVG(confidence) as avg FROM memories'
+  ).get().avg || 0;
+
+  const totalRecalled = db.prepare(
+    'SELECT COUNT(*) as count FROM memories WHERE access_count > 0'
+  ).get().count;
+
+  const accessedLast30Days = db.prepare(
+    'SELECT COUNT(*) as count FROM memories WHERE last_accessed >= ?'
+  ).get(thirtyDaysAgo).count;
+
+  return {
+    totalMemories: stats.total,
+    byCategory: stats.byCategory,
+    byNamespace: stats.byNamespace,
+    withEmbeddings: stats.withEmbeddings,
+    createdLast7Days,
+    createdLast30Days,
+    avgConfidence: Math.round(avgConfidence * 100) / 100,
+    totalRecalled,
+    accessedLast30Days,
+    recallRate: stats.total > 0
+      ? Math.round((totalRecalled / stats.total) * 100)
+      : 0
+  };
+}
+
+/**
+ * Get stale memories (not accessed in N+ days)
+ * @param {Database} db
+ * @param {number} daysThreshold - Days since last access (default 30)
+ * @param {number} limit - Max results (default 50)
+ * @returns {Object} { items, count }
+ */
+export function getStaleMemories(db, daysThreshold = 30, limit = 50) {
+  const now = Date.now();
+  const threshold = now - daysThreshold * 24 * 60 * 60 * 1000;
+
+  // Stale = last_accessed before threshold, OR never accessed AND created before threshold
+  const countResult = db.prepare(`
+    SELECT COUNT(*) as count FROM memories
+    WHERE (last_accessed IS NOT NULL AND last_accessed < ?)
+       OR (last_accessed IS NULL AND created_at < ?)
+  `).get(threshold, threshold);
+
+  const rows = db.prepare(`
+    SELECT id, content, category, entity, confidence, last_accessed, created_at, access_count
+    FROM memories
+    WHERE (last_accessed IS NOT NULL AND last_accessed < ?)
+       OR (last_accessed IS NULL AND created_at < ?)
+    ORDER BY COALESCE(last_accessed, created_at) ASC
+    LIMIT ?
+  `).all(threshold, threshold, limit);
+
+  const items = rows.map(row => ({
+    id: row.id,
+    content: row.content,
+    category: row.category,
+    entity: row.entity,
+    confidence: row.confidence,
+    lastAccessed: row.last_accessed,
+    accessCount: row.access_count,
+    daysSinceAccess: Math.floor(
+      (now - (row.last_accessed || row.created_at)) / (1000 * 60 * 60 * 24)
+    )
+  }));
+
+  return { items, count: countResult.count };
+}
+
+/**
+ * Get memories that have never been recalled (access_count = 0)
+ * @param {Database} db
+ * @param {number} limit
+ * @returns {Object} { items, count }
+ */
+export function getNeverRecalled(db, limit = 50) {
+  const now = Date.now();
+
+  const countResult = db.prepare(
+    'SELECT COUNT(*) as count FROM memories WHERE access_count = 0'
+  ).get();
+
+  const rows = db.prepare(`
+    SELECT id, content, category, entity, confidence, created_at
+    FROM memories
+    WHERE access_count = 0
+    ORDER BY created_at ASC
+    LIMIT ?
+  `).all(limit);
+
+  const items = rows.map(row => ({
+    id: row.id,
+    content: row.content,
+    category: row.category,
+    entity: row.entity,
+    confidence: row.confidence,
+    createdAt: row.created_at,
+    daysSinceCreation: Math.floor(
+      (now - row.created_at) / (1000 * 60 * 60 * 24)
+    )
+  }));
+
+  return { items, count: countResult.count };
+}
+
+/**
+ * Find clusters of duplicate/similar memories
+ * Uses cosine similarity on embeddings, threshold 0.85
+ * @param {Database} db
+ * @param {number} threshold - Similarity threshold (default 0.85)
+ * @returns {Object} { clusters, totalDuplicates }
+ */
+export function getDuplicateClusters(db, threshold = 0.85) {
+  const memories = getMemoriesWithEmbeddings(db);
+  const pairs = [];
+
+  // O(n²) comparison — same pattern as consolidate.js but lower threshold
+  for (let i = 0; i < memories.length; i++) {
+    for (let j = i + 1; j < memories.length; j++) {
+      const memA = memories[i];
+      const memB = memories[j];
+
+      if (memA.namespace !== memB.namespace) continue;
+
+      const similarity = cosineSimilarity(memA.embedding, memB.embedding);
+      if (similarity > threshold) {
+        pairs.push({ a: memA.id, b: memB.id, similarity, memA, memB });
+      }
+    }
+  }
+
+  // Group pairs into clusters using union-find
+  const parent = new Map();
+  function find(id) {
+    if (!parent.has(id)) parent.set(id, id);
+    if (parent.get(id) !== id) parent.set(id, find(parent.get(id)));
+    return parent.get(id);
+  }
+  function union(a, b) {
+    parent.set(find(a), find(b));
+  }
+
+  const memById = new Map();
+  const clusterSimilarity = new Map();
+
+  for (const { a, b, similarity, memA, memB } of pairs) {
+    memById.set(a, memA);
+    memById.set(b, memB);
+
+    const rootA = find(a);
+    const rootB = find(b);
+    union(a, b);
+    const newRoot = find(a);
+
+    const minSim = Math.min(
+      clusterSimilarity.get(rootA) ?? similarity,
+      clusterSimilarity.get(rootB) ?? similarity,
+      similarity
+    );
+    clusterSimilarity.set(newRoot, minSim);
+  }
+
+  // Build clusters
+  const clusterMap = new Map();
+  for (const id of memById.keys()) {
+    const root = find(id);
+    if (!clusterMap.has(root)) clusterMap.set(root, []);
+    clusterMap.get(root).push(id);
+  }
+
+  const clusters = [];
+  for (const [root, ids] of clusterMap.entries()) {
+    if (ids.length < 2) continue;
+    clusters.push({
+      memories: ids.map(id => {
+        const m = memById.get(id);
+        return {
+          id: m.id,
+          content: m.content,
+          category: m.category,
+          confidence: m.confidence,
+          accessCount: m.access_count
+        };
+      }),
+      similarity: Math.round((clusterSimilarity.get(root) || threshold) * 100) / 100
+    });
+  }
+
+  const totalDuplicates = clusters.reduce((sum, c) => sum + c.memories.length - 1, 0);
+
+  return { clusters, totalDuplicates };
+}
+
+/**
+ * Get daily trends data for the last N days
+ * @param {Database} db
+ * @param {number} days - Number of days to look back (default 30)
+ * @returns {Object} { daily: [{ date, created, avgConfidence }] }
+ */
+export function getTrends(db, days = 30) {
+  const now = Date.now();
+  const startTime = now - days * 24 * 60 * 60 * 1000;
+
+  // Get daily creation counts and average confidence
+  const rows = db.prepare(`
+    SELECT
+      date(created_at / 1000, 'unixepoch') as date,
+      COUNT(*) as created,
+      ROUND(AVG(confidence), 2) as avgConfidence
+    FROM memories
+    WHERE created_at >= ?
+    GROUP BY date(created_at / 1000, 'unixepoch')
+    ORDER BY date ASC
+  `).all(startTime);
+
+  // Fill in missing days with zeros
+  const daily = [];
+  const dateSet = new Map(rows.map(r => [r.date, r]));
+  const current = new Date(startTime);
+  const end = new Date(now);
+
+  while (current <= end) {
+    const dateStr = current.toISOString().split('T')[0];
+    const row = dateSet.get(dateStr);
+    daily.push({
+      date: dateStr,
+      created: row ? row.created : 0,
+      avgConfidence: row ? row.avgConfidence : null
+    });
+    current.setDate(current.getDate() + 1);
+  }
+
+  return { daily };
+}

package/src/memory/health.js

@@ -0,0 +1,100 @@
+/**
+ * Calculate a composite memory health score (0-100)
+ *
+ * Factors and weights (sum to 1.0):
+ * - Recall rate:   % of memories accessed at least once        (0.25)
+ * - Freshness:     % of memories accessed in last 30 days      (0.20)
+ * - Confidence:    average confidence score                     (0.20)
+ * - Diversity:     category distribution entropy                (0.15)
+ * - Growth:        positive trend in memory creation            (0.10)
+ * - Cleanliness:   inverse of duplicate ratio                   (0.10)
+ *
+ * @param {Object} overview - From analytics.getOverview()
+ * @param {number} duplicateCount - From analytics.getDuplicateClusters().totalDuplicates
+ * @param {Object} trends - From analytics.getTrends()
+ * @returns {number} Health score 0-100
+ */
+export function calculateHealthScore(overview, duplicateCount = 0, trends = null) {
+  if (overview.totalMemories === 0) return 0;
+
+  // 1. Recall rate (0.25): % of memories ever accessed
+  const recallRate = overview.totalRecalled / overview.totalMemories;
+
+  // 2. Freshness (0.20): % of memories accessed in last 30 days
+  const freshness = overview.accessedLast30Days / overview.totalMemories;
+
+  // 3. Confidence avg (0.20): already 0-1
+  const confidence = overview.avgConfidence;
+
+  // 4. Diversity (0.15): Shannon entropy of category distribution, normalized
+  const diversity = calculateDiversity(overview.byCategory);
+
+  // 5. Growth (0.10): whether memories are being actively created
+  const growth = calculateGrowth(overview, trends);
+
+  // 6. Cleanliness (0.10): inverse of duplicate ratio
+  const duplicateRatio = duplicateCount / overview.totalMemories;
+  const cleanliness = Math.max(0, 1 - duplicateRatio * 5); // 20% duplicates = 0
+
+  const weightedScore =
+    recallRate * 0.25 +
+    freshness * 0.20 +
+    confidence * 0.20 +
+    diversity * 0.15 +
+    growth * 0.10 +
+    cleanliness * 0.10;
+
+  return Math.round(Math.min(1, Math.max(0, weightedScore)) * 100);
+}
+
+/**
+ * Calculate category diversity using normalized Shannon entropy
+ * Max entropy = all 5 categories equally distributed
+ * @param {Object} byCategory - { preference: n, fact: n, ... }
+ * @returns {number} 0-1 diversity score
+ */
+function calculateDiversity(byCategory) {
+  const counts = Object.values(byCategory);
+  const total = counts.reduce((s, c) => s + c, 0);
+  if (total === 0 || counts.length <= 1) return 0;
+
+  let entropy = 0;
+  for (const count of counts) {
+    if (count === 0) continue;
+    const p = count / total;
+    entropy -= p * Math.log2(p);
+  }
+
+  // Normalize by max possible entropy (5 categories)
+  const maxEntropy = Math.log2(5);
+  return entropy / maxEntropy;
+}
+
+/**
+ * Calculate growth score based on recent creation activity
+ * @param {Object} overview
+ * @param {Object|null} trends
+ * @returns {number} 0-1 growth score
+ */
+function calculateGrowth(overview, trends) {
+  // If we have trends data, compare recent week vs prior weeks
+  if (trends && trends.daily && trends.daily.length >= 14) {
+    const daily = trends.daily;
+    const recentWeek = daily.slice(-7).reduce((s, d) => s + d.created, 0);
+    const priorWeek = daily.slice(-14, -7).reduce((s, d) => s + d.created, 0);
+
+    if (priorWeek === 0 && recentWeek > 0) return 1;
+    if (priorWeek === 0 && recentWeek === 0) return 0.3;
+    const ratio = recentWeek / priorWeek;
+    return Math.min(1, ratio / 2); // ratio of 2 = max growth score
+  }
+
+  // Fallback: use 7-day vs 30-day ratio
+  if (overview.createdLast30Days === 0) return 0;
+  const weeklyRate = overview.createdLast7Days / 7;
+  const monthlyRate = overview.createdLast30Days / 30;
+  if (monthlyRate === 0) return 0;
+
+  const ratio = weeklyRate / monthlyRate;
+  return Math.min(1, ratio / 2);
+}