@hbarefoot/engram 1.1.0 → 1.2.0

package/src/import/parsers/ssh.js

@@ -0,0 +1,132 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+
+const SSH_CONFIG_PATH = path.join(os.homedir(), '.ssh/config');
+
+/**
+ * Detect if SSH config exists
+ */
+export function detect() {
+  return {
+    found: fs.existsSync(SSH_CONFIG_PATH),
+    path: fs.existsSync(SSH_CONFIG_PATH) ? SSH_CONFIG_PATH : null
+  };
+}
+
+/**
+ * Parse ~/.ssh/config for host names only.
+ * NEVER extracts keys, IdentityFile contents, passwords, or passphrases.
+ */
+export async function parse(options = {}) {
+  const result = { source: 'ssh', memories: [], skipped: [], warnings: [] };
+
+  const filePath = options.filePath || SSH_CONFIG_PATH;
+
+  if (!fs.existsSync(filePath)) {
+    result.warnings.push('No ~/.ssh/config found');
+    return result;
+  }
+
+  const content = fs.readFileSync(filePath, 'utf-8');
+  const hosts = parseSSHConfig(content);
+
+  if (hosts.length === 0) {
+    result.warnings.push('No SSH hosts found in config');
+    return result;
+  }
+
+  // Filter out wildcard-only hosts
+  const namedHosts = hosts.filter(h => h.name !== '*' && !h.name.includes('*'));
+
+  if (namedHosts.length === 0) {
+    result.warnings.push('Only wildcard SSH hosts found');
+    return result;
+  }
+
+  // Create a summary memory with all host names
+  const hostNames = namedHosts.map(h => h.name);
+  result.memories.push({
+    content: `SSH configured hosts: ${hostNames.join(', ')}`,
+    category: 'fact',
+    entity: 'ssh',
+    confidence: 0.9,
+    tags: ['ssh-config', 'servers'],
+    source: 'import:ssh'
+  });
+
+  // Individual host entries with hostname (not key info)
+  for (const host of namedHosts) {
+    if (host.hostname) {
+      result.memories.push({
+        content: `SSH host "${host.name}" connects to ${host.hostname}${host.user ? ` as ${host.user}` : ''}${host.port ? ` on port ${host.port}` : ''}`,
+        category: 'fact',
+        entity: 'ssh',
+        confidence: 0.9,
+        tags: ['ssh-config', 'servers'],
+        source: 'import:ssh'
+      });
+    }
+  }
+
+  // Warn about skipped sensitive fields
+  const skippedCount = hosts.reduce((sum, h) => sum + (h.identityFile ? 1 : 0), 0);
+  if (skippedCount > 0) {
+    result.warnings.push(`Skipped ${skippedCount} IdentityFile entries (security)`);
+  }
+
+  return result;
+}
+
+/**
+ * Parse SSH config file into host objects
+ * Only extracts safe fields: Host, HostName, User, Port
+ */
+function parseSSHConfig(content) {
+  const hosts = [];
+  let current = null;
+
+  for (const rawLine of content.split('\n')) {
+    const line = rawLine.trim();
+
+    if (!line || line.startsWith('#')) continue;
+
+    const [key, ...valueParts] = line.split(/\s+/);
+    const value = valueParts.join(' ');
+    const keyLower = key.toLowerCase();
+
+    if (keyLower === 'host') {
+      if (current) hosts.push(current);
+      current = { name: value };
+    } else if (current) {
+      // ONLY extract safe, non-secret fields
+      switch (keyLower) {
+        case 'hostname':
+          current.hostname = value;
+          break;
+        case 'user':
+          current.user = value;
+          break;
+        case 'port':
+          current.port = value;
+          break;
+        case 'identityfile':
+          // Note existence but NEVER store the path or contents
+          current.identityFile = true;
+          break;
+        // All other fields are intentionally ignored for security
+      }
+    }
+  }
+
+  if (current) hosts.push(current);
+  return hosts;
+}
+
+export const meta = {
+  name: 'ssh',
+  label: '~/.ssh/config',
+  description: 'Server names and hosts (NEVER extracts keys)',
+  category: 'fact',
+  locations: ['~/.ssh/config']
+};

package/src/import/wizard.js

@@ -0,0 +1,280 @@
+import { createInterface } from 'readline';
+import { detectSources, scanSources, commitMemories } from './index.js';
+import { initDatabase } from '../memory/store.js';
+import { loadConfig, getDatabasePath } from '../config/index.js';
+import {
+  printHeader, printSection, warning, info,
+  categoryBadge, confidenceColor, truncate,
+  createTable, spinner
+} from '../utils/format.js';
+import chalk from 'chalk';
+
+/**
+ * Interactive CLI import wizard
+ * @param {Object} [options] - Wizard options
+ * @param {string} [options.source] - Single source for non-interactive mode
+ * @param {boolean} [options.dryRun] - Preview without committing
+ * @param {string} [options.namespace] - Override namespace
+ * @param {string} [options.config] - Config file path
+ */
+export async function runWizard(options = {}) {
+  const config = loadConfig(options.config);
+
+  // Non-interactive single-source mode
+  if (options.source) {
+    return runNonInteractive(config, options);
+  }
+
+  const rl = createInterface({
+    input: process.stdin,
+    output: process.stdout
+  });
+
+  const ask = (question) => new Promise(resolve => rl.question(question, resolve));
+
+  try {
+    printHeader('1.1.0');
+    printSection('Smart Import Wizard');
+    console.log('');
+    info('Scanning your system for developer artifacts to import as memories.');
+    console.log('');
+
+    // Step 1: Detect available sources
+    const spin = spinner('Scanning for sources...');
+    spin.start();
+    const sources = await detectSources({ cwd: process.cwd() });
+    const foundSources = sources.filter(s => s.detected.found);
+    const notFoundSources = sources.filter(s => !s.detected.found);
+
+    if (foundSources.length === 0) {
+      spin.fail('No importable sources detected');
+      info('Try running this from a project directory with package.json, .cursorrules, etc.');
+      console.log('');
+      rl.close();
+      return;
+    }
+
+    spin.succeed(`Found ${foundSources.length} source${foundSources.length === 1 ? '' : 's'}`);
+    console.log('');
+
+    const srcTable = createTable({ head: ['#', 'Source', 'Description', 'Path'] });
+    foundSources.forEach((s, i) => {
+      srcTable.push([
+        String(i + 1),
+        chalk.bold(s.label),
+        s.description,
+        chalk.dim(s.detected.path || '-')
+      ]);
+    });
+    console.log(srcTable.toString());
+
+    if (notFoundSources.length > 0) {
+      console.log('');
+      info(`Not found: ${chalk.dim(notFoundSources.map(s => s.label).join(', '))}`);
+    }
+
+    // Step 2: Select sources
+    console.log('');
+    const selection = await ask(`${chalk.bold('Select sources')} (comma-separated numbers, or "all"): `);
+
+    let selectedIds;
+    if (selection.trim().toLowerCase() === 'all') {
+      selectedIds = foundSources.map(s => s.id);
+    } else {
+      const indices = selection.split(',').map(s => parseInt(s.trim()) - 1);
+      selectedIds = indices
+        .filter(i => i >= 0 && i < foundSources.length)
+        .map(i => foundSources[i].id);
+    }
+
+    if (selectedIds.length === 0) {
+      info('No sources selected. Exiting.');
+      rl.close();
+      return;
+    }
+
+    info(`Selected: ${chalk.bold(selectedIds.join(', '))}`);
+
+    // Step 3: Scan
+    console.log('');
+    const scanSpin = spinner('Scanning sources...');
+    scanSpin.start();
+    const scanResult = await scanSources(selectedIds, { cwd: process.cwd() });
+    scanSpin.succeed(`Found ${scanResult.memories.length} memories from ${selectedIds.length} source${selectedIds.length === 1 ? '' : 's'}`);
+
+    if (scanResult.skipped.length > 0) {
+      warning(`Skipped: ${scanResult.skipped.length} items (security or parse errors)`);
+    }
+    if (scanResult.warnings.length > 0) {
+      for (const w of scanResult.warnings) {
+        warning(w);
+      }
+    }
+
+    if (scanResult.memories.length === 0) {
+      console.log('');
+      info('No memories to import. Exiting.');
+      rl.close();
+      return;
+    }
+
+    // Step 4: Preview
+    printSection('Preview');
+
+    const previewCount = Math.min(scanResult.memories.length, 20);
+    const previewTable = createTable({ head: ['#', 'Category', 'Content', 'Confidence', 'Source'] });
+    for (let i = 0; i < previewCount; i++) {
+      const m = scanResult.memories[i];
+      previewTable.push([
+        String(i + 1),
+        categoryBadge(m.category),
+        truncate(m.content, 50),
+        confidenceColor(m.confidence),
+        chalk.dim(m.source)
+      ]);
+    }
+    console.log(previewTable.toString());
+
+    if (scanResult.memories.length > 20) {
+      info(`... and ${scanResult.memories.length - 20} more`);
+    }
+
+    // Step 5: Confirm
+    if (options.dryRun) {
+      console.log('');
+      info('Dry run mode — no memories were stored.');
+      console.log('');
+      printSummary(scanResult);
+      rl.close();
+      return;
+    }
+
+    console.log('');
+    const confirm = await ask(`${chalk.bold(`Commit ${scanResult.memories.length} memories?`)} (y/N) `);
+    if (confirm.trim().toLowerCase() !== 'y') {
+      info('Import cancelled.');
+      rl.close();
+      return;
+    }
+
+    // Step 6: Commit
+    const commitSpin = spinner('Committing memories...');
+    commitSpin.start();
+    const db = initDatabase(getDatabasePath(config));
+
+    const commitResult = await commitMemories(db, scanResult.memories, {
+      namespace: options.namespace
+    });
+
+    db.close();
+    commitSpin.succeed('Import complete');
+
+    // Step 7: Summary
+    console.log('');
+    const summaryTable = createTable({ head: ['Metric', 'Count'] });
+    summaryTable.push(
+      ['Created',    String(commitResult.created)],
+      ['Duplicates', `${commitResult.duplicates} (skipped)`],
+      ['Merged',     String(commitResult.merged)],
+      ['Rejected',   `${commitResult.rejected} (security)`]
+    );
+    if (commitResult.errors.length > 0) {
+      summaryTable.push(['Errors', String(commitResult.errors.length)]);
+    }
+    summaryTable.push(['Duration', `${commitResult.duration}ms`]);
+    console.log(summaryTable.toString());
+
+    console.log('');
+    info(`View your memories at ${chalk.cyan('http://localhost:3838')}`);
+    console.log('');
+
+    rl.close();
+    return commitResult;
+  } catch (err) {
+    rl.close();
+    throw err;
+  }
+}
+
+/**
+ * Non-interactive single-source import
+ */
+async function runNonInteractive(config, options) {
+  printSection(`Import: ${options.source}`);
+  console.log('');
+
+  const scanSpin = spinner(`Scanning ${options.source}...`);
+  scanSpin.start();
+  const scanResult = await scanSources([options.source], { cwd: process.cwd() });
+  scanSpin.succeed(`Found ${scanResult.memories.length} memories`);
+
+  if (scanResult.warnings.length > 0) {
+    for (const w of scanResult.warnings) {
+      warning(w);
+    }
+  }
+
+  if (scanResult.memories.length === 0) {
+    info('No memories to import.');
+    console.log('');
+    return;
+  }
+
+  if (options.dryRun) {
+    printSection('Preview');
+    const previewTable = createTable({ head: ['Category', 'Content'] });
+    for (const m of scanResult.memories) {
+      previewTable.push([
+        categoryBadge(m.category),
+        truncate(m.content, 60)
+      ]);
+    }
+    console.log(previewTable.toString());
+    console.log('');
+    info('Dry run — no memories stored.');
+    console.log('');
+    return;
+  }
+
+  const commitSpin = spinner('Committing memories...');
+  commitSpin.start();
+  const db = initDatabase(getDatabasePath(config));
+  const commitResult = await commitMemories(db, scanResult.memories, {
+    namespace: options.namespace
+  });
+  db.close();
+  commitSpin.succeed(`Imported ${commitResult.created} memories (${commitResult.duplicates} duplicates skipped)`);
+  console.log('');
+  return commitResult;
+}
+
+/**
+ * Print scan summary
+ */
+function printSummary(scanResult) {
+  const byCategory = {};
+  const bySource = {};
+
+  for (const m of scanResult.memories) {
+    byCategory[m.category] = (byCategory[m.category] || 0) + 1;
+    bySource[m.source] = (bySource[m.source] || 0) + 1;
+  }
+
+  if (Object.keys(byCategory).length > 0) {
+    const catTable = createTable({ head: ['Category', 'Count'] });
+    for (const [cat, count] of Object.entries(byCategory)) {
+      catTable.push([categoryBadge(cat), String(count)]);
+    }
+    console.log(catTable.toString());
+  }
+
+  if (Object.keys(bySource).length > 0) {
+    console.log('');
+    const srcTable = createTable({ head: ['Source', 'Count'] });
+    for (const [src, count] of Object.entries(bySource)) {
+      srcTable.push([src, String(count)]);
+    }
+    console.log(srcTable.toString());
+  }
+  console.log('');
+}

package/src/server/mcp.js

@@ -575,6 +575,10 @@ export class EngramMCPServer {
       modelInfo = {
         name: 'unknown',
         available: false,
+        cached: false,
+        loading: false,
+        sizeMB: 0,
+        path: '',
         error: error.message
       };
     }
@@ -590,7 +594,7 @@ export class EngramMCPServer {
 
 🤖 Embedding Model:
 - Name: ${modelInfo.name}
-- Available: ${modelInfo.available ? 'Yes' : 'No'}
+- Status: ${modelInfo.cached ? 'Ready' : modelInfo.loading ? 'Loading...' : modelInfo.available ? 'Available (not loaded)' : 'Not available'}
 - Cached: ${modelInfo.cached ? 'Yes' : 'No'}
 - Size: ${modelInfo.sizeMB} MB
 - Path: ${modelInfo.path}

package/src/server/rest.js

@@ -1,5 +1,6 @@
 import Fastify from 'fastify';
 import fastifyStatic from '@fastify/static';
+import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { loadConfig, getDatabasePath, getModelsPath } from '../config/index.js';
@@ -64,6 +65,10 @@ export function createRESTServer(config) {
         modelInfo = {
           name: 'unknown',
           available: false,
+          cached: false,
+          loading: false,
+          sizeMB: 0,
+          path: '',
           error: error.message
         };
       }
@@ -79,6 +84,7 @@ export function createRESTServer(config) {
         model: {
           name: modelInfo.name,
           available: modelInfo.available,
+          loading: modelInfo.loading || false,
           cached: modelInfo.cached,
           size: modelInfo.sizeMB,
           path: modelInfo.path
@@ -193,6 +199,13 @@ export function createRESTServer(config) {
         namespace
       });
 
+      // Get total count for pagination
+      let countQuery = 'SELECT COUNT(*) as count FROM memories WHERE 1=1';
+      const countParams = [];
+      if (namespace) { countQuery += ' AND namespace = ?'; countParams.push(namespace); }
+      if (category) { countQuery += ' AND category = ?'; countParams.push(category); }
+      const totalCount = db.prepare(countQuery).get(...countParams).count;
+
       return {
         success: true,
         memories: memories.map(m => ({
@@ -210,7 +223,7 @@ export function createRESTServer(config) {
         pagination: {
           limit: parseInt(limit),
           offset: parseInt(offset),
-          total: memories.length
+          total: totalCount
         }
       };
     } catch (error) {
@@ -472,18 +485,118 @@ export function createRESTServer(config) {
     }
   });
 
-  // Serve dashboard static files
-  const dashboardPath = path.resolve(__dirname, '../../dashboard/dist');
+  // === Import Wizard Endpoints ===
+
+  // Get available import sources with detection status
+  fastify.get('/api/import/sources', async (request, reply) => {
+    try {
+      const { detectSources } = await import('../import/index.js');
+      const cwd = request.query.cwd || process.cwd();
+      const sources = await detectSources({ cwd });
+
+      return {
+        success: true,
+        sources: sources.map(s => ({
+          id: s.id,
+          name: s.name,
+          label: s.label,
+          description: s.description,
+          category: s.category,
+          detected: s.detected
+        }))
+      };
+    } catch (error) {
+      logger.error('Import sources error', { error: error.message });
+      reply.code(500);
+      return { error: error.message };
+    }
+  });
+
+  // Scan selected sources for memory candidates
+  fastify.post('/api/import/scan', async (request, reply) => {
+    try {
+      const { sources } = request.body;
+
+      if (!sources || !Array.isArray(sources) || sources.length === 0) {
+        reply.code(400);
+        return { error: 'sources array is required' };
+      }
+
+      const { scanSources } = await import('../import/index.js');
+      const result = await scanSources(sources, { cwd: process.cwd() });
+
+      return {
+        success: true,
+        memories: result.memories.map((m, i) => ({
+          _index: i,
+          content: m.content,
+          category: m.category,
+          entity: m.entity,
+          confidence: m.confidence,
+          tags: m.tags,
+          source: m.source
+        })),
+        skipped: result.skipped,
+        warnings: result.warnings,
+        sources: result.sources,
+        duration: result.duration
+      };
+    } catch (error) {
+      logger.error('Import scan error', { error: error.message });
+      reply.code(500);
+      return { error: error.message };
+    }
+  });
 
-  fastify.register(fastifyStatic, {
-    root: dashboardPath,
-    prefix: '/'
+  // Commit imported memories
+  fastify.post('/api/import/commit', async (request, reply) => {
+    try {
+      const { memories, namespace } = request.body;
+
+      if (!memories || !Array.isArray(memories) || memories.length === 0) {
+        reply.code(400);
+        return { error: 'memories array is required' };
+      }
+
+      const { commitMemories } = await import('../import/index.js');
+      const result = await commitMemories(db, memories, { namespace });
+
+      return {
+        success: true,
+        results: {
+          total: result.total,
+          created: result.created,
+          duplicates: result.duplicates,
+          merged: result.merged,
+          rejected: result.rejected,
+          errors: result.errors,
+          duration: result.duration
+        }
+      };
+    } catch (error) {
+      logger.error('Import commit error', { error: error.message });
+      reply.code(500);
+      return { error: error.message };
+    }
   });
 
-  // Fallback to index.html for SPA routing
+  // Serve dashboard static files (skip if dist dir doesn't exist, e.g. in sidecar bundle)
+  const dashboardPath = path.resolve(__dirname, '../../dashboard/dist');
+  const hasDashboard = fs.existsSync(dashboardPath);
+
+  if (hasDashboard) {
+    fastify.register(fastifyStatic, {
+      root: dashboardPath,
+      prefix: '/'
+    });
+  } else {
+    logger.warn('Dashboard dist not found, skipping static file serving', { path: dashboardPath });
+  }
+
+  // Fallback to index.html for SPA routing (only when dashboard is available)
   fastify.setNotFoundHandler((request, reply) => {
-    if (!request.url.startsWith('/api') && !request.url.startsWith('/health')) {
-      reply.sendFile('index.html');
+    if (!request.url.startsWith('/api') && !request.url.startsWith('/health') && hasDashboard) {
+      reply.type('text/html').sendFile('index.html');
     } else {
       reply.code(404).send({ error: 'Not found' });
     }
@@ -514,6 +627,21 @@ export async function startRESTServer(config, port = 3838) {
 
     logger.info('REST API server started', { port, url: `http://localhost:${port}` });
 
+    // Set TRANSFORMERS_CACHE early so isModelAvailable() can find cached models
+    const modelsPath = getModelsPath(config);
+    process.env.TRANSFORMERS_CACHE = modelsPath;
+
+    // Pre-warm embedding pipeline in background so status reports correctly
+    import('../embed/index.js').then(({ initializePipeline }) => {
+      initializePipeline(modelsPath).then(() => {
+        logger.info('Embedding pipeline pre-warmed');
+      }).catch(err => {
+        logger.warn('Embedding pipeline pre-warm failed (will retry on first use)', { error: err.message });
+      });
+    }).catch(err => {
+      logger.warn('Failed to import embedding module for pre-warm', { error: err?.message ?? String(err) });
+    });
+
     return fastify;
   } catch (error) {
     logger.error('Failed to start REST server', { error: error.message });