@hbarefoot/engram 1.0.0 → 1.2.0

package/src/import/parsers/obsidian.js

@@ -0,0 +1,219 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { validateContent } from '../../extract/secrets.js';
+
+/**
+ * Common Obsidian vault locations
+ */
+const VAULT_SEARCH_DIRS = [
+  os.homedir(),
+  path.join(os.homedir(), 'Documents'),
+  path.join(os.homedir(), 'Notes'),
+  path.join(os.homedir(), 'Obsidian')
+];
+
+/**
+ * Detect Obsidian vaults by looking for .obsidian directories
+ */
+export function detect(options = {}) {
+  const searchDirs = options.vaultPaths || VAULT_SEARCH_DIRS;
+  const vaults = [];
+
+  for (const dir of searchDirs) {
+    if (!fs.existsSync(dir)) continue;
+
+    try {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (!entry.isDirectory()) continue;
+        const obsidianDir = path.join(dir, entry.name, '.obsidian');
+        if (fs.existsSync(obsidianDir)) {
+          vaults.push(path.join(dir, entry.name));
+        }
+      }
+    } catch {
+      // Permission denied or other error, skip
+    }
+  }
+
+  return { found: vaults.length > 0, path: vaults[0] || null, vaults };
+}
+
+/**
+ * Parse Obsidian vaults for notes tagged with #engram or in a specific folder
+ */
+export async function parse(options = {}) {
+  const result = { source: 'obsidian', memories: [], skipped: [], warnings: [] };
+
+  const detection = detect(options);
+  const vaults = options.vaultPaths || detection.vaults || [];
+
+  if (vaults.length === 0) {
+    result.warnings.push('No Obsidian vaults found');
+    return result;
+  }
+
+  const tag = options.tag || '#engram';
+  const folder = options.folder || 'engram';
+  const maxNotes = options.maxNotes || 50;
+
+  let notesProcessed = 0;
+
+  for (const vaultPath of vaults) {
+    if (notesProcessed >= maxNotes) break;
+
+    // Look for notes with #engram tag
+    const taggedNotes = findTaggedNotes(vaultPath, tag, maxNotes - notesProcessed);
+    for (const note of taggedNotes) {
+      processNote(note, result);
+      notesProcessed++;
+    }
+
+    // Look for notes in engram/ folder
+    const folderPath = path.join(vaultPath, folder);
+    if (fs.existsSync(folderPath)) {
+      const folderNotes = findNotesInFolder(folderPath, maxNotes - notesProcessed);
+      for (const note of folderNotes) {
+        processNote(note, result);
+        notesProcessed++;
+      }
+    }
+  }
+
+  if (result.memories.length === 0) {
+    result.warnings.push(`No notes found with tag "${tag}" or in "${folder}/" folder`);
+  }
+
+  return result;
+}
+
+/**
+ * Find markdown files containing a specific tag
+ */
+function findTaggedNotes(vaultPath, tag, maxResults) {
+  const notes = [];
+  const tagPattern = tag.startsWith('#') ? tag : `#${tag}`;
+
+  function walk(dir, depth = 0) {
+    if (depth > 5 || notes.length >= maxResults) return;
+
+    try {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (notes.length >= maxResults) break;
+
+        const fullPath = path.join(dir, entry.name);
+
+        if (entry.isDirectory() && !entry.name.startsWith('.')) {
+          walk(fullPath, depth + 1);
+        } else if (entry.name.endsWith('.md')) {
+          try {
+            const content = fs.readFileSync(fullPath, 'utf-8');
+            if (content.includes(tagPattern)) {
+              notes.push({ path: fullPath, content, name: entry.name });
+            }
+          } catch {
+            // Skip unreadable files
+          }
+        }
+      }
+    } catch {
+      // Skip unreadable directories
+    }
+  }
+
+  walk(vaultPath);
+  return notes;
+}
+
+/**
+ * Find markdown files in a specific folder
+ */
+function findNotesInFolder(folderPath, maxResults) {
+  const notes = [];
+
+  try {
+    const entries = fs.readdirSync(folderPath, { withFileTypes: true });
+    for (const entry of entries) {
+      if (notes.length >= maxResults) break;
+
+      if (entry.name.endsWith('.md')) {
+        const fullPath = path.join(folderPath, entry.name);
+        try {
+          const content = fs.readFileSync(fullPath, 'utf-8');
+          notes.push({ path: fullPath, content, name: entry.name });
+        } catch {
+          // Skip unreadable files
+        }
+      }
+    }
+  } catch {
+    // Skip unreadable directories
+  }
+
+  return notes;
+}
+
+/**
+ * Process a single note into memory candidates
+ */
+function processNote(note, result) {
+  const lines = note.content.split('\n');
+  let currentHeading = path.basename(note.name, '.md');
+
+  for (const rawLine of lines) {
+    const line = rawLine.trim();
+
+    // Track headings for context
+    if (/^#{1,3}\s+/.test(line)) {
+      currentHeading = line.replace(/^#{1,3}\s+/, '').trim();
+      continue;
+    }
+
+    // Skip empty, tags-only, or very short lines
+    if (!line || line === '---' || line.startsWith('```')) continue;
+
+    // Strip markdown formatting
+    const cleaned = line
+      .replace(/^[-*]\s+/, '')
+      .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')  // Inline links
+      .replace(/[*_`~]/g, '')                     // Bold, italic, code
+      .replace(/#\w+/g, '')                       // Tags
+      .trim();
+
+    if (cleaned.length < 15) continue;
+
+    // Validate for secrets
+    const validation = validateContent(cleaned, { autoRedact: false });
+    if (!validation.valid) {
+      result.skipped.push({ content: cleaned.substring(0, 50), reason: 'Contains secrets' });
+      continue;
+    }
+
+    // Determine category from content
+    let category = 'fact';
+    if (/prefer|like|dislike|always use|never use/i.test(cleaned)) {
+      category = 'preference';
+    } else if (/decided|chose|switched/i.test(cleaned)) {
+      category = 'decision';
+    }
+
+    result.memories.push({
+      content: `From Obsidian (${currentHeading}): ${cleaned}`,
+      category,
+      entity: null,
+      confidence: 0.75,
+      tags: ['obsidian', 'notes'],
+      source: 'import:obsidian'
+    });
+  }
+}
+
+export const meta = {
+  name: 'obsidian',
+  label: 'Obsidian vaults',
+  description: 'Notes tagged with #engram or in engram/ folder',
+  category: 'fact',
+  locations: ['~/Documents/*/engram/', '~/*/.obsidian']
+};

package/src/import/parsers/package.js

@@ -0,0 +1,177 @@
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Detect if package.json exists
+ */
+export function detect(options = {}) {
+  const cwd = options.cwd || process.cwd();
+  const filePath = path.resolve(cwd, 'package.json');
+  return { found: fs.existsSync(filePath), path: fs.existsSync(filePath) ? filePath : null };
+}
+
+/**
+ * Parse package.json into memory candidates
+ */
+export async function parse(options = {}) {
+  const result = { source: 'package', memories: [], skipped: [], warnings: [] };
+  const cwd = options.cwd || process.cwd();
+  const filePath = options.filePath || path.resolve(cwd, 'package.json');
+
+  if (!fs.existsSync(filePath)) {
+    result.warnings.push('No package.json found');
+    return result;
+  }
+
+  let pkg;
+  try {
+    pkg = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  } catch {
+    result.warnings.push('Failed to parse package.json');
+    return result;
+  }
+
+  // Project name and description
+  if (pkg.name) {
+    const desc = pkg.description ? ` — ${pkg.description}` : '';
+    result.memories.push({
+      content: `Project "${pkg.name}" (v${pkg.version || '0.0.0'})${desc}`,
+      category: 'fact',
+      entity: pkg.name,
+      confidence: 1.0,
+      tags: ['package-json', 'project-info'],
+      source: 'import:package'
+    });
+  }
+
+  // Module type
+  if (pkg.type) {
+    result.memories.push({
+      content: `Project uses ${pkg.type === 'module' ? 'ESM (ES modules)' : 'CommonJS'} module system`,
+      category: 'fact',
+      entity: 'modules',
+      confidence: 1.0,
+      tags: ['package-json', 'module-system'],
+      source: 'import:package'
+    });
+  }
+
+  // Engines
+  if (pkg.engines) {
+    const engines = Object.entries(pkg.engines)
+      .map(([name, version]) => `${name} ${version}`)
+      .join(', ');
+    result.memories.push({
+      content: `Project requires: ${engines}`,
+      category: 'fact',
+      entity: 'runtime',
+      confidence: 1.0,
+      tags: ['package-json', 'engines'],
+      source: 'import:package'
+    });
+  }
+
+  // Scripts (key developer commands)
+  if (pkg.scripts) {
+    const importantScripts = ['start', 'dev', 'build', 'test', 'lint', 'deploy'];
+    const relevantScripts = Object.entries(pkg.scripts)
+      .filter(([key]) => importantScripts.some(s => key.includes(s)));
+
+    if (relevantScripts.length > 0) {
+      const scriptList = relevantScripts
+        .map(([key, cmd]) => `"${key}": ${cmd}`)
+        .join('; ');
+      result.memories.push({
+        content: `Project npm scripts: ${scriptList}`,
+        category: 'fact',
+        entity: 'npm',
+        confidence: 0.95,
+        tags: ['package-json', 'scripts'],
+        source: 'import:package'
+      });
+    }
+  }
+
+  // Key dependencies (frameworks, databases, etc.)
+  const allDeps = {
+    ...pkg.dependencies,
+    ...pkg.devDependencies
+  };
+
+  if (Object.keys(allDeps).length > 0) {
+    // Categorize notable dependencies
+    const frameworks = [];
+    const databases = [];
+    const tools = [];
+    const testing = [];
+
+    const categories = {
+      frameworks: ['react', 'vue', 'angular', 'svelte', 'next', 'nuxt', 'express', 'fastify', 'koa', 'hapi', 'nest', 'electron', 'tauri'],
+      databases: ['better-sqlite3', 'sqlite3', 'pg', 'mysql2', 'mongoose', 'prisma', 'sequelize', 'typeorm', 'drizzle', 'redis', 'ioredis'],
+      tools: ['typescript', 'vite', 'webpack', 'rollup', 'esbuild', 'tailwindcss', 'eslint', 'prettier', 'commander', 'inquirer'],
+      testing: ['vitest', 'jest', 'mocha', 'chai', 'cypress', 'playwright']
+    };
+
+    for (const dep of Object.keys(allDeps)) {
+      const base = dep.replace(/^@[^/]+\//, '');
+      if (categories.frameworks.some(f => base.includes(f))) frameworks.push(dep);
+      else if (categories.databases.some(d => base.includes(d))) databases.push(dep);
+      else if (categories.testing.some(t => base.includes(t))) testing.push(dep);
+      else if (categories.tools.some(t => base.includes(t))) tools.push(dep);
+    }
+
+    if (frameworks.length > 0) {
+      result.memories.push({
+        content: `Project frameworks/libraries: ${frameworks.join(', ')}`,
+        category: 'fact',
+        entity: 'tech-stack',
+        confidence: 1.0,
+        tags: ['package-json', 'frameworks'],
+        source: 'import:package'
+      });
+    }
+
+    if (databases.length > 0) {
+      result.memories.push({
+        content: `Project database dependencies: ${databases.join(', ')}`,
+        category: 'fact',
+        entity: 'database',
+        confidence: 1.0,
+        tags: ['package-json', 'databases'],
+        source: 'import:package'
+      });
+    }
+
+    if (testing.length > 0) {
+      result.memories.push({
+        content: `Project testing tools: ${testing.join(', ')}`,
+        category: 'fact',
+        entity: 'testing',
+        confidence: 1.0,
+        tags: ['package-json', 'testing'],
+        source: 'import:package'
+      });
+    }
+
+    if (tools.length > 0) {
+      result.memories.push({
+        content: `Project dev tools: ${tools.join(', ')}`,
+        category: 'fact',
+        entity: 'tooling',
+        confidence: 1.0,
+        tags: ['package-json', 'dev-tools'],
+        source: 'import:package'
+      });
+    }
+  }
+
+  return result;
+}
+
+export const meta = {
+  name: 'package',
+  label: 'package.json',
+  description: 'Tech stack, scripts, and dependencies',
+  category: 'fact',
+  locations: ['package.json']
+};

package/src/import/parsers/shell.js

@@ -0,0 +1,256 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { detectSecrets } from '../../extract/secrets.js';
+
+const HISTORY_LOCATIONS = [
+  path.join(os.homedir(), '.zsh_history'),
+  path.join(os.homedir(), '.bash_history'),
+  path.join(os.homedir(), '.local/share/fish/fish_history')
+];
+
+/**
+ * Commands that likely contain secrets and should be skipped
+ */
+const SECRET_COMMAND_PATTERNS = [
+  /export\s+\w*(KEY|SECRET|TOKEN|PASSWORD|PASS|API|AUTH)/i,
+  /curl.*(-H|--header)\s+['"]?(Authorization|X-Api-Key|Bearer)/i,
+  /curl.*(-u|--user)\s+/i,
+  /mysql.*-p/,
+  /psql.*password/i,
+  /echo\s+['"]?(sk-|pk_|ghp_|AKIA)/i,
+  /aws\s+configure/i,
+  /login.*--password/i,
+  /--token\s+\S+/i
+];
+
+/**
+ * Detect if shell history exists
+ */
+export function detect() {
+  for (const loc of HISTORY_LOCATIONS) {
+    if (fs.existsSync(loc)) {
+      return { found: true, path: loc };
+    }
+  }
+  return { found: false, path: null };
+}
+
+/**
+ * Parse shell history to extract command patterns
+ */
+export async function parse(options = {}) {
+  const result = { source: 'shell', memories: [], skipped: [], warnings: [] };
+
+  const filePath = options.filePath || (() => {
+    const detected = detect();
+    return detected.path;
+  })();
+
+  if (!filePath || !fs.existsSync(filePath)) {
+    result.warnings.push('No shell history found');
+    return result;
+  }
+
+  const isZsh = filePath.includes('zsh');
+  const isFish = filePath.includes('fish');
+  const raw = fs.readFileSync(filePath, 'utf-8');
+
+  // Extract commands from history
+  const commands = extractCommands(raw, { isZsh, isFish });
+
+  // Count command frequency
+  const frequency = {};
+  const baseCommands = {};
+
+  for (const cmd of commands) {
+    // Skip commands with potential secrets
+    if (containsSecret(cmd)) {
+      result.skipped.push({ content: cmd.substring(0, 50), reason: 'Potential secret in command' });
+      continue;
+    }
+
+    // Get base command (first word)
+    const base = cmd.split(/\s+/)[0];
+    if (!base || base.length < 2) continue;
+
+    baseCommands[base] = (baseCommands[base] || 0) + 1;
+
+    // Track full commands for pattern detection (normalize args)
+    const normalized = normalizeCommand(cmd);
+    frequency[normalized] = (frequency[normalized] || 0) + 1;
+  }
+
+  // Top base commands (tools the user uses most)
+  const topCommands = Object.entries(baseCommands)
+    .sort((a, b) => b[1] - a[1])
+    .filter(([cmd]) => !isBoringCommand(cmd))
+    .slice(0, 15);
+
+  if (topCommands.length > 0) {
+    const cmdStr = topCommands
+      .map(([cmd, count]) => `${cmd} (${count}x)`)
+      .join(', ');
+
+    result.memories.push({
+      content: `Most used shell commands: ${cmdStr}`,
+      category: 'pattern',
+      entity: 'shell',
+      confidence: 0.8,
+      tags: ['shell-history', 'command-patterns'],
+      source: 'import:shell'
+    });
+  }
+
+  // Detect frequently used tool patterns
+  const patterns = detectPatterns(frequency);
+  for (const pattern of patterns) {
+    result.memories.push({
+      content: pattern.description,
+      category: 'pattern',
+      entity: pattern.entity,
+      confidence: pattern.confidence,
+      tags: ['shell-history', 'workflow-patterns'],
+      source: 'import:shell'
+    });
+  }
+
+  // Detect package manager preference
+  const pmCounts = {
+    npm: baseCommands['npm'] || 0,
+    yarn: baseCommands['yarn'] || 0,
+    pnpm: baseCommands['pnpm'] || 0,
+    bun: baseCommands['bun'] || 0
+  };
+  const topPM = Object.entries(pmCounts).sort((a, b) => b[1] - a[1])[0];
+  if (topPM && topPM[1] > 5) {
+    result.memories.push({
+      content: `Preferred package manager: ${topPM[0]} (used ${topPM[1]} times in history)`,
+      category: 'preference',
+      entity: topPM[0],
+      confidence: 0.75,
+      tags: ['shell-history', 'package-manager'],
+      source: 'import:shell'
+    });
+  }
+
+  return result;
+}
+
+/**
+ * Extract clean commands from history file content
+ */
+function extractCommands(raw, { isZsh, isFish }) {
+  const lines = raw.split('\n');
+  const commands = [];
+
+  for (const line of lines) {
+    let cmd;
+
+    if (isZsh) {
+      // Zsh history format: ": timestamp:0;command"
+      const match = line.match(/^:\s*\d+:\d+;(.+)/);
+      cmd = match ? match[1] : line;
+    } else if (isFish) {
+      // Fish history format: "- cmd: command"
+      const match = line.match(/^- cmd:\s*(.+)/);
+      if (!match) continue;
+      cmd = match[1];
+    } else {
+      cmd = line;
+    }
+
+    cmd = cmd.trim();
+    if (cmd && cmd.length > 3 && cmd.length < 500) {
+      commands.push(cmd);
+    }
+  }
+
+  return commands;
+}
+
+/**
+ * Check if a command likely contains secrets
+ */
+function containsSecret(cmd) {
+  for (const pattern of SECRET_COMMAND_PATTERNS) {
+    if (pattern.test(cmd)) return true;
+  }
+
+  const detection = detectSecrets(cmd);
+  return detection.hasSecrets;
+}
+
+/**
+ * Normalize a command for frequency counting
+ */
+function normalizeCommand(cmd) {
+  // Replace specific paths/args with placeholders
+  return cmd
+    .replace(/['"]/g, '')
+    .replace(/\s+\S*\/\S+/g, ' <path>')
+    .replace(/\s+-\w\s+\S+/g, ' -<flag> <arg>')
+    .split(/\s+/)
+    .slice(0, 3)
+    .join(' ')
+    .trim();
+}
+
+/**
+ * Check if a command is too generic to be interesting
+ */
+function isBoringCommand(cmd) {
+  const boring = ['ls', 'cd', 'pwd', 'echo', 'cat', 'clear', 'exit', 'history',
+    'which', 'whoami', 'date', 'cal', 'man', 'true', 'false', 'test'];
+  return boring.includes(cmd);
+}
+
+/**
+ * Detect interesting workflow patterns from command frequency
+ */
+function detectPatterns(frequency) {
+  const patterns = [];
+
+  const dockerCount = Object.entries(frequency)
+    .filter(([cmd]) => cmd.startsWith('docker'))
+    .reduce((sum, [, c]) => sum + c, 0);
+  if (dockerCount > 10) {
+    patterns.push({
+      description: `Uses Docker frequently (${dockerCount} commands in history)`,
+      entity: 'docker',
+      confidence: 0.8
+    });
+  }
+
+  const gitCount = Object.entries(frequency)
+    .filter(([cmd]) => cmd.startsWith('git'))
+    .reduce((sum, [, c]) => sum + c, 0);
+  if (gitCount > 20) {
+    patterns.push({
+      description: `Heavy git user (${gitCount} commands in history)`,
+      entity: 'git',
+      confidence: 0.8
+    });
+  }
+
+  const sshCount = Object.entries(frequency)
+    .filter(([cmd]) => cmd.startsWith('ssh'))
+    .reduce((sum, [, c]) => sum + c, 0);
+  if (sshCount > 5) {
+    patterns.push({
+      description: `Regularly uses SSH for remote access (${sshCount} connections)`,
+      entity: 'ssh',
+      confidence: 0.75
+    });
+  }
+
+  return patterns;
+}
+
+export const meta = {
+  name: 'shell',
+  label: 'Shell history',
+  description: 'Frequent commands and workflow patterns',
+  category: 'pattern',
+  locations: HISTORY_LOCATIONS.map(p => p.replace(os.homedir(), '~'))
+};