npm - @hbarefoot/engram - Versions diffs - 1.0.0 - Mend

@hbarefoot/engram 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (24) hide show

package/LICENSE +21 -0
package/README.md +535 -0
package/bin/engram.js +421 -0
package/dashboard/dist/assets/index-BHkLa5w_.css +1 -0
package/dashboard/dist/assets/index-D9QR_Cnu.js +45 -0
package/dashboard/dist/index.html +14 -0
package/dashboard/package.json +21 -0
package/package.json +76 -0
package/src/config/index.js +150 -0
package/src/embed/index.js +249 -0
package/src/export/static.js +396 -0
package/src/extract/rules.js +233 -0
package/src/extract/secrets.js +114 -0
package/src/index.js +54 -0
package/src/memory/consolidate.js +420 -0
package/src/memory/context.js +346 -0
package/src/memory/feedback.js +197 -0
package/src/memory/recall.js +350 -0
package/src/memory/store.js +626 -0
package/src/server/mcp.js +668 -0
package/src/server/rest.js +499 -0
package/src/utils/id.js +9 -0
package/src/utils/logger.js +79 -0
package/src/utils/time.js +296 -0

package/dashboard/dist/index.html ADDED Viewed

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Engram Dashboard</title>
+    <script type="module" crossorigin src="/assets/index-D9QR_Cnu.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-BHkLa5w_.css">
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

package/dashboard/package.json ADDED Viewed

@@ -0,0 +1,21 @@
+{
+  "name": "engram-dashboard",
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1"
+  },
+  "devDependencies": {
+    "@vitejs/plugin-react": "^4.3.4",
+    "autoprefixer": "^10.4.20",
+    "postcss": "^8.4.49",
+    "tailwindcss": "^3.4.17",
+    "vite": "^6.0.5"
+  }
+}

package/package.json ADDED Viewed

@@ -0,0 +1,76 @@
+{
+  "name": "@hbarefoot/engram",
+  "version": "1.0.0",
+  "description": "Persistent memory for AI agents. SQLite for agent state.",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "engram": "bin/engram.js"
+  },
+  "files": [
+    "bin",
+    "src",
+    "dashboard/dist",
+    "dashboard/package.json",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "start": "node bin/engram.js start",
+    "mcp": "node bin/engram.js start --mcp-only",
+    "dev": "concurrently \"node bin/engram.js start\" \"cd dashboard && npm run dev\"",
+    "build": "cd dashboard && npm run build",
+    "prepublishOnly": "npm run build",
+    "test": "vitest",
+    "test:run": "vitest run",
+    "lint": "eslint src/",
+    "pm2:start": "pm2 start ecosystem.config.cjs",
+    "pm2:stop": "pm2 stop engram",
+    "pm2:restart": "pm2 restart engram",
+    "pm2:delete": "pm2 delete engram",
+    "pm2:logs": "pm2 logs engram",
+    "pm2:status": "pm2 status engram",
+    "pm2:monit": "pm2 monit"
+  },
+  "keywords": [
+    "ai",
+    "agent",
+    "memory",
+    "mcp",
+    "claude",
+    "llm",
+    "persistent",
+    "knowledge",
+    "sqlite",
+    "embedding"
+  ],
+  "license": "MIT",
+  "author": "Engram Contributors",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/HBarefoot/engram.git"
+  },
+  "bugs": {
+    "url": "https://github.com/HBarefoot/engram/issues"
+  },
+  "homepage": "https://github.com/HBarefoot/engram#readme",
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "better-sqlite3": "^11.0.0",
+    "fastify": "^5.0.0",
+    "@fastify/static": "^8.0.0",
+    "@fastify/multipart": "^9.0.0",
+    "@fastify/cors": "^10.0.0",
+    "commander": "^12.0.0",
+    "@xenova/transformers": "^2.17.0",
+    "yaml": "^2.4.0"
+  },
+  "devDependencies": {
+    "vitest": "^2.0.0",
+    "eslint": "^9.0.0",
+    "concurrently": "^9.0.0"
+  }
+}

package/src/config/index.js ADDED Viewed

@@ -0,0 +1,150 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+/**
+ * Default configuration for Engram
+ */
+const DEFAULT_CONFIG = {
+  port: 3838,
+  dataDir: path.join(os.homedir(), '.engram'),
+  defaults: {
+    namespace: 'default',
+    recallLimit: 5,
+    confidenceThreshold: 0.3,
+    tokenBudget: 500,
+    maxRecallResults: 20
+  },
+  embedding: {
+    provider: 'local',
+    model: 'Xenova/all-MiniLM-L6-v2',
+    endpoint: null
+  },
+  llm: {
+    provider: null,
+    endpoint: null,
+    model: null,
+    apiKey: null
+  },
+  consolidation: {
+    enabled: true,
+    intervalHours: 24,
+    duplicateThreshold: 0.92,
+    decayEnabled: true
+  },
+  security: {
+    secretDetection: true,
+    auditLog: false
+  }
+};
+/**
+ * Deep merge two objects
+ * @param {Object} target - Target object
+ * @param {Object} source - Source object
+ * @returns {Object} Merged object
+ */
+function deepMerge(target, source) {
+  const result = { ...target };
+  for (const key in source) {
+    if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
+      result[key] = deepMerge(target[key] || {}, source[key]);
+    } else {
+      result[key] = source[key];
+    }
+  }
+  return result;
+}
+/**
+ * Ensure the data directory exists
+ * @param {string} dataDir - Path to data directory
+ */
+function ensureDataDir(dataDir) {
+  if (!fs.existsSync(dataDir)) {
+    fs.mkdirSync(dataDir, { recursive: true });
+  }
+  // Create subdirectories
+  const modelsDir = path.join(dataDir, 'models');
+  if (!fs.existsSync(modelsDir)) {
+    fs.mkdirSync(modelsDir, { recursive: true });
+  }
+}
+/**
+ * Load configuration from file or create default
+ * @param {string} [configPath] - Optional custom config path
+ * @returns {Object} Configuration object
+ */
+export function loadConfig(configPath) {
+  const config = { ...DEFAULT_CONFIG };
+  // Determine config file path
+  const actualConfigPath = configPath || path.join(config.dataDir, 'config.json');
+  // Ensure data directory exists
+  ensureDataDir(config.dataDir);
+  // Load config from file if it exists
+  if (fs.existsSync(actualConfigPath)) {
+    try {
+      const fileConfig = JSON.parse(fs.readFileSync(actualConfigPath, 'utf-8'));
+      const merged = deepMerge(config, fileConfig);
+      // Ensure data directory from loaded config exists
+      if (merged.dataDir !== config.dataDir) {
+        ensureDataDir(merged.dataDir);
+      }
+      return merged;
+    } catch (error) {
+      console.warn(`Failed to load config from ${actualConfigPath}:`, error.message);
+      console.warn('Using default configuration');
+    }
+  }
+  // Save default config to file if it doesn't exist
+  try {
+    fs.writeFileSync(actualConfigPath, JSON.stringify(config, null, 2), 'utf-8');
+  } catch (error) {
+    console.warn(`Failed to save default config to ${actualConfigPath}:`, error.message);
+  }
+  return config;
+}
+/**
+ * Save configuration to file
+ * @param {Object} config - Configuration object
+ * @param {string} [configPath] - Optional custom config path
+ */
+export function saveConfig(config, configPath) {
+  const actualConfigPath = configPath || path.join(config.dataDir, 'config.json');
+  try {
+    fs.writeFileSync(actualConfigPath, JSON.stringify(config, null, 2), 'utf-8');
+  } catch (error) {
+    throw new Error(`Failed to save config to ${actualConfigPath}: ${error.message}`);
+  }
+}
+/**
+ * Get the database path from config
+ * @param {Object} config - Configuration object
+ * @returns {string} Path to SQLite database
+ */
+export function getDatabasePath(config) {
+  return path.join(config.dataDir, 'memory.db');
+}
+/**
+ * Get the models directory from config
+ * @param {Object} config - Configuration object
+ * @returns {string} Path to models directory
+ */
+export function getModelsPath(config) {
+  return path.join(config.dataDir, 'models');
+}

package/src/embed/index.js ADDED Viewed

@@ -0,0 +1,249 @@
+import { pipeline } from '@xenova/transformers';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import * as logger from '../utils/logger.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+/**
+ * Cached pipeline instance
+ */
+let cachedPipeline = null;
+/**
+ * Model configuration
+ */
+const MODEL_CONFIG = {
+  name: 'Xenova/all-MiniLM-L6-v2',
+  task: 'feature-extraction'
+};
+/**
+ * Initialize the embedding pipeline
+ * Downloads model on first use and caches it
+ * @param {string} modelsPath - Path to models directory
+ * @returns {Promise<Object>} Pipeline instance
+ */
+export async function initializePipeline(modelsPath) {
+  if (cachedPipeline) {
+    logger.debug('Using cached embedding pipeline');
+    return cachedPipeline;
+  }
+  try {
+    logger.info('Initializing embedding model', { model: MODEL_CONFIG.name });
+    // Ensure models directory exists
+    if (!fs.existsSync(modelsPath)) {
+      fs.mkdirSync(modelsPath, { recursive: true });
+    }
+    // Set cache directory for transformers
+    process.env.TRANSFORMERS_CACHE = modelsPath;
+    logger.info('Loading embedding model (this may take a moment on first run)...');
+    // Create pipeline
+    cachedPipeline = await pipeline(
+      MODEL_CONFIG.task,
+      MODEL_CONFIG.name,
+      {
+        quantized: true,
+        progress_callback: (progress) => {
+          if (progress.status === 'downloading') {
+            const percent = progress.progress ? Math.round(progress.progress) : 0;
+            logger.debug(`Downloading model: ${percent}%`);
+          }
+        }
+      }
+    );
+    logger.info('Embedding model loaded successfully');
+    return cachedPipeline;
+  } catch (error) {
+    logger.error('Failed to initialize embedding pipeline', { error: error.message });
+    throw error;
+  }
+}
+/**
+ * Generate embedding for text
+ * @param {string} text - Text to embed
+ * @param {string} modelsPath - Path to models directory
+ * @returns {Promise<Float32Array>} Embedding vector
+ */
+export async function generateEmbedding(text, modelsPath) {
+  try {
+    const pipe = await initializePipeline(modelsPath);
+    logger.debug('Generating embedding', { textLength: text.length });
+    // Generate embedding
+    const result = await pipe(text, {
+      pooling: 'mean',
+      normalize: true
+    });
+    // Extract the embedding array
+    // The result is a tensor, we need to convert it to Float32Array
+    const embedding = new Float32Array(result.data);
+    logger.debug('Embedding generated', { dimensions: embedding.length });
+    return embedding;
+  } catch (error) {
+    logger.error('Failed to generate embedding', { error: error.message });
+    throw error;
+  }
+}
+/**
+ * Generate embeddings for multiple texts in batch
+ * @param {string[]} texts - Array of texts to embed
+ * @param {string} modelsPath - Path to models directory
+ * @returns {Promise<Float32Array[]>} Array of embedding vectors
+ */
+export async function generateEmbeddings(texts, modelsPath) {
+  try {
+    const pipe = await initializePipeline(modelsPath);
+    logger.debug('Generating batch embeddings', { count: texts.length });
+    const embeddings = [];
+    for (const text of texts) {
+      const result = await pipe(text, {
+        pooling: 'mean',
+        normalize: true
+      });
+      embeddings.push(new Float32Array(result.data));
+    }
+    logger.debug('Batch embeddings generated', { count: embeddings.length });
+    return embeddings;
+  } catch (error) {
+    logger.error('Failed to generate batch embeddings', { error: error.message });
+    throw error;
+  }
+}
+/**
+ * Calculate cosine similarity between two embeddings
+ * @param {Float32Array} a - First embedding
+ * @param {Float32Array} b - Second embedding
+ * @returns {number} Similarity score (0-1)
+ */
+export function cosineSimilarity(a, b) {
+  if (a.length !== b.length) {
+    throw new Error('Embeddings must have the same dimensions');
+  }
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+  // Clamp to [0, 1] range (should already be in this range for normalized embeddings)
+  return Math.max(0, Math.min(1, similarity));
+}
+/**
+ * Check if embedding model is available (cached)
+ * @param {string} modelsPath - Path to models directory (may be overridden)
+ * @returns {Object} Object with available flag and actual path
+ */
+export function isModelAvailable(modelsPath) {
+  // First check the provided modelsPath
+  if (fs.existsSync(modelsPath) && fs.readdirSync(modelsPath).length > 0) {
+    return { available: true, path: modelsPath };
+  }
+  // Check Xenova transformers cache in node_modules (most common for local dev)
+  const possiblePaths = [
+    path.resolve(__dirname, '../../node_modules/@xenova/transformers/.cache/Xenova/all-MiniLM-L6-v2'),
+  ];
+  // Add home directory cache paths
+  const homeDir = process.env.HOME || process.env.USERPROFILE;
+  if (homeDir) {
+    possiblePaths.push(path.join(homeDir, '.cache', 'huggingface', 'hub', 'models--Xenova--all-MiniLM-L6-v2'));
+  }
+  for (const cachePath of possiblePaths) {
+    try {
+      if (fs.existsSync(cachePath) && fs.readdirSync(cachePath).length > 0) {
+        return { available: true, path: cachePath };
+      }
+    } catch (e) {
+      // Continue checking other paths
+    }
+  }
+  return { available: false, path: modelsPath };
+}
+/**
+ * Calculate directory size recursively
+ * @param {string} dirPath - Directory path
+ * @returns {number} Size in bytes
+ */
+function getDirectorySize(dirPath) {
+  let size = 0;
+  try {
+    const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(dirPath, entry.name);
+      if (entry.isFile()) {
+        try {
+          const stats = fs.statSync(fullPath);
+          size += stats.size;
+        } catch (e) {
+          // Skip unreadable files
+        }
+      } else if (entry.isDirectory()) {
+        size += getDirectorySize(fullPath);
+      }
+    }
+  } catch (e) {
+    // Return 0 if directory can't be read
+  }
+  return size;
+}
+/**
+ * Get model information
+ * @param {string} modelsPath - Path to models directory
+ * @returns {Object} Model information
+ */
+export function getModelInfo(modelsPath) {
+  const modelCheck = isModelAvailable(modelsPath);
+  const available = modelCheck.available;
+  const actualPath = modelCheck.path;
+  let size = 0;
+  if (available) {
+    size = getDirectorySize(actualPath);
+  }
+  return {
+    name: MODEL_CONFIG.name,
+    task: MODEL_CONFIG.task,
+    available,
+    cached: cachedPipeline !== null,
+    sizeBytes: size,
+    sizeMB: Math.round(size / (1024 * 1024)),
+    path: actualPath
+  };
+}