@yamo/memory-mesh 2.0.1

package/lib/lancedb/config.js

@@ -0,0 +1,215 @@
+/**
+ * LanceDB Configuration Loader
+ * Loads and validates configuration from environment variables
+ */
+
+import path from "path";
+
+/**
+ * Default configuration values
+ */
+const DEFAULTS = {
+  // LanceDB Configuration
+  LANCEDB_URI: './runtime/data/lancedb',
+  LANCEDB_MEMORY_TABLE: 'memory_entries',
+  LANCEDB_MAX_CACHE_SIZE: '2GB',
+
+  // Embedding Model Configuration
+  EMBEDDING_MODEL_TYPE: 'local',
+  EMBEDDING_MODEL_NAME: 'Xenova/all-MiniLM-L6-v2',
+  EMBEDDING_DIMENSION: '384',
+  EMBEDDING_BATCH_SIZE: '32',
+  EMBEDDING_NORMALIZE: 'true',
+
+  // API-based Embeddings
+  OPENAI_EMBEDDING_MODEL: 'text-embedding-3-small',
+
+  // Search Configuration
+  DEFAULT_TOP_K: '10',
+  DEFAULT_SIMILARITY_THRESHOLD: '0.7',
+  ENABLE_HYBRID_SEARCH: 'true',
+  HYBRID_SEARCH_ALPHA: '0.5',
+
+  // Performance Tuning
+  VECTOR_INDEX_TYPE: 'ivf_pq',
+  IVF_PARTITIONS: '256',
+  PQ_BITS: '8',
+  ENABLE_QUERY_CACHE: 'true',
+  QUERY_CACHE_TTL: '300'
+};
+
+/**
+ * Memory system configuration defaults
+ */
+const MEMORY_DEFAULTS = {
+  // Feature flags
+  MEMORY_ENABLED: 'true',
+  MEMORY_AUTO_CAPTURE: 'true',
+  MEMORY_AUTO_RECALL: 'true',
+
+  // Recall settings
+  MEMORY_MAX_CONTEXT: '5',
+  MEMORY_RELEVANCE_THRESHOLD: '0.7',
+  MEMORY_IMPORTANCE_BOOST: '1.5',
+  MEMORY_RECENCY_WEIGHT: '0.3',
+
+  // Capture settings
+  MEMORY_MIN_IMPORTANCE: '0.3',
+  MEMORY_DEDUP_THRESHOLD: '0.9',
+  MEMORY_CAPTURE_TOOL_RESULTS: 'true',
+  MEMORY_CAPTURE_FILE_OPS: 'true',
+
+  // Retention settings
+  MEMORY_RETENTION_ENABLED: 'true',
+  MEMORY_RETENTION_DAYS: '90',
+  MEMORY_MAX_PER_SESSION: '100',
+  MEMORY_MIN_IMPORTANCE_TO_KEEP: '0.5',
+
+  // Privacy settings
+  MEMORY_REDACT_PII: 'false',
+  MEMORY_ENCRYPTION_ENABLED: 'false',
+};
+
+
+/**
+ * Load configuration with validation
+ */
+function loadConfig() {
+  const config = {};
+
+
+  for (const [key, defaultValue] of Object.entries(DEFAULTS)) {
+    config[key] = process.env[key] || defaultValue;
+  }
+
+  // Resolve relative paths to absolute
+  if (config.LANCEDB_URI.startsWith('./') || config.LANCEDB_URI.startsWith('../')) {
+    config.LANCEDB_URI = path.resolve(process.cwd(), config.LANCEDB_URI);
+  }
+
+  return config;
+}
+
+/**
+ * Load memory-specific configuration
+ * @returns {Object} Memory configuration object
+ */
+function loadMemoryConfig() {
+  return {
+    enabled: process.env.MEMORY_ENABLED !== 'false',
+    autoCapture: process.env.MEMORY_AUTO_CAPTURE !== 'false',
+    autoRecall: process.env.MEMORY_AUTO_RECALL !== 'false',
+    maxContext: parseInt(process.env.MEMORY_MAX_CONTEXT || '5'),
+    relevanceThreshold: parseFloat(process.env.MEMORY_RELEVANCE_THRESHOLD || '0.7'),
+    importanceBoost: parseFloat(process.env.MEMORY_IMPORTANCE_BOOST || '1.5'),
+    recencyWeight: parseFloat(process.env.MEMORY_RECENCY_WEIGHT || '0.3'),
+    minImportance: parseFloat(process.env.MEMORY_MIN_IMPORTANCE || '0.3'),
+    dedupThreshold: parseFloat(process.env.MEMORY_DEDUP_THRESHOLD || '0.9'),
+    captureToolResults: process.env.MEMORY_CAPTURE_TOOL_RESULTS !== 'false',
+    captureFileOps: process.env.MEMORY_CAPTURE_FILE_OPS !== 'false',
+    retention: {
+      enabled: process.env.MEMORY_RETENTION_ENABLED !== 'false',
+      days: parseInt(process.env.MEMORY_RETENTION_DAYS || '90'),
+      maxPerSession: parseInt(process.env.MEMORY_MAX_PER_SESSION || '100'),
+      minImportanceToKeep: parseFloat(process.env.MEMORY_MIN_IMPORTANCE_TO_KEEP || '0.5'),
+    },
+    privacy: {
+      redactPii: process.env.MEMORY_REDACT_PII === 'true',
+      encryptionEnabled: process.env.MEMORY_ENCRYPTION_ENABLED === 'true',
+    },
+  };
+}
+
+/**
+ * Validate configuration
+ */
+function validateConfig(config) {
+  const errors = [];
+
+  // Validate embedding model type
+  const validModelTypes = ['local', 'openai', 'cohere', 'voyage'];
+  if (!validModelTypes.includes(config.EMBEDDING_MODEL_TYPE)) {
+    errors.push(`Invalid EMBEDDING_MODEL_TYPE: ${config.EMBEDDING_MODEL_TYPE}`);
+  }
+
+  // Validate numeric values
+  const dimension = parseInt(config.EMBEDDING_DIMENSION);
+  if (isNaN(dimension) || dimension <= 0) {
+    errors.push(`Invalid EMBEDDING_DIMENSION: ${config.EMBEDDING_DIMENSION}`);
+  }
+
+  const topK = parseInt(config.DEFAULT_TOP_K);
+  if (isNaN(topK) || topK <= 0) {
+    errors.push(`Invalid DEFAULT_TOP_K: ${config.DEFAULT_TOP_K}`);
+  }
+
+  // Validate boolean strings
+  const boolFields = ['EMBEDDING_NORMALIZE', 'ENABLE_HYBRID_SEARCH', 'ENABLE_QUERY_CACHE'];
+  for (const field of boolFields) {
+    const value = config[field].toLowerCase();
+    if (value !== 'true' && value !== 'false') {
+      errors.push(`Invalid ${field}: must be 'true' or 'false'`);
+    }
+  }
+
+  // Validate similarity threshold (0-1 range)
+  const threshold = parseFloat(config.DEFAULT_SIMILARITY_THRESHOLD);
+  if (isNaN(threshold) || threshold < 0 || threshold > 1) {
+    errors.push(`Invalid DEFAULT_SIMILARITY_THRESHOLD: must be between 0 and 1`);
+  }
+
+  // Validate hybrid search alpha (0-1 range)
+  const alpha = parseFloat(config.HYBRID_SEARCH_ALPHA);
+  if (isNaN(alpha) || alpha < 0 || alpha > 1) {
+    errors.push(`Invalid HYBRID_SEARCH_ALPHA: must be between 0 and 1`);
+  }
+
+  // Validate positive integers
+  const positiveIntFields = ['EMBEDDING_BATCH_SIZE', 'IVF_PARTITIONS', 'PQ_BITS', 'QUERY_CACHE_TTL'];
+  for (const field of positiveIntFields) {
+    const value = parseInt(config[field]);
+    if (isNaN(value) || value <= 0) {
+      errors.push(`Invalid ${field}: must be a positive integer`);
+    }
+  }
+
+  // Validate cache size format (e.g., "2GB", "500MB")
+  const cacheSizePattern = /^\d+(\.\d+)?(KB|MB|GB|TB)$/;
+  if (!cacheSizePattern.test(config.LANCEDB_MAX_CACHE_SIZE)) {
+    errors.push(`Invalid LANCEDB_MAX_CACHE_SIZE: must match pattern like "2GB", "500MB"`);
+  }
+
+  return errors;
+}
+
+/**
+ * Get validated configuration
+ */
+function getConfig() {
+  const config = loadConfig();
+  const errors = validateConfig(config);
+
+  if (errors.length > 0) {
+    throw new Error(`Configuration validation failed:\n${errors.join('\n')}`);
+  }
+
+  return config;
+}
+
+export {
+  loadConfig,
+  validateConfig,
+  getConfig,
+  loadMemoryConfig,
+  DEFAULTS,
+  MEMORY_DEFAULTS,
+};
+
+export default {
+  loadConfig,
+  validateConfig,
+  getConfig,
+  loadMemoryConfig,
+  DEFAULTS,
+  MEMORY_DEFAULTS,
+};

package/lib/lancedb/errors.js

@@ -0,0 +1,144 @@
+/**
+ * Custom error classes for LanceDB operations
+ *
+ * Base error class for all LanceDB-related errors. Captures proper stack traces
+ * to ensure debugging information points to where errors are thrown, not to the
+ * error constructor.
+ */
+class LanceDBError extends Error {
+  /**
+   * Create a new LanceDBError
+   * @param {string} message - Human-readable error message
+   * @param {string} code - Machine-readable error code (e.g., 'EMBEDDING_ERROR')
+   * @param {Object} details - Additional error context and metadata
+   */
+  constructor(message, code, details = {}) {
+    super(message);
+    this.name = 'LanceDBError';
+    this.code = code;
+    this.details = details;
+    this.timestamp = new Date().toISOString();
+
+    // Capture stack trace for proper debugging (Node.js best practice)
+    // This ensures stack traces point to where the error was thrown,
+    // not to the error constructor itself
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+/**
+ * Error raised when embedding generation or comparison fails
+ */
+class EmbeddingError extends LanceDBError {
+  constructor(message, details) {
+    super(message, 'EMBEDDING_ERROR', details);
+    this.name = 'EmbeddingError';
+  }
+}
+
+/**
+ * Error raised when storage operations (read/write/delete) fail
+ */
+class StorageError extends LanceDBError {
+  constructor(message, details) {
+    super(message, 'STORAGE_ERROR', details);
+    this.name = 'StorageError';
+  }
+}
+
+/**
+ * Error raised when database queries fail or return invalid results
+ */
+class QueryError extends LanceDBError {
+  constructor(message, details) {
+    super(message, 'QUERY_ERROR', details);
+    this.name = 'QueryError';
+  }
+}
+
+/**
+ * Error raised when configuration is missing or invalid
+ */
+class ConfigurationError extends LanceDBError {
+  constructor(message, details) {
+    super(message, 'CONFIGURATION_ERROR', details);
+    this.name = 'ConfigurationError';
+  }
+}
+
+/**
+ * Sanitize error messages by redacting sensitive information
+ * @param {string} message - Error message to sanitize
+ * @returns {string} Sanitized error message
+ */
+function sanitizeErrorMessage(message) {
+  if (typeof message !== 'string') {
+    return '[Non-string error message]';
+  }
+
+  // Redact common sensitive patterns
+  return message
+    // Redact Bearer tokens
+    .replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, 'Bearer [REDACTED]')
+    // Redact OpenAI API keys (sk- followed by 32+ chars)
+    .replace(/sk-[A-Za-z0-9]{32,}/g, 'sk-[REDACTED]')
+    // Redact generic API keys (20+ alphanumeric chars after api_key)
+    .replace(/api_key["\s:]+[A-Za-z0-9]{20,}/gi, 'api_key: [REDACTED]')
+    // Redact environment variable patterns that might contain secrets
+    .replace(/(OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY)[="'\s]+[A-Za-z0-9\-_]+/gi, '$1=[REDACTED]')
+    // Redact Authorization headers
+    .replace(/Authorization:\s*[^"\r\n]+/gi, 'Authorization: [REDACTED]')
+    // Redact potential JWT tokens
+    .replace(/eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g, '[JWT_REDACTED]');
+}
+
+/**
+ * Normalize errors into a consistent response format
+ * @param {Error} error - The error to handle
+ * @param {Object} context - Additional context about where/when the error occurred
+ * @returns {Object} Formatted error response with success: false
+ */
+function handleError(error, context = {}) {
+  if (error instanceof LanceDBError) {
+    return {
+      success: false,
+      error: {
+        code: error.code,
+        message: sanitizeErrorMessage(error.message),
+        details: error.details,
+        context
+      }
+    };
+  }
+
+  // Wrap unknown errors
+  return {
+    success: false,
+    error: {
+      code: 'UNKNOWN_ERROR',
+      message: sanitizeErrorMessage(error.message),
+      stack: process.env.NODE_ENV === 'development' ? error.stack : undefined,
+      context
+    }
+  };
+}
+
+export {
+  LanceDBError,
+  EmbeddingError,
+  StorageError,
+  QueryError,
+  ConfigurationError,
+  handleError,
+  sanitizeErrorMessage
+};
+
+export default {
+  LanceDBError,
+  EmbeddingError,
+  StorageError,
+  QueryError,
+  ConfigurationError,
+  handleError,
+  sanitizeErrorMessage
+};

package/lib/lancedb/index.js

@@ -0,0 +1,4 @@
+export { LanceDBClient } from './client.js';
+export { loadConfig, validateConfig, getConfig, DEFAULTS } from './config.js';
+export { MEMORY_SCHEMA, INDEX_CONFIG, createMemoryTable, createMemoryTableWithDimension, createMemorySchema, getEmbeddingDimension, DEFAULT_VECTOR_DIMENSION, EMBEDDING_DIMENSIONS } from './schema.js';
+export { LanceDBError, EmbeddingError, StorageError, QueryError, ConfigurationError, handleError, sanitizeErrorMessage } from './errors.js';

package/lib/lancedb/schema.js

@@ -0,0 +1,197 @@
+/**
+ * LanceDB Schema Definitions for MemoryManager
+ * Uses Apache Arrow Schema format for LanceDB JavaScript SDK
+ *
+ * Supports dynamic vector dimensions for different embedding models:
+ * - all-MiniLM-L6-v2: 384 dimensions
+ * - all-mpnet-base-v2: 768 dimensions
+ * - text-embedding-3-small: 1536 dimensions
+ */
+
+import * as arrow from "apache-arrow";
+
+/**
+ * Default vector dimension (all-MiniLM-L6-v2)
+ */
+export const DEFAULT_VECTOR_DIMENSION = 384;
+
+/**
+ * Common embedding model dimensions
+ */
+export const EMBEDDING_DIMENSIONS = {
+  'Xenova/all-MiniLM-L6-v2': 384,
+  'Xenova/all-mpnet-base-v2': 768,
+  'Xenova/distiluse-base-multilingual-cased-v1': 512,
+  'sentence-transformers/all-MiniLM-L6-v2': 384,
+  'sentence-transformers/all-mpnet-base-v2': 768,
+  'openai/text-embedding-3-small': 1536,
+  'openai/text-embedding-3-large': 3072,
+  'cohere/embed-english-light-v3.0': 1024,
+  'cohere/embed-english-v3.0': 1024,
+};
+
+/**
+ * Get dimension for a given embedding model
+ * @param {string} modelName - Embedding model name or path
+ * @returns {number} Vector dimension
+ */
+export function getEmbeddingDimension(modelName) {
+  if (!modelName) return DEFAULT_VECTOR_DIMENSION;
+
+  // Check exact match
+  if (EMBEDDING_DIMENSIONS[modelName]) {
+    return EMBEDDING_DIMENSIONS[modelName];
+  }
+
+  // Check for partial matches
+  for (const [key, dimension] of Object.entries(EMBEDDING_DIMENSIONS)) {
+    if (modelName.toLowerCase().includes(key.toLowerCase())) {
+      return dimension;
+    }
+  }
+
+  // Fallback to default
+  return DEFAULT_VECTOR_DIMENSION;
+}
+
+/**
+ * Create a memory schema with a specific vector dimension
+ * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
+ * @returns {import('apache-arrow').Schema} Arrow schema with specified dimension
+ */
+export function createMemorySchema(vectorDim = DEFAULT_VECTOR_DIMENSION) {
+  return new arrow.Schema([
+    new arrow.Field('id', new arrow.Utf8(), false),
+    new arrow.Field('vector',
+      new arrow.FixedSizeList(vectorDim, new arrow.Field('item', new arrow.Float32(), true)),
+      false
+    ),
+    new arrow.Field('content', new arrow.Utf8(), false),
+    new arrow.Field('metadata', new arrow.Utf8(), true),  // Stored as JSON string
+    new arrow.Field('created_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
+    new arrow.Field('updated_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true)
+  ]);
+}
+
+/**
+ * Create V2 memory schema with automatic recall fields
+ * All new fields are nullable for backward compatibility
+ * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
+ * @returns {import('apache-arrow').Schema} Arrow schema with V2 fields
+ */
+function createMemorySchemaV2(vectorDim = DEFAULT_VECTOR_DIMENSION) {
+  return new arrow.Schema([
+    // ========== V1 Fields (Backward Compatible) ==========
+    new arrow.Field('id', new arrow.Utf8(), false),
+    new arrow.Field('vector',
+      new arrow.FixedSizeList(vectorDim, new arrow.Field('item', new arrow.Float32(), true)),
+      false
+    ),
+    new arrow.Field('content', new arrow.Utf8(), false),
+    new arrow.Field('metadata', new arrow.Utf8(), true),
+    new arrow.Field('created_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
+    new arrow.Field('updated_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
+
+    // ========== V2 Fields (All Nullable) ==========
+    new arrow.Field('session_id', new arrow.Utf8(), true),        // Session association
+    new arrow.Field('agent_id', new arrow.Utf8(), true),          // Agent/skill that created memory
+    new arrow.Field('memory_type', new arrow.Utf8(), true),       // 'global', 'session', 'agent'
+    new arrow.Field('importance_score', new arrow.Float32(), true),   // 0.0-1.0 importance
+    new arrow.Field('access_count', new arrow.Int32(), true),     // Popularity tracking
+    new arrow.Field('last_accessed', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
+  ]);
+}
+
+/**
+ * Check if a table is using V2 schema
+ * @param {import('apache-arrow').Schema} schema - Table schema to check
+ * @returns {boolean} True if V2 schema detected
+ */
+function isSchemaV2(schema) {
+  return schema.fields.some(f => f.name === 'session_id');
+}
+
+/**
+ * Memory table schema using Apache Arrow format (default 384 dimensions)
+ * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
+ */
+const MEMORY_SCHEMA = createMemorySchema(DEFAULT_VECTOR_DIMENSION);
+
+/**
+ * Index configuration for memory table
+ * Indices should be created after data is inserted
+ */
+export const INDEX_CONFIG = {
+  vector: {
+    index_type: 'ivf_pq',
+    metric: 'cosine',
+    num_partitions: 256,
+    num_sub_vectors: 8
+  },
+  full_text: {
+    fields: ['content']
+  }
+};
+
+/**
+ * Creates a memory table in LanceDB with the predefined schema (384 dimensions)
+ * @param {import('@lancedb/lancedb').Connection} db - LanceDB connection
+ * @param {string} tableName - Name of the table to create (default: 'memory_entries')
+ * @returns {Promise<import('@lancedb/lancedb').Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ * @deprecated Use createMemoryTableWithDimension() for dynamic dimensions
+ */
+async function createMemoryTable(db, tableName = 'memory_entries') {
+  return createMemoryTableWithDimension(db, tableName, DEFAULT_VECTOR_DIMENSION);
+}
+
+/**
+ * Creates a memory table in LanceDB with a specific vector dimension
+ * @param {import('@lancedb/lancedb').Connection} db - LanceDB connection
+ * @param {string} tableName - Name of the table to create
+ * @param {number} vectorDim - Vector dimension (384, 768, 1536, etc.)
+ * @returns {Promise<import('@lancedb/lancedb').Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ */
+async function createMemoryTableWithDimension(db, tableName, vectorDim) {
+  try {
+    // Check if table already exists
+    const existingTables = await db.tableNames();
+
+    if (existingTables.includes(tableName)) {
+      return await db.openTable(tableName);
+    }
+
+    // Create schema with specified dimension
+    const schema = createMemorySchema(vectorDim);
+
+    // Create table with schema
+    // LanceDB v0.23.0+ accepts empty array as initial data with schema option
+    const table = await db.createTable(tableName, [], { schema });
+    return table;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(`Failed to create memory table with dimension ${vectorDim}: ${message}`);
+  }
+}
+
+export {
+  MEMORY_SCHEMA,
+  createMemoryTable,
+  createMemoryTableWithDimension,
+  createMemorySchemaV2,
+  isSchemaV2
+};
+
+export default {
+  MEMORY_SCHEMA,
+  INDEX_CONFIG,
+  createMemoryTable,
+  createMemoryTableWithDimension,
+  createMemorySchema,
+  createMemorySchemaV2,
+  isSchemaV2,
+  getEmbeddingDimension,
+  DEFAULT_VECTOR_DIMENSION,
+  EMBEDDING_DIMENSIONS
+};

package/lib/memory/index.js

@@ -0,0 +1,3 @@
+export { default as VectorMemory } from './vector-memory.js';
+export { default as MemoryMesh } from './memory-mesh.js';
+export { default as MigrateMemory } from './migrate-memory.js';