@yamo/memory-mesh 2.3.2 → 3.0.0

package/lib/lancedb/config.js

@@ -1,215 +0,0 @@
-/**
- * LanceDB Configuration Loader
- * Loads and validates configuration from environment variables
- */
-
-import path from "path";
-
-/**
- * Default configuration values
- */
-const DEFAULTS = {
-  // LanceDB Configuration
-  LANCEDB_URI: './runtime/data/lancedb',
-  LANCEDB_MEMORY_TABLE: 'memory_entries',
-  LANCEDB_MAX_CACHE_SIZE: '2GB',
-
-  // Embedding Model Configuration
-  EMBEDDING_MODEL_TYPE: 'local',
-  EMBEDDING_MODEL_NAME: 'Xenova/all-MiniLM-L6-v2',
-  EMBEDDING_DIMENSION: '384',
-  EMBEDDING_BATCH_SIZE: '32',
-  EMBEDDING_NORMALIZE: 'true',
-
-  // API-based Embeddings
-  OPENAI_EMBEDDING_MODEL: 'text-embedding-3-small',
-
-  // Search Configuration
-  DEFAULT_TOP_K: '10',
-  DEFAULT_SIMILARITY_THRESHOLD: '0.7',
-  ENABLE_HYBRID_SEARCH: 'true',
-  HYBRID_SEARCH_ALPHA: '0.5',
-
-  // Performance Tuning
-  VECTOR_INDEX_TYPE: 'ivf_pq',
-  IVF_PARTITIONS: '256',
-  PQ_BITS: '8',
-  ENABLE_QUERY_CACHE: 'true',
-  QUERY_CACHE_TTL: '300'
-};
-
-/**
- * Memory system configuration defaults
- */
-const MEMORY_DEFAULTS = {
-  // Feature flags
-  MEMORY_ENABLED: 'true',
-  MEMORY_AUTO_CAPTURE: 'true',
-  MEMORY_AUTO_RECALL: 'true',
-
-  // Recall settings
-  MEMORY_MAX_CONTEXT: '5',
-  MEMORY_RELEVANCE_THRESHOLD: '0.7',
-  MEMORY_IMPORTANCE_BOOST: '1.5',
-  MEMORY_RECENCY_WEIGHT: '0.3',
-
-  // Capture settings
-  MEMORY_MIN_IMPORTANCE: '0.3',
-  MEMORY_DEDUP_THRESHOLD: '0.9',
-  MEMORY_CAPTURE_TOOL_RESULTS: 'true',
-  MEMORY_CAPTURE_FILE_OPS: 'true',
-
-  // Retention settings
-  MEMORY_RETENTION_ENABLED: 'true',
-  MEMORY_RETENTION_DAYS: '90',
-  MEMORY_MAX_PER_SESSION: '100',
-  MEMORY_MIN_IMPORTANCE_TO_KEEP: '0.5',
-
-  // Privacy settings
-  MEMORY_REDACT_PII: 'false',
-  MEMORY_ENCRYPTION_ENABLED: 'false',
-};
-
-
-/**
- * Load configuration with validation
- */
-function loadConfig() {
-  const config = {};
-
-
-  for (const [key, defaultValue] of Object.entries(DEFAULTS)) {
-    config[key] = process.env[key] || defaultValue;
-  }
-
-  // Resolve relative paths to absolute
-  if (config.LANCEDB_URI.startsWith('./') || config.LANCEDB_URI.startsWith('../')) {
-    config.LANCEDB_URI = path.resolve(process.cwd(), config.LANCEDB_URI);
-  }
-
-  return config;
-}
-
-/**
- * Load memory-specific configuration
- * @returns {Object} Memory configuration object
- */
-function loadMemoryConfig() {
-  return {
-    enabled: process.env.MEMORY_ENABLED !== 'false',
-    autoCapture: process.env.MEMORY_AUTO_CAPTURE !== 'false',
-    autoRecall: process.env.MEMORY_AUTO_RECALL !== 'false',
-    maxContext: parseInt(process.env.MEMORY_MAX_CONTEXT || '5'),
-    relevanceThreshold: parseFloat(process.env.MEMORY_RELEVANCE_THRESHOLD || '0.7'),
-    importanceBoost: parseFloat(process.env.MEMORY_IMPORTANCE_BOOST || '1.5'),
-    recencyWeight: parseFloat(process.env.MEMORY_RECENCY_WEIGHT || '0.3'),
-    minImportance: parseFloat(process.env.MEMORY_MIN_IMPORTANCE || '0.3'),
-    dedupThreshold: parseFloat(process.env.MEMORY_DEDUP_THRESHOLD || '0.9'),
-    captureToolResults: process.env.MEMORY_CAPTURE_TOOL_RESULTS !== 'false',
-    captureFileOps: process.env.MEMORY_CAPTURE_FILE_OPS !== 'false',
-    retention: {
-      enabled: process.env.MEMORY_RETENTION_ENABLED !== 'false',
-      days: parseInt(process.env.MEMORY_RETENTION_DAYS || '90'),
-      maxPerSession: parseInt(process.env.MEMORY_MAX_PER_SESSION || '100'),
-      minImportanceToKeep: parseFloat(process.env.MEMORY_MIN_IMPORTANCE_TO_KEEP || '0.5'),
-    },
-    privacy: {
-      redactPii: process.env.MEMORY_REDACT_PII === 'true',
-      encryptionEnabled: process.env.MEMORY_ENCRYPTION_ENABLED === 'true',
-    },
-  };
-}
-
-/**
- * Validate configuration
- */
-function validateConfig(config) {
-  const errors = [];
-
-  // Validate embedding model type
-  const validModelTypes = ['local', 'openai', 'cohere', 'voyage'];
-  if (!validModelTypes.includes(config.EMBEDDING_MODEL_TYPE)) {
-    errors.push(`Invalid EMBEDDING_MODEL_TYPE: ${config.EMBEDDING_MODEL_TYPE}`);
-  }
-
-  // Validate numeric values
-  const dimension = parseInt(config.EMBEDDING_DIMENSION);
-  if (isNaN(dimension) || dimension <= 0) {
-    errors.push(`Invalid EMBEDDING_DIMENSION: ${config.EMBEDDING_DIMENSION}`);
-  }
-
-  const topK = parseInt(config.DEFAULT_TOP_K);
-  if (isNaN(topK) || topK <= 0) {
-    errors.push(`Invalid DEFAULT_TOP_K: ${config.DEFAULT_TOP_K}`);
-  }
-
-  // Validate boolean strings
-  const boolFields = ['EMBEDDING_NORMALIZE', 'ENABLE_HYBRID_SEARCH', 'ENABLE_QUERY_CACHE'];
-  for (const field of boolFields) {
-    const value = config[field].toLowerCase();
-    if (value !== 'true' && value !== 'false') {
-      errors.push(`Invalid ${field}: must be 'true' or 'false'`);
-    }
-  }
-
-  // Validate similarity threshold (0-1 range)
-  const threshold = parseFloat(config.DEFAULT_SIMILARITY_THRESHOLD);
-  if (isNaN(threshold) || threshold < 0 || threshold > 1) {
-    errors.push(`Invalid DEFAULT_SIMILARITY_THRESHOLD: must be between 0 and 1`);
-  }
-
-  // Validate hybrid search alpha (0-1 range)
-  const alpha = parseFloat(config.HYBRID_SEARCH_ALPHA);
-  if (isNaN(alpha) || alpha < 0 || alpha > 1) {
-    errors.push(`Invalid HYBRID_SEARCH_ALPHA: must be between 0 and 1`);
-  }
-
-  // Validate positive integers
-  const positiveIntFields = ['EMBEDDING_BATCH_SIZE', 'IVF_PARTITIONS', 'PQ_BITS', 'QUERY_CACHE_TTL'];
-  for (const field of positiveIntFields) {
-    const value = parseInt(config[field]);
-    if (isNaN(value) || value <= 0) {
-      errors.push(`Invalid ${field}: must be a positive integer`);
-    }
-  }
-
-  // Validate cache size format (e.g., "2GB", "500MB")
-  const cacheSizePattern = /^\d+(\.\d+)?(KB|MB|GB|TB)$/;
-  if (!cacheSizePattern.test(config.LANCEDB_MAX_CACHE_SIZE)) {
-    errors.push(`Invalid LANCEDB_MAX_CACHE_SIZE: must match pattern like "2GB", "500MB"`);
-  }
-
-  return errors;
-}
-
-/**
- * Get validated configuration
- */
-function getConfig() {
-  const config = loadConfig();
-  const errors = validateConfig(config);
-
-  if (errors.length > 0) {
-    throw new Error(`Configuration validation failed:\n${errors.join('\n')}`);
-  }
-
-  return config;
-}
-
-export {
-  loadConfig,
-  validateConfig,
-  getConfig,
-  loadMemoryConfig,
-  DEFAULTS,
-  MEMORY_DEFAULTS,
-};
-
-export default {
-  loadConfig,
-  validateConfig,
-  getConfig,
-  loadMemoryConfig,
-  DEFAULTS,
-  MEMORY_DEFAULTS,
-};

package/lib/lancedb/errors.js

@@ -1,144 +0,0 @@
-/**
- * Custom error classes for LanceDB operations
- *
- * Base error class for all LanceDB-related errors. Captures proper stack traces
- * to ensure debugging information points to where errors are thrown, not to the
- * error constructor.
- */
-class LanceDBError extends Error {
-  /**
-   * Create a new LanceDBError
-   * @param {string} message - Human-readable error message
-   * @param {string} code - Machine-readable error code (e.g., 'EMBEDDING_ERROR')
-   * @param {Object} details - Additional error context and metadata
-   */
-  constructor(message, code, details = {}) {
-    super(message);
-    this.name = 'LanceDBError';
-    this.code = code;
-    this.details = details;
-    this.timestamp = new Date().toISOString();
-
-    // Capture stack trace for proper debugging (Node.js best practice)
-    // This ensures stack traces point to where the error was thrown,
-    // not to the error constructor itself
-    Error.captureStackTrace(this, this.constructor);
-  }
-}
-
-/**
- * Error raised when embedding generation or comparison fails
- */
-class EmbeddingError extends LanceDBError {
-  constructor(message, details) {
-    super(message, 'EMBEDDING_ERROR', details);
-    this.name = 'EmbeddingError';
-  }
-}
-
-/**
- * Error raised when storage operations (read/write/delete) fail
- */
-class StorageError extends LanceDBError {
-  constructor(message, details) {
-    super(message, 'STORAGE_ERROR', details);
-    this.name = 'StorageError';
-  }
-}
-
-/**
- * Error raised when database queries fail or return invalid results
- */
-class QueryError extends LanceDBError {
-  constructor(message, details) {
-    super(message, 'QUERY_ERROR', details);
-    this.name = 'QueryError';
-  }
-}
-
-/**
- * Error raised when configuration is missing or invalid
- */
-class ConfigurationError extends LanceDBError {
-  constructor(message, details) {
-    super(message, 'CONFIGURATION_ERROR', details);
-    this.name = 'ConfigurationError';
-  }
-}
-
-/**
- * Sanitize error messages by redacting sensitive information
- * @param {string} message - Error message to sanitize
- * @returns {string} Sanitized error message
- */
-function sanitizeErrorMessage(message) {
-  if (typeof message !== 'string') {
-    return '[Non-string error message]';
-  }
-
-  // Redact common sensitive patterns
-  return message
-    // Redact Bearer tokens
-    .replace(/Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, 'Bearer [REDACTED]')
-    // Redact OpenAI API keys (sk- followed by 32+ chars)
-    .replace(/sk-[A-Za-z0-9]{32,}/g, 'sk-[REDACTED]')
-    // Redact generic API keys (20+ alphanumeric chars after api_key)
-    .replace(/api_key["\s:]+[A-Za-z0-9]{20,}/gi, 'api_key: [REDACTED]')
-    // Redact environment variable patterns that might contain secrets
-    .replace(/(OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY)[="'\s]+[A-Za-z0-9\-_]+/gi, '$1=[REDACTED]')
-    // Redact Authorization headers
-    .replace(/Authorization:\s*[^"\r\n]+/gi, 'Authorization: [REDACTED]')
-    // Redact potential JWT tokens
-    .replace(/eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*/g, '[JWT_REDACTED]');
-}
-
-/**
- * Normalize errors into a consistent response format
- * @param {Error} error - The error to handle
- * @param {Object} context - Additional context about where/when the error occurred
- * @returns {Object} Formatted error response with success: false
- */
-function handleError(error, context = {}) {
-  if (error instanceof LanceDBError) {
-    return {
-      success: false,
-      error: {
-        code: error.code,
-        message: sanitizeErrorMessage(error.message),
-        details: error.details,
-        context
-      }
-    };
-  }
-
-  // Wrap unknown errors
-  return {
-    success: false,
-    error: {
-      code: 'UNKNOWN_ERROR',
-      message: sanitizeErrorMessage(error.message),
-      stack: process.env.NODE_ENV === 'development' ? error.stack : undefined,
-      context
-    }
-  };
-}
-
-export {
-  LanceDBError,
-  EmbeddingError,
-  StorageError,
-  QueryError,
-  ConfigurationError,
-  handleError,
-  sanitizeErrorMessage
-};
-
-export default {
-  LanceDBError,
-  EmbeddingError,
-  StorageError,
-  QueryError,
-  ConfigurationError,
-  handleError,
-  sanitizeErrorMessage
-};

package/lib/lancedb/index.js

@@ -1,4 +0,0 @@
-export { LanceDBClient } from './client.js';
-export { loadConfig, validateConfig, getConfig, DEFAULTS } from './config.js';
-export { MEMORY_SCHEMA, INDEX_CONFIG, createMemoryTable, createMemoryTableWithDimension, createMemorySchema, getEmbeddingDimension, DEFAULT_VECTOR_DIMENSION, EMBEDDING_DIMENSIONS } from './schema.js';
-export { LanceDBError, EmbeddingError, StorageError, QueryError, ConfigurationError, handleError, sanitizeErrorMessage } from './errors.js';

package/lib/lancedb/schema.js

@@ -1,217 +0,0 @@
-/**
- * LanceDB Schema Definitions for MemoryManager
- * Uses Apache Arrow Schema format for LanceDB JavaScript SDK
- *
- * Supports dynamic vector dimensions for different embedding models:
- * - all-MiniLM-L6-v2: 384 dimensions
- * - all-mpnet-base-v2: 768 dimensions
- * - text-embedding-3-small: 1536 dimensions
- */
-
-import * as arrow from "apache-arrow";
-
-/**
- * Default vector dimension (all-MiniLM-L6-v2)
- */
-export const DEFAULT_VECTOR_DIMENSION = 384;
-
-/**
- * Common embedding model dimensions
- */
-export const EMBEDDING_DIMENSIONS = {
-  'Xenova/all-MiniLM-L6-v2': 384,
-  'Xenova/all-mpnet-base-v2': 768,
-  'Xenova/distiluse-base-multilingual-cased-v1': 512,
-  'sentence-transformers/all-MiniLM-L6-v2': 384,
-  'sentence-transformers/all-mpnet-base-v2': 768,
-  'openai/text-embedding-3-small': 1536,
-  'openai/text-embedding-3-large': 3072,
-  'cohere/embed-english-light-v3.0': 1024,
-  'cohere/embed-english-v3.0': 1024,
-};
-
-/**
- * Get dimension for a given embedding model
- * @param {string} modelName - Embedding model name or path
- * @returns {number} Vector dimension
- */
-export function getEmbeddingDimension(modelName) {
-  if (!modelName) return DEFAULT_VECTOR_DIMENSION;
-
-  // Check exact match
-  if (EMBEDDING_DIMENSIONS[modelName]) {
-    return EMBEDDING_DIMENSIONS[modelName];
-  }
-
-  // Check for partial matches
-  for (const [key, dimension] of Object.entries(EMBEDDING_DIMENSIONS)) {
-    if (modelName.toLowerCase().includes(key.toLowerCase())) {
-      return dimension;
-    }
-  }
-
-  // Fallback to default
-  return DEFAULT_VECTOR_DIMENSION;
-}
-
-/**
- * Create a memory schema with a specific vector dimension
- * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
- * @returns {import('apache-arrow').Schema} Arrow schema with specified dimension
- */
-export function createMemorySchema(vectorDim = DEFAULT_VECTOR_DIMENSION) {
-  return new arrow.Schema([
-    new arrow.Field('id', new arrow.Utf8(), false),
-    new arrow.Field('vector',
-      new arrow.FixedSizeList(vectorDim, new arrow.Field('item', new arrow.Float32(), true)),
-      false
-    ),
-    new arrow.Field('content', new arrow.Utf8(), false),
-    new arrow.Field('metadata', new arrow.Utf8(), true),  // Stored as JSON string
-    new arrow.Field('created_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
-    new arrow.Field('updated_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true)
-  ]);
-}
-
-/**
- * Create V2 memory schema with automatic recall fields
- * All new fields are nullable for backward compatibility
- * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
- * @returns {import('apache-arrow').Schema} Arrow schema with V2 fields
- */
-function createMemorySchemaV2(vectorDim = DEFAULT_VECTOR_DIMENSION) {
-  return new arrow.Schema([
-    // ========== V1 Fields (Backward Compatible) ==========
-    new arrow.Field('id', new arrow.Utf8(), false),
-    new arrow.Field('vector',
-      new arrow.FixedSizeList(vectorDim, new arrow.Field('item', new arrow.Float32(), true)),
-      false
-    ),
-    new arrow.Field('content', new arrow.Utf8(), false),
-    new arrow.Field('metadata', new arrow.Utf8(), true),
-    new arrow.Field('created_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
-    new arrow.Field('updated_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
-
-    // ========== V2 Fields (All Nullable) ==========
-    new arrow.Field('session_id', new arrow.Utf8(), true),        // Session association
-    new arrow.Field('agent_id', new arrow.Utf8(), true),          // Agent/skill that created memory
-    new arrow.Field('memory_type', new arrow.Utf8(), true),       // 'global', 'session', 'agent'
-    new arrow.Field('importance_score', new arrow.Float32(), true),   // 0.0-1.0 importance
-    new arrow.Field('access_count', new arrow.Int32(), true),     // Popularity tracking
-    new arrow.Field('last_accessed', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
-  ]);
-}
-
-/**
- * Create schema for synthesized skills (Recursive Skill Synthesis)
- * @param {number} vectorDim - Vector dimension for intent embedding
- * @returns {import('apache-arrow').Schema} Arrow schema
- */
-export function createSynthesizedSkillSchema(vectorDim = DEFAULT_VECTOR_DIMENSION) {
-  return new arrow.Schema([
-    new arrow.Field('id', new arrow.Utf8(), false),
-    new arrow.Field('name', new arrow.Utf8(), false),
-    new arrow.Field('intent', new arrow.Utf8(), false),
-    new arrow.Field('yamo_text', new arrow.Utf8(), false),
-    new arrow.Field('vector',
-      new arrow.FixedSizeList(vectorDim, new arrow.Field('item', new arrow.Float32(), true)),
-      false
-    ),
-    new arrow.Field('metadata', new arrow.Utf8(), true), // Stored as JSON: {reliability, use_count, created_at}
-    new arrow.Field('created_at', new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false)
-  ]);
-}
-
-/**
- * Check if a table is using V2 schema
- * @param {import('apache-arrow').Schema} schema - Table schema to check
- * @returns {boolean} True if V2 schema detected
- */
-function isSchemaV2(schema) {
-  return schema.fields.some(f => f.name === 'session_id');
-}
-
-/**
- * Memory table schema using Apache Arrow format (default 384 dimensions)
- * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
- */
-const MEMORY_SCHEMA = createMemorySchema(DEFAULT_VECTOR_DIMENSION);
-
-/**
- * Index configuration for memory table
- * Indices should be created after data is inserted
- */
-export const INDEX_CONFIG = {
-  vector: {
-    index_type: 'ivf_pq',
-    metric: 'cosine',
-    num_partitions: 256,
-    num_sub_vectors: 8
-  },
-  full_text: {
-    fields: ['content']
-  }
-};
-
-/**
- * Creates a memory table in LanceDB with the predefined schema (384 dimensions)
- * @param {import('@lancedb/lancedb').Connection} db - LanceDB connection
- * @param {string} tableName - Name of the table to create (default: 'memory_entries')
- * @returns {Promise<import('@lancedb/lancedb').Table>} The created or opened table
- * @throws {Error} If table creation fails
- * @deprecated Use createMemoryTableWithDimension() for dynamic dimensions
- */
-async function createMemoryTable(db, tableName = 'memory_entries') {
-  return createMemoryTableWithDimension(db, tableName, DEFAULT_VECTOR_DIMENSION);
-}
-
-/**
- * Creates a memory table in LanceDB with a specific vector dimension
- * @param {import('@lancedb/lancedb').Connection} db - LanceDB connection
- * @param {string} tableName - Name of the table to create
- * @param {number} vectorDim - Vector dimension (384, 768, 1536, etc.)
- * @returns {Promise<import('@lancedb/lancedb').Table>} The created or opened table
- * @throws {Error} If table creation fails
- */
-async function createMemoryTableWithDimension(db, tableName, vectorDim) {
-  try {
-    // Check if table already exists
-    const existingTables = await db.tableNames();
-
-    if (existingTables.includes(tableName)) {
-      return await db.openTable(tableName);
-    }
-
-    // Create schema with specified dimension
-    const schema = createMemorySchema(vectorDim);
-
-    // Create table with schema
-    // LanceDB v0.23.0+ accepts empty array as initial data with schema option
-    const table = await db.createTable(tableName, [], { schema });
-    return table;
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    throw new Error(`Failed to create memory table with dimension ${vectorDim}: ${message}`);
-  }
-}
-
-export {
-  MEMORY_SCHEMA,
-  createMemoryTable,
-  createMemoryTableWithDimension,
-  createMemorySchemaV2,
-  isSchemaV2
-};
-
-export default {
-  MEMORY_SCHEMA,
-  INDEX_CONFIG,
-  createMemoryTable,
-  createMemoryTableWithDimension,
-  createMemorySchema,
-  createMemorySchemaV2,
-  isSchemaV2,
-  getEmbeddingDimension,
-  DEFAULT_VECTOR_DIMENSION,
-  EMBEDDING_DIMENSIONS
-};

package/lib/scrubber/errors/scrubber-error.js

@@ -1,43 +0,0 @@
-/**
- * S-MORA Layer 0 Scrubber Error Classes
- * @module smora/scrubber/errors/scrubber-error
- */
-
-export class ScrubberError extends Error {
-  constructor(message, details = {}) {
-    super(message);
-    this.name = 'ScrubberError';
-    this.details = details;
-    this.timestamp = new Date().toISOString();
-  }
-
-  toJSON() {
-    return {
-      name: this.name,
-      message: this.message,
-      details: this.details,
-      timestamp: this.timestamp
-    };
-  }
-}
-
-export class StructuralCleaningError extends ScrubberError {
-  constructor(message, details = {}) {
-    super(message, details);
-    this.name = 'StructuralCleaningError';
-  }
-}
-
-export class ChunkingError extends ScrubberError {
-  constructor(message, details = {}) {
-    super(message, details);
-    this.name = 'ChunkingError';
-  }
-}
-
-export class ValidationError extends ScrubberError {
-  constructor(message, details = {}) {
-    super(message, details);
-    this.name = 'ValidationError';
-  }
-}