@iflow-mcp/joleyline-mcp-memory-libsql 0.0.14

package/dist/db/migrations/fix-vector-dimension.js

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+import { DatabaseManager } from '../core.js';
+import { databaseConfig } from '../../config/index.js';
+import { logger } from '../../utils/logger.js';
+import { EMBEDDING_DIMENSION } from '../../services/embedding-service.js';
+/**
+ * Migration script to fix the database schema to support the correct vector dimensions
+ * This script will:
+ * 1. Disable foreign key constraints
+ * 2. Create backups of all tables (entities, observations, relations)
+ * 3. Drop the existing tables in the correct order
+ * 4. Create new tables with the updated vector dimension
+ * 5. Restore the data from the backup tables (excluding incompatible embeddings)
+ * 6. Re-enable foreign key constraints
+ */
+async function fixVectorDimension() {
+    logger.info('Starting vector dimension fix...');
+    // Get database connection
+    const config = databaseConfig;
+    const dbManager = await DatabaseManager.getInstance(config);
+    const client = dbManager.getClient();
+    try {
+        // Start transaction
+        const txn = await client.transaction('write');
+        try {
+            // 1. Disable foreign key constraints
+            logger.info('Disabling foreign key constraints...');
+            await txn.execute({
+                sql: `PRAGMA foreign_keys = OFF;`
+            });
+            // 2. Check if backup tables exist and drop them if they do
+            logger.info('Checking for existing backup tables...');
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS entities_backup`
+            });
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS observations_backup`
+            });
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS relations_backup`
+            });
+            // 3. Create backups of all tables
+            logger.info('Creating backup of entities table...');
+            await txn.execute({
+                sql: `CREATE TABLE entities_backup AS SELECT * FROM entities`
+            });
+            logger.info('Creating backup of observations table...');
+            await txn.execute({
+                sql: `CREATE TABLE observations_backup AS SELECT * FROM observations`
+            });
+            logger.info('Creating backup of relations table...');
+            await txn.execute({
+                sql: `CREATE TABLE relations_backup AS SELECT * FROM relations`
+            });
+            // 4. Drop existing tables in the correct order (respecting foreign key relationships)
+            logger.info('Dropping existing tables...');
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS observations`
+            });
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS relations`
+            });
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS entities`
+            });
+            // 5. Create new tables with updated vector dimension
+            logger.info(`Creating new entities table with ${EMBEDDING_DIMENSION} dimensions...`);
+            await txn.execute({
+                sql: `
+                    CREATE TABLE entities (
+                        name TEXT PRIMARY KEY,
+                        entity_type TEXT NOT NULL,
+                        embedding F32_BLOB(${EMBEDDING_DIMENSION}), -- Updated dimension
+                        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+                    )
+                `
+            });
+            logger.info('Creating new observations table...');
+            await txn.execute({
+                sql: `
+                    CREATE TABLE observations (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        entity_name TEXT NOT NULL,
+                        content TEXT NOT NULL,
+                        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+                        FOREIGN KEY (entity_name) REFERENCES entities(name)
+                    )
+                `
+            });
+            logger.info('Creating new relations table...');
+            await txn.execute({
+                sql: `
+                    CREATE TABLE relations (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        source TEXT NOT NULL,
+                        target TEXT NOT NULL,
+                        relation_type TEXT NOT NULL,
+                        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+                        FOREIGN KEY (source) REFERENCES entities(name),
+                        FOREIGN KEY (target) REFERENCES entities(name)
+                    )
+                `
+            });
+            // 6. Restore data from backups (excluding embeddings as they're incompatible)
+            logger.info('Restoring entities data from backup (without embeddings)...');
+            await txn.execute({
+                sql: `
+                    INSERT INTO entities (name, entity_type, created_at)
+                    SELECT name, entity_type, created_at FROM entities_backup
+                `
+            });
+            logger.info('Restoring observations data from backup...');
+            await txn.execute({
+                sql: `
+                    INSERT INTO observations (entity_name, content, created_at)
+                    SELECT entity_name, content, created_at FROM observations_backup
+                `
+            });
+            logger.info('Restoring relations data from backup...');
+            await txn.execute({
+                sql: `
+                    INSERT INTO relations (source, target, relation_type, created_at)
+                    SELECT source, target, relation_type, created_at FROM relations_backup
+                `
+            });
+            // 7. Recreate indexes
+            logger.info('Recreating indexes...');
+            await txn.execute({
+                sql: 'CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name)'
+            });
+            await txn.execute({
+                sql: 'CREATE INDEX IF NOT EXISTS idx_observations_entity ON observations(entity_name)'
+            });
+            await txn.execute({
+                sql: 'CREATE INDEX IF NOT EXISTS idx_relations_source ON relations(source)'
+            });
+            await txn.execute({
+                sql: 'CREATE INDEX IF NOT EXISTS idx_relations_target ON relations(target)'
+            });
+            await txn.execute({
+                sql: 'CREATE INDEX IF NOT EXISTS idx_entities_embedding ON entities(libsql_vector_idx(embedding))'
+            });
+            // 8. Re-enable foreign key constraints
+            logger.info('Re-enabling foreign key constraints...');
+            await txn.execute({
+                sql: `PRAGMA foreign_keys = ON;`
+            });
+            // 9. Drop backup tables
+            logger.info('Dropping backup tables...');
+            await txn.execute({
+                sql: `DROP TABLE entities_backup`
+            });
+            await txn.execute({
+                sql: `DROP TABLE observations_backup`
+            });
+            await txn.execute({
+                sql: `DROP TABLE relations_backup`
+            });
+            // Commit transaction
+            await txn.commit();
+            logger.info('Vector dimension fix completed successfully!');
+        }
+        catch (error) {
+            // Rollback transaction on error
+            await txn.rollback();
+            throw error;
+        }
+    }
+    catch (error) {
+        logger.error('Migration failed:', error);
+        throw new Error(`Vector dimension fix failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    finally {
+        // Close database connection
+        await dbManager.close();
+    }
+}
+// Run migration
+fixVectorDimension()
+    .then(() => {
+    logger.info('Migration completed successfully');
+    process.exit(0);
+})
+    .catch((error) => {
+    logger.error('Migration failed:', error);
+    process.exit(1);
+});

package/dist/db/migrations/regenerate-embeddings-after-migration.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+import { DatabaseManager } from '../index.js';
+import { databaseConfig } from '../../config/index.js';
+import { logger } from '../../utils/logger.js';
+import { embeddingService } from '../../services/embedding-service.js';
+/**
+ * Regenerates embeddings for all entities in the database after migration
+ * This script should be run after the fix-vector-dimension.ts migration
+ * to ensure all entities have embeddings with the correct dimension
+ */
+async function regenerateEmbeddingsAfterMigration() {
+    logger.info('Starting embedding regeneration after migration...');
+    // Get database connection
+    const config = databaseConfig;
+    const dbManager = await DatabaseManager.get_instance(config);
+    try {
+        // Get all entities from the database
+        logger.info('Retrieving all entities from the database...');
+        const entities = await dbManager.get_recent_entities(1000); // Set a high limit to get all entities
+        const totalEntities = entities.length;
+        logger.info(`Found ${totalEntities} entities to process`);
+        // Process each entity
+        let successCount = 0;
+        let errorCount = 0;
+        for (let i = 0; i < totalEntities; i++) {
+            const entity = entities[i];
+            const entityName = entity.name;
+            const entityType = entity.entityType;
+            try {
+                logger.info(`Processing entity ${i + 1}/${totalEntities}: ${entityName} (${entityType})`);
+                // Get observations from the entity
+                const observations = entity.observations;
+                if (!observations || observations.length === 0) {
+                    logger.warn(`Entity "${entityName}" has no observations, skipping embedding generation`);
+                    continue;
+                }
+                // Generate embedding from observations
+                logger.info(`Generating embedding for entity "${entityName}" using ${embeddingService.getModelName()}`);
+                const text = observations.join(' ');
+                const embedding = await embeddingService.generateEmbedding(text);
+                logger.info(`Successfully generated embedding with dimension: ${embedding.length}`);
+                // Update entity with new embedding
+                await dbManager.create_entities([{
+                        name: entityName,
+                        entityType: entityType,
+                        observations: observations,
+                        embedding: embedding
+                    }]);
+                logger.info(`Successfully updated embedding for entity "${entityName}"`);
+                successCount++;
+            }
+            catch (error) {
+                logger.error(`Error processing entity "${entityName}":`, error);
+                errorCount++;
+            }
+            // Log progress every 10 entities
+            if ((i + 1) % 10 === 0 || i === totalEntities - 1) {
+                logger.info(`Progress: ${i + 1}/${totalEntities} entities processed (${successCount} succeeded, ${errorCount} failed)`);
+            }
+        }
+        logger.info(`Embedding regeneration complete. ${successCount} entities updated successfully, ${errorCount} entities failed.`);
+    }
+    catch (error) {
+        logger.error('Error during embedding regeneration:', error);
+        throw error;
+    }
+    finally {
+        // Close database connection
+        await dbManager.close();
+    }
+}
+// Run the regeneration
+regenerateEmbeddingsAfterMigration()
+    .then(() => {
+    logger.info('Embedding regeneration process completed successfully');
+    process.exit(0);
+})
+    .catch((error) => {
+    logger.error('Embedding regeneration process failed:', error);
+    process.exit(1);
+});

package/dist/db/migrations/run.js

@@ -0,0 +1,30 @@
+import { DatabaseManager } from '../index.js';
+import { databaseConfig } from '../../config/index.js';
+import { schema } from './schema.js';
+async function run_migrations() {
+    const config = databaseConfig;
+    const db_manager = await DatabaseManager.get_instance(config);
+    const db = db_manager.get_client();
+    try {
+        console.log('Starting migrations...');
+        for (const migration of schema) {
+            console.log(`Executing: ${migration.slice(0, 50)}...`);
+            await db.execute(migration);
+        }
+        console.log('Migrations completed successfully');
+    }
+    catch (error) {
+        console.error('Error running migrations:', error);
+        throw error;
+    }
+}
+// Run migrations if this file is executed directly
+if (require.main === module) {
+    run_migrations()
+        .then(() => process.exit(0))
+        .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });
+}
+export { run_migrations };

package/dist/db/migrations/schema.js

@@ -0,0 +1,34 @@
+export const schema = [
+    // Create entities table
+    `CREATE TABLE IF NOT EXISTS entities (
+    name TEXT PRIMARY KEY,
+    entity_type TEXT NOT NULL,
+    embedding F32_BLOB(384), -- 4 dimensions for testing
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+  )`,
+    // Create observations table
+    `CREATE TABLE IF NOT EXISTS observations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    entity_name TEXT NOT NULL,
+    content TEXT NOT NULL,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (entity_name) REFERENCES entities(name)
+  )`,
+    // Create relations table
+    `CREATE TABLE IF NOT EXISTS relations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    target TEXT NOT NULL,
+    relation_type TEXT NOT NULL,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (source) REFERENCES entities(name),
+    FOREIGN KEY (target) REFERENCES entities(name)
+  )`,
+    // Create indexes
+    `CREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name)`,
+    `CREATE INDEX IF NOT EXISTS idx_observations_entity ON observations(entity_name)`,
+    `CREATE INDEX IF NOT EXISTS idx_relations_source ON relations(source)`,
+    `CREATE INDEX IF NOT EXISTS idx_relations_target ON relations(target)`,
+    // Create vector index for similarity search
+    `CREATE INDEX IF NOT EXISTS idx_entities_embedding ON entities(libsql_vector_idx(embedding))`,
+];

package/dist/db/migrations/update-vector-dimension.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+import { DatabaseManager } from '../core.js';
+import { databaseConfig } from '../../config/index.js';
+import { logger } from '../../utils/logger.js';
+import { EMBEDDING_DIMENSION } from '../../services/embedding-service.js';
+/**
+ * Migration script to update the database schema to support larger vector dimensions
+ * This script will:
+ * 1. Create a backup of the entities table
+ * 2. Drop the existing entities table
+ * 3. Create a new entities table with the updated vector dimension
+ * 4. Restore the data from the backup table
+ * 5. Drop the backup table
+ */
+async function migrateVectorDimension() {
+    logger.info('Starting vector dimension migration...');
+    // Get database connection
+    const config = databaseConfig;
+    const dbManager = await DatabaseManager.getInstance(config);
+    const client = dbManager.getClient();
+    try {
+        // Start transaction
+        const txn = await client.transaction('write');
+        try {
+            // 1. Check if backup table exists and drop it if it does
+            logger.info('Checking for existing backup table...');
+            await txn.execute({
+                sql: `DROP TABLE IF EXISTS entities_backup`
+            });
+            // 2. Create backup of entities table
+            logger.info('Creating backup of entities table...');
+            await txn.execute({
+                sql: `CREATE TABLE entities_backup AS SELECT * FROM entities`
+            });
+            // 3. Drop existing entities table
+            logger.info('Dropping existing entities table...');
+            await txn.execute({
+                sql: `DROP TABLE entities`
+            });
+            // 4. Create new entities table with updated vector dimension
+            logger.info(`Creating new entities table with ${EMBEDDING_DIMENSION} dimensions...`);
+            await txn.execute({
+                sql: `
+                    CREATE TABLE entities (
+                        name TEXT PRIMARY KEY,
+                        entity_type TEXT NOT NULL,
+                        embedding F32_BLOB(${EMBEDDING_DIMENSION}), -- Updated dimension
+                        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+                    )
+                `
+            });
+            // 5. Restore data from backup (excluding embeddings as they're incompatible)
+            logger.info('Restoring data from backup (without embeddings)...');
+            await txn.execute({
+                sql: `
+                    INSERT INTO entities (name, entity_type, created_at)
+                    SELECT name, entity_type, created_at FROM entities_backup
+                `
+            });
+            // 6. Recreate index
+            logger.info('Recreating vector index...');
+            await txn.execute({
+                sql: 'CREATE INDEX IF NOT EXISTS idx_entities_embedding ON entities(libsql_vector_idx(embedding))'
+            });
+            // 7. Drop backup table
+            logger.info('Dropping backup table...');
+            await txn.execute({
+                sql: `DROP TABLE entities_backup`
+            });
+            // Commit transaction
+            await txn.commit();
+            logger.info('Vector dimension migration completed successfully!');
+        }
+        catch (error) {
+            // Rollback transaction on error
+            await txn.rollback();
+            throw error;
+        }
+    }
+    catch (error) {
+        logger.error('Migration failed:', error);
+        throw new Error(`Vector dimension migration failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    finally {
+        // Close database connection
+        await dbManager.close();
+    }
+}
+// Run migration
+migrateVectorDimension()
+    .then(() => {
+    logger.info('Migration completed successfully');
+    process.exit(0);
+})
+    .catch((error) => {
+    logger.error('Migration failed:', error);
+    process.exit(1);
+});

package/dist/db/regenerate-embeddings.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+import { DatabaseManager } from './index.js';
+import { databaseConfig } from '../config/index.js';
+import { logger } from '../utils/logger.js';
+import { embeddingService } from '../services/embedding-service.js';
+/**
+ * Regenerates embeddings for all entities in the database
+ * This is useful when:
+ * 1. Migrating from a previous version without embeddings
+ * 2. Changing the embedding model
+ * 3. Fixing corrupted embeddings
+ */
+async function regenerateAllEmbeddings() {
+    logger.info('Starting embedding regeneration for all entities...');
+    // Get database connection
+    const config = databaseConfig;
+    const dbManager = await DatabaseManager.get_instance(config);
+    try {
+        // Get all entities from the database using get_recent_entities with a high limit
+        logger.info('Retrieving all entities from the database...');
+        const entities = await dbManager.get_recent_entities(1000); // Set a high limit to get all entities
+        const totalEntities = entities.length;
+        logger.info(`Found ${totalEntities} entities to process`);
+        // Process each entity
+        let successCount = 0;
+        let errorCount = 0;
+        for (let i = 0; i < totalEntities; i++) {
+            const entity = entities[i];
+            const entityName = entity.name;
+            const entityType = entity.entityType;
+            try {
+                logger.info(`Processing entity ${i + 1}/${totalEntities}: ${entityName} (${entityType})`);
+                // Get observations from the entity
+                const observations = entity.observations;
+                if (!observations || observations.length === 0) {
+                    logger.warn(`Entity "${entityName}" has no observations, skipping embedding generation`);
+                    continue;
+                }
+                // Generate embedding from observations
+                logger.info(`Generating embedding for entity "${entityName}" using ${embeddingService.getModelName()}`);
+                const text = observations.join(' ');
+                const embedding = await embeddingService.generateEmbedding(text);
+                logger.info(`Successfully generated embedding with dimension: ${embedding.length}`);
+                // Update entity with new embedding
+                await dbManager.create_entities([{
+                        name: entityName,
+                        entityType: entityType,
+                        observations: observations,
+                        embedding: embedding
+                    }]);
+                logger.info(`Successfully updated embedding for entity "${entityName}"`);
+                successCount++;
+            }
+            catch (error) {
+                logger.error(`Error processing entity "${entityName}":`, error);
+                errorCount++;
+            }
+            // Log progress every 10 entities
+            if ((i + 1) % 10 === 0 || i === totalEntities - 1) {
+                logger.info(`Progress: ${i + 1}/${totalEntities} entities processed (${successCount} succeeded, ${errorCount} failed)`);
+            }
+        }
+        logger.info(`Embedding regeneration complete. ${successCount} entities updated successfully, ${errorCount} entities failed.`);
+    }
+    catch (error) {
+        logger.error('Error during embedding regeneration:', error);
+        throw error;
+    }
+    finally {
+        // Close database connection
+        await dbManager.close();
+    }
+}
+// Run the regeneration
+regenerateAllEmbeddings()
+    .then(() => {
+    logger.info('Embedding regeneration process completed successfully');
+    process.exit(0);
+})
+    .catch((error) => {
+    logger.error('Embedding regeneration process failed:', error);
+    process.exit(1);
+});

package/dist/db/test-embeddings.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+import { DatabaseManager } from './index.js';
+import { databaseConfig } from '../config/index.js';
+import { logger } from '../utils/logger.js';
+import { embeddingService } from '../services/embedding-service.js';
+/**
+ * Test script to verify the embedding functionality
+ */
+async function testEmbeddings() {
+    logger.info('Starting embedding test...');
+    // Get database connection
+    const config = databaseConfig;
+    const dbManager = await DatabaseManager.get_instance(config);
+    try {
+        // Test 1: Generate an embedding
+        logger.info('Test 1: Generating embedding for test text...');
+        const testText = "This is a test of the embedding functionality";
+        const embedding = await embeddingService.generateEmbedding(testText);
+        logger.info(`Successfully generated embedding with dimension: ${embedding.length}`);
+        // Test 2: Create an entity with automatic embedding
+        logger.info('Test 2: Creating entity with automatic embedding...');
+        await dbManager.create_entities([
+            {
+                name: "TestEntityAuto",
+                entityType: "test",
+                observations: ["This is a test entity with automatic embedding generation"]
+            }
+        ]);
+        logger.info('Successfully created entity with automatic embedding');
+        // Test 3: Create an entity with explicit embedding
+        logger.info('Test 3: Creating entity with explicit embedding...');
+        await dbManager.create_entities([
+            {
+                name: "TestEntityExplicit",
+                entityType: "test",
+                observations: ["This is a test entity with explicit embedding"],
+                embedding: embedding
+            }
+        ]);
+        logger.info('Successfully created entity with explicit embedding');
+        // Test 4: Search for entities using text query
+        logger.info('Test 4: Searching for entities using text query...');
+        const textSearchResults = await dbManager.search_nodes("test entity");
+        logger.info(`Text search found ${textSearchResults.entities.length} entities`);
+        // Test 5: Search for entities using vector query
+        logger.info('Test 5: Searching for entities using vector query...');
+        const vectorSearchResults = await dbManager.search_nodes(embedding);
+        logger.info(`Vector search found ${vectorSearchResults.entities.length} entities`);
+        // Test 6: Retrieve and verify entity embeddings
+        logger.info('Test 6: Retrieving and verifying entity embeddings...');
+        const entity1 = await dbManager.get_entity("TestEntityAuto");
+        const entity2 = await dbManager.get_entity("TestEntityExplicit");
+        logger.info(`TestEntityAuto embedding dimension: ${entity1.embedding?.length || 'undefined'}`);
+        logger.info(`TestEntityExplicit embedding dimension: ${entity2.embedding?.length || 'undefined'}`);
+        // Test 7: Clean up test entities
+        logger.info('Test 7: Cleaning up test entities...');
+        await dbManager.delete_entity("TestEntityAuto");
+        await dbManager.delete_entity("TestEntityExplicit");
+        logger.info('Successfully deleted test entities');
+        logger.info('All embedding tests completed successfully!');
+    }
+    catch (error) {
+        logger.error('Embedding test failed:', error);
+        throw error;
+    }
+    finally {
+        // Close database connection
+        await dbManager.close();
+    }
+}
+// Run tests
+testEmbeddings()
+    .then(() => {
+    logger.info('Embedding tests completed successfully');
+    process.exit(0);
+})
+    .catch((error) => {
+    logger.error('Embedding tests failed:', error);
+    process.exit(1);
+});