cozo-memory 1.1.8 → 1.2.1

package/dist/test-hierarchical-mcp.js

@@ -0,0 +1,135 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_1 = require("./index");
+const fs = __importStar(require("fs"));
+const TEST_DB_PATH = 'test_hierarchical_mcp';
+async function runTest() {
+    console.error('=== Testing compress_memory_levels and analyze_memory_distribution MCP Tools ===\n');
+    // Clean up old test database
+    const dbFile = `${TEST_DB_PATH}.db`;
+    if (fs.existsSync(dbFile)) {
+        try {
+            fs.unlinkSync(dbFile);
+            console.error('[Cleanup] Removed old test database');
+        }
+        catch (e) {
+            console.error('[Cleanup] Warning: Could not remove old database:', e);
+        }
+    }
+    const server = new index_1.MemoryServer(TEST_DB_PATH);
+    await server.initPromise;
+    try {
+        console.error('1. Creating test entity...');
+        const entity = await server.createEntity({
+            name: 'Test Project',
+            type: 'Project',
+            metadata: {}
+        });
+        const entityId = entity.id;
+        console.error(`✓ Created entity: ${entityId.substring(0, 8)}...\n`);
+        console.error('2. Adding observations at L0 (raw) level...');
+        // Add multiple observations with L0 level and varied content to avoid deduplication
+        for (let i = 0; i < 15; i++) {
+            await server.addObservation({
+                entity_id: entityId,
+                text: `Observation ${i + 1}: This is a test observation about topic ${i * 7} with unique content ${Math.random().toString(36).substring(7)}`,
+                metadata: { memory_level: 0 },
+                deduplicate: false // Disable deduplication for this test
+            });
+        }
+        console.error(`✓ Added 15 observations at L0 level\n`);
+        console.error('3. Testing analyze_memory_distribution...');
+        const stats = await server.getHierarchicalMemoryService().getMemoryStats(entityId);
+        console.error(`✓ Memory distribution analysis completed:`);
+        console.error(`  - Total observations: ${stats.total_observations}`);
+        console.error(`  - L0 (Raw): ${stats.by_level[0] || 0}`);
+        console.error(`  - L1 (Session): ${stats.by_level[1] || 0}`);
+        console.error(`  - L2 (Weekly): ${stats.by_level[2] || 0}`);
+        console.error(`  - L3 (Monthly): ${stats.by_level[3] || 0}\n`);
+        if (stats.total_observations !== 15) {
+            console.error(`⚠ Warning: Expected 15 observations, got ${stats.total_observations}`);
+        }
+        console.error('4. Testing compress_memory_levels (will likely not compress due to recency)...');
+        // Note: Compression requires observations older than retention period
+        // For L0, default is 24 hours, so fresh observations won't be compressed
+        const compressionResult = await server.getHierarchicalMemoryService().compressMemoryLevel(entityId, 0);
+        if (compressionResult) {
+            console.error(`✓ Compression completed:`);
+            console.error(`  - Level: ${compressionResult.level}`);
+            console.error(`  - Compressed observations: ${compressionResult.compressed_observations}`);
+            console.error(`  - Summary ID: ${compressionResult.summary_id.substring(0, 8)}...`);
+            console.error(`  - Preserved: ${compressionResult.preserved_observations.length}`);
+            console.error(`  - Deleted: ${compressionResult.deleted_observations.length}`);
+        }
+        else {
+            console.error(`✓ No compression performed (observations too recent or insufficient count)`);
+            console.error(`  This is expected behavior - observations must be older than retention period`);
+        }
+        console.error('\n5. Re-checking memory distribution after compression attempt...');
+        const statsAfter = await server.getHierarchicalMemoryService().getMemoryStats(entityId);
+        console.error(`✓ Updated memory distribution:`);
+        console.error(`  - Total observations: ${statsAfter.total_observations}`);
+        console.error(`  - L0 (Raw): ${statsAfter.by_level[0] || 0}`);
+        console.error(`  - L1 (Session): ${statsAfter.by_level[1] || 0}`);
+        console.error(`  - L2 (Weekly): ${statsAfter.by_level[2] || 0}`);
+        console.error(`  - L3 (Monthly): ${statsAfter.by_level[3] || 0}`);
+        console.error('\n6. Testing with manually aged observations...');
+        // Create observations with old timestamps by manipulating metadata
+        const oldTimestamp = Date.now() - (30 * 24 * 60 * 60 * 1000); // 30 days ago
+        console.error(`  Note: CozoDB Validity uses system time, so we cannot easily simulate old observations`);
+        console.error(`  In production, compression would work on observations older than retention periods`);
+        console.error('\n=== ✓ Hierarchical Memory MCP Tools Test Passed ===\n');
+    }
+    catch (error) {
+        console.error('\n=== ✗ Test Failed ===');
+        console.error('Error:', error);
+        throw error;
+    }
+    finally {
+        // Cleanup
+        server.db.close();
+        if (fs.existsSync(dbFile)) {
+            try {
+                fs.unlinkSync(dbFile);
+                console.error('[Cleanup] Test database removed');
+            }
+            catch (e) {
+                console.error('[Cleanup] Warning: Could not remove test database');
+            }
+        }
+    }
+}
+runTest().catch(console.error);

package/dist/test-large-dataset.js

@@ -0,0 +1,502 @@
+"use strict";
+/**
+ * Large Dataset Performance Test
+ *
+ * Tests system performance with realistic data volumes
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const embedding_service_1 = require("./embedding-service");
+const hybrid_search_1 = require("./hybrid-search");
+const performance_monitor_1 = require("./performance-monitor");
+const logger_1 = require("./logger");
+const uuid_1 = require("uuid");
+// Set log level to INFO for cleaner output
+logger_1.logger.setLevel(logger_1.LogLevel.INFO);
+const CONFIGS = {
+    small: {
+        numEntities: 50, // Reduced to keep total observations under 100
+        numObservationsPerEntity: 1,
+        numRelationships: 75,
+        searchQueries: 20
+    },
+    medium: {
+        numEntities: 200,
+        numObservationsPerEntity: 2,
+        numRelationships: 400,
+        searchQueries: 50
+    },
+    large: {
+        numEntities: 500,
+        numObservationsPerEntity: 3,
+        numRelationships: 1000,
+        searchQueries: 100
+    }
+};
+// Sample data generators
+const ENTITY_TYPES = ['Person', 'Project', 'Technology', 'Document', 'Task'];
+const RELATION_TYPES = ['works_on', 'uses', 'depends_on', 'created_by', 'related_to'];
+function generateEntityName(type, index) {
+    const names = {
+        Person: ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve', 'Frank', 'Grace', 'Henry'],
+        Project: ['Alpha', 'Beta', 'Gamma', 'Delta', 'Epsilon', 'Zeta', 'Eta', 'Theta'],
+        Technology: ['TypeScript', 'React', 'Node.js', 'Python', 'Go', 'Rust', 'Java', 'C++'],
+        Document: ['Spec', 'Guide', 'Manual', 'Report', 'Analysis', 'Design', 'Plan', 'Review'],
+        Task: ['Implement', 'Test', 'Deploy', 'Review', 'Refactor', 'Document', 'Optimize', 'Debug']
+    };
+    const nameList = names[type] || ['Item'];
+    const baseName = nameList[index % nameList.length];
+    return `${baseName} ${Math.floor(index / nameList.length) + 1}`;
+}
+function generateObservation(entityName, type, index) {
+    const templates = [
+        `${entityName} is a ${type.toLowerCase()} that focuses on innovation and quality.`,
+        `Key characteristics of ${entityName} include reliability and performance.`,
+        `${entityName} has been actively developed and maintained since 2020.`,
+        `The primary goal of ${entityName} is to deliver exceptional results.`,
+        `${entityName} integrates seamlessly with modern development workflows.`
+    ];
+    return templates[index % templates.length];
+}
+async function createTestData(db, embeddingService, config) {
+    const startTime = Date.now();
+    const entityIds = [];
+    logger_1.logger.info('TestLargeDataset', `Creating ${config.numEntities} entities...`);
+    // Pre-generate embeddings for entity types to speed up creation
+    const typeEmbeddings = new Map();
+    for (const type of ENTITY_TYPES) {
+        const contentEmbed = await embeddingService.embed(`${type} entity`);
+        const nameEmbed = await embeddingService.embed(type);
+        typeEmbeddings.set(type, { content: contentEmbed, name: nameEmbed });
+    }
+    // Create entities directly in CozoDB
+    for (let i = 0; i < config.numEntities; i++) {
+        const type = ENTITY_TYPES[i % ENTITY_TYPES.length];
+        const name = generateEntityName(type, i);
+        const id = (0, uuid_1.v4)();
+        const endTimer = performance_monitor_1.perfMonitor.startTimer('create_entity');
+        try {
+            // Reuse type embeddings for speed
+            const embeddings = typeEmbeddings.get(type);
+            // Insert into CozoDB
+            await db.run(`
+        ?[id, name, type, content_embedding, name_embedding, metadata, created_at] <- [
+          [$id, $name, $type, $content_embedding, $name_embedding, $metadata, $created_at]
+        ]
+        :put entity { id => name, type, content_embedding, name_embedding, metadata, created_at }
+      `, {
+                id,
+                name,
+                type,
+                content_embedding: embeddings.content,
+                name_embedding: embeddings.name,
+                metadata: { index: i, category: type.toLowerCase(), created_at: Date.now() },
+                created_at: [Date.now() * 1000, true]
+            });
+            entityIds.push(id);
+            endTimer();
+        }
+        catch (error) {
+            performance_monitor_1.perfMonitor.recordMetric('create_entity', 0, true);
+            logger_1.logger.error('TestLargeDataset', `Failed to create entity ${name}:`, error);
+        }
+        if ((i + 1) % 50 === 0) {
+            logger_1.logger.info('TestLargeDataset', `Created ${i + 1}/${config.numEntities} entities`);
+        }
+    }
+    logger_1.logger.info('TestLargeDataset', `Creating observations...`);
+    // Pre-generate a few observation embeddings to reuse (for speed)
+    const observationTemplates = [
+        'This entity focuses on innovation and quality.',
+        'Key characteristics include reliability and performance.',
+        'Has been actively developed since 2020.',
+        'Primary goal is to deliver exceptional results.',
+        'Integrates seamlessly with modern workflows.'
+    ];
+    logger_1.logger.info('TestLargeDataset', `Pre-generating ${observationTemplates.length} observation embeddings...`);
+    const templateEmbeddings = await Promise.all(observationTemplates.map(t => embeddingService.embed(t)));
+    // Create observations in batches for better performance
+    let obsCount = 0;
+    const totalObservations = entityIds.length * config.numObservationsPerEntity;
+    let lastLogTime = Date.now();
+    const BATCH_SIZE = 50; // Insert 50 observations at once
+    const observationBatch = [];
+    for (const entityId of entityIds) {
+        for (let j = 0; j < config.numObservationsPerEntity; j++) {
+            // Reuse pre-generated embeddings for speed
+            const templateIdx = j % templateEmbeddings.length;
+            const text = observationTemplates[templateIdx];
+            const embedding = templateEmbeddings[templateIdx];
+            observationBatch.push({
+                id: (0, uuid_1.v4)(),
+                entity_id: entityId,
+                text,
+                embedding,
+                metadata: { confidence: 0.8 + Math.random() * 0.2 },
+                session_id: '',
+                task_id: '',
+                created_at: [Date.now() * 1000, true]
+            });
+            // Insert batch when it reaches BATCH_SIZE
+            if (observationBatch.length >= BATCH_SIZE) {
+                const endTimer = performance_monitor_1.perfMonitor.startTimer('add_observation');
+                try {
+                    // Build batch insert query - remove hyphens from UUIDs for variable names
+                    const rows = observationBatch.map(obs => {
+                        const cleanId = obs.id.replace(/-/g, '_');
+                        return `[$id_${cleanId}, $entity_id_${cleanId}, $text_${cleanId}, $embedding_${cleanId}, $metadata_${cleanId}, $session_id_${cleanId}, $task_id_${cleanId}, $created_at_${cleanId}]`;
+                    }).join(',\n            ');
+                    const params = {};
+                    for (const obs of observationBatch) {
+                        const cleanId = obs.id.replace(/-/g, '_');
+                        params[`id_${cleanId}`] = obs.id;
+                        params[`entity_id_${cleanId}`] = obs.entity_id;
+                        params[`text_${cleanId}`] = obs.text;
+                        params[`embedding_${cleanId}`] = obs.embedding;
+                        params[`metadata_${cleanId}`] = obs.metadata;
+                        params[`session_id_${cleanId}`] = obs.session_id;
+                        params[`task_id_${cleanId}`] = obs.task_id;
+                        params[`created_at_${cleanId}`] = obs.created_at;
+                    }
+                    await db.run(`
+            ?[id, entity_id, text, embedding, metadata, session_id, task_id, created_at] <- [
+              ${rows}
+            ]
+            :put observation { id => entity_id, text, embedding, metadata, session_id, task_id, created_at }
+          `, params);
+                    obsCount += observationBatch.length;
+                    endTimer();
+                    observationBatch.length = 0; // Clear batch
+                }
+                catch (error) {
+                    performance_monitor_1.perfMonitor.recordMetric('add_observation', 0, true);
+                    logger_1.logger.error('TestLargeDataset', `Failed to create observation batch at ${obsCount}:`, error);
+                    observationBatch.length = 0; // Clear batch on error
+                }
+                // Log progress
+                const now = Date.now();
+                if (obsCount % 50 === 0 || (now - lastLogTime) > 10000) {
+                    logger_1.logger.info('TestLargeDataset', `Created ${obsCount}/${totalObservations} observations (${((obsCount / totalObservations) * 100).toFixed(1)}%)`);
+                    lastLogTime = now;
+                }
+            }
+        }
+    }
+    // Insert remaining observations
+    if (observationBatch.length > 0) {
+        const endTimer = performance_monitor_1.perfMonitor.startTimer('add_observation');
+        try {
+            const rows = observationBatch.map(obs => {
+                const cleanId = obs.id.replace(/-/g, '_');
+                return `[$id_${cleanId}, $entity_id_${cleanId}, $text_${cleanId}, $embedding_${cleanId}, $metadata_${cleanId}, $session_id_${cleanId}, $task_id_${cleanId}, $created_at_${cleanId}]`;
+            }).join(',\n            ');
+            const params = {};
+            for (const obs of observationBatch) {
+                const cleanId = obs.id.replace(/-/g, '_');
+                params[`id_${cleanId}`] = obs.id;
+                params[`entity_id_${cleanId}`] = obs.entity_id;
+                params[`text_${cleanId}`] = obs.text;
+                params[`embedding_${cleanId}`] = obs.embedding;
+                params[`metadata_${cleanId}`] = obs.metadata;
+                params[`session_id_${cleanId}`] = obs.session_id;
+                params[`task_id_${cleanId}`] = obs.task_id;
+                params[`created_at_${cleanId}`] = obs.created_at;
+            }
+            await db.run(`
+        ?[id, entity_id, text, embedding, metadata, session_id, task_id, created_at] <- [
+          ${rows}
+        ]
+        :put observation { id => entity_id, text, embedding, metadata, session_id, task_id, created_at }
+      `, params);
+            obsCount += observationBatch.length;
+            endTimer();
+        }
+        catch (error) {
+            performance_monitor_1.perfMonitor.recordMetric('add_observation', 0, true);
+            logger_1.logger.error('TestLargeDataset', `Failed to create final observation batch:`, error);
+        }
+        logger_1.logger.info('TestLargeDataset', `Created ${obsCount}/${totalObservations} observations (100.0%)`);
+    }
+    logger_1.logger.info('TestLargeDataset', `Creating ${config.numRelationships} relationships...`);
+    // Create relationships
+    for (let i = 0; i < config.numRelationships; i++) {
+        const fromId = entityIds[Math.floor(Math.random() * entityIds.length)];
+        let toId = entityIds[Math.floor(Math.random() * entityIds.length)];
+        // Avoid self-references
+        while (toId === fromId) {
+            toId = entityIds[Math.floor(Math.random() * entityIds.length)];
+        }
+        const relationType = RELATION_TYPES[i % RELATION_TYPES.length];
+        const endTimer = performance_monitor_1.perfMonitor.startTimer('create_relation');
+        try {
+            await db.run(`
+        ?[from_id, to_id, relation_type, strength, metadata, created_at] <- [
+          [$from_id, $to_id, $relation_type, $strength, $metadata, $created_at]
+        ]
+        :put relationship { from_id, to_id, relation_type => strength, metadata, created_at }
+      `, {
+                from_id: fromId,
+                to_id: toId,
+                relation_type: relationType,
+                strength: 0.5 + Math.random() * 0.5,
+                metadata: {},
+                created_at: [Date.now() * 1000, true]
+            });
+            endTimer();
+        }
+        catch (error) {
+            performance_monitor_1.perfMonitor.recordMetric('create_relation', 0, true);
+        }
+        if ((i + 1) % 500 === 0) {
+            logger_1.logger.info('TestLargeDataset', `Created ${i + 1}/${config.numRelationships} relationships`);
+        }
+    }
+    const duration = Date.now() - startTime;
+    logger_1.logger.info('TestLargeDataset', `Data creation completed in ${(duration / 1000).toFixed(2)}s`);
+    return { entityIds, duration };
+}
+async function runSearchTests(hybridSearch, config) {
+    logger_1.logger.info('TestLargeDataset', `Running ${config.searchQueries} search queries...`);
+    const queries = [
+        'project management',
+        'software development',
+        'team collaboration',
+        'technical documentation',
+        'code review process',
+        'deployment pipeline',
+        'testing strategy',
+        'performance optimization'
+    ];
+    for (let i = 0; i < config.searchQueries; i++) {
+        const query = queries[i % queries.length];
+        const endTimer = performance_monitor_1.perfMonitor.startTimer('hybrid_search');
+        try {
+            await hybridSearch.search({ query, limit: 10 });
+            endTimer();
+        }
+        catch (error) {
+            performance_monitor_1.perfMonitor.recordMetric('hybrid_search', 0, true);
+            logger_1.logger.error('TestLargeDataset', `Search failed for query "${query}":`, error);
+        }
+        if ((i + 1) % 20 === 0) {
+            logger_1.logger.info('TestLargeDataset', `Completed ${i + 1}/${config.searchQueries} searches`);
+        }
+    }
+}
+async function runTest(configName, cleanStart = false) {
+    const config = CONFIGS[configName];
+    if (!config) {
+        logger_1.logger.error('TestLargeDataset', `Unknown config: ${configName}`);
+        return;
+    }
+    logger_1.logger.info('TestLargeDataset', `\n=== Starting ${configName.toUpperCase()} dataset test ===`);
+    logger_1.logger.info('TestLargeDataset', `Config: ${JSON.stringify(config, null, 2)}`);
+    // Use the real CozoDB setup like in index.ts
+    const { CozoDb } = await import('cozo-node');
+    const dbPath = `test_large_${configName}.cozo.db`;
+    // Delete old database only if cleanStart flag is set
+    const fs = await import('fs');
+    if (cleanStart && fs.existsSync(dbPath)) {
+        logger_1.logger.info('TestLargeDataset', `Removing old database: ${dbPath}`);
+        fs.unlinkSync(dbPath);
+    }
+    else if (fs.existsSync(dbPath)) {
+        logger_1.logger.info('TestLargeDataset', `Using existing database: ${dbPath}`);
+    }
+    const db = new CozoDb('sqlite', dbPath);
+    const embeddingService = new embedding_service_1.EmbeddingService();
+    const hybridSearch = new hybrid_search_1.HybridSearch(db, embeddingService);
+    // Initialize schema like the real server does
+    try {
+        // Create entity table
+        await db.run(`
+      :create entity {
+        id: String,
+        =>
+        name: String,
+        type: String,
+        content_embedding: <F32; 1024>,
+        name_embedding: <F32; 1024>,
+        metadata: Json,
+        created_at: Validity
+      }
+    `);
+        // Create content HNSW index
+        await db.run(`
+      ::hnsw create entity:semantic {
+        dim: 1024,
+        m: 50,
+        dtype: F32,
+        ef_construction: 200,
+        fields: [content_embedding],
+        distance: Cosine,
+        extend_candidates: true,
+        keep_pruned_connections: true
+      }
+    `);
+        // Create name HNSW index
+        await db.run(`
+      ::hnsw create entity:name_semantic {
+        dim: 1024,
+        m: 50,
+        dtype: F32,
+        ef_construction: 200,
+        fields: [name_embedding],
+        distance: Cosine,
+        extend_candidates: true,
+        keep_pruned_connections: true
+      }
+    `);
+        // Create FTS index for entity names
+        await db.run(`
+      ::fts create entity:fts {
+        extractor: name,
+        tokenizer: Simple,
+        filters: [Lowercase, Stemmer('english'), Stopwords('en')]
+      }
+    `);
+        // Create observation table
+        await db.run(`
+      :create observation {
+        id: String,
+        =>
+        entity_id: String,
+        text: String,
+        embedding: <F32; 1024>,
+        metadata: Json,
+        session_id: String,
+        task_id: String,
+        created_at: Validity
+      }
+    `);
+        // Create observation HNSW index
+        await db.run(`
+      ::hnsw create observation:semantic {
+        dim: 1024,
+        m: 50,
+        dtype: F32,
+        ef_construction: 200,
+        fields: [embedding],
+        distance: Cosine,
+        extend_candidates: true,
+        keep_pruned_connections: true
+      }
+    `);
+        // Create FTS index for observation text
+        await db.run(`
+      ::fts create observation:fts {
+        extractor: text,
+        tokenizer: Simple,
+        filters: [Lowercase, Stemmer('english'), Stopwords('en')]
+      }
+    `);
+        // Create relationship table
+        await db.run(`
+      :create relationship {
+        from_id: String,
+        to_id: String,
+        relation_type: String,
+        =>
+        strength: Float,
+        metadata: Json,
+        created_at: Validity
+      }
+    `);
+        // Create search cache table
+        await db.run(`
+      :create search_cache {
+        query_hash: String,
+        =>
+        query_text: String,
+        results: Json,
+        options: Json,
+        embedding: <F32; 1024>,
+        created_at: Int
+      }
+    `);
+        // Create search cache HNSW index
+        await db.run(`
+      ::hnsw create search_cache:semantic {
+        dim: 1024,
+        m: 16,
+        dtype: F32,
+        ef_construction: 200,
+        fields: [embedding],
+        distance: Cosine
+      }
+    `);
+        // Create entity_rank table (for PageRank scores)
+        await db.run(`
+      :create entity_rank {
+        entity_id: String
+        =>
+        pagerank: Float
+      }
+    `);
+        logger_1.logger.info('TestLargeDataset', 'Database schema initialized with all indexes');
+    }
+    catch (error) {
+        if (!error.message?.includes('already exists')) {
+            logger_1.logger.error('TestLargeDataset', 'Schema initialization failed:', error);
+            throw error;
+        }
+        logger_1.logger.info('TestLargeDataset', 'Schema already exists, continuing...');
+    }
+    try {
+        // Check if database already has data - simple approach
+        let existingEntityCount = 0;
+        try {
+            const statsQuery = await db.run(`?[id] := *entity{id} :limit 1`);
+            existingEntityCount = statsQuery.rows.length > 0 ? 1 : 0;
+            if (existingEntityCount > 0) {
+                // Get actual count
+                const countQuery = await db.run(`?[count(id)] := *entity{id}`);
+                existingEntityCount = countQuery.rows[0]?.[0] || 0;
+            }
+        }
+        catch (e) {
+            // Table doesn't exist yet, that's fine
+            existingEntityCount = 0;
+        }
+        if (existingEntityCount > 0 && !cleanStart) {
+            logger_1.logger.info('TestLargeDataset', `Database already contains ${existingEntityCount} entities, skipping data creation`);
+            logger_1.logger.info('TestLargeDataset', `Use --clean flag to recreate database from scratch`);
+        }
+        else {
+            // Create test data
+            const { entityIds, duration: createDuration } = await createTestData(db, embeddingService, config);
+            const totalOps = config.numEntities +
+                (config.numEntities * config.numObservationsPerEntity) +
+                config.numRelationships;
+            const totalTime = createDuration / 1000;
+            const throughput = totalOps / totalTime;
+            logger_1.logger.info('TestLargeDataset', `\nData creation stats:`);
+            logger_1.logger.info('TestLargeDataset', `Total operations: ${totalOps}`);
+            logger_1.logger.info('TestLargeDataset', `Total time: ${totalTime.toFixed(2)}s`);
+            logger_1.logger.info('TestLargeDataset', `Throughput: ${throughput.toFixed(2)} ops/sec`);
+        }
+        // Run search tests
+        await runSearchTests(hybridSearch, config);
+        // Print performance summary
+        logger_1.logger.info('TestLargeDataset', '\n=== Performance Summary ===');
+        performance_monitor_1.perfMonitor.logSummary();
+    }
+    catch (error) {
+        logger_1.logger.error('TestLargeDataset', 'Test failed:', error);
+    }
+    finally {
+        db.close();
+    }
+}
+// Run tests
+async function main() {
+    const configName = process.argv[2] || 'small';
+    const cleanStart = process.argv.includes('--clean');
+    if (cleanStart) {
+        logger_1.logger.info('TestLargeDataset', 'Clean start mode: will delete existing database');
+    }
+    await runTest(configName, cleanStart);
+    logger_1.logger.info('TestLargeDataset', '\n=== Test completed ===');
+    logger_1.logger.info('TestLargeDataset', `\nUsage: npx ts-node src/test-large-dataset.ts [small|medium|large] [--clean]`);
+    logger_1.logger.info('TestLargeDataset', `  --clean: Delete existing database before test`);
+}
+main().catch(console.error);