moflo 4.8.19 → 4.8.20

package/src/@claude-flow/memory/dist/database-provider.js

@@ -0,0 +1,372 @@
+/**
+ * DatabaseProvider - Platform-aware database selection
+ *
+ * Automatically selects best backend:
+ * - All platforms: sql.js (WASM, no native deps)
+ * - Windows: sql.js (WASM, universal) when native fails
+ * - Fallback: JSON file storage
+ *
+ * @module v3/memory/database-provider
+ */
+import { platform } from 'node:os';
+import { existsSync } from 'node:fs';
+import { SqlJsBackend } from './sqljs-backend.js';
+/**
+ * Detect platform and recommend provider
+ */
+function detectPlatform() {
+    const os = platform();
+    const isWindows = os === 'win32';
+    const isMacOS = os === 'darwin';
+    const isLinux = os === 'linux';
+    // Recommend better-sqlite3 for Unix-like systems, sql.js for Windows
+    const recommendedProvider = 'sql.js';
+    return {
+        os,
+        isWindows,
+        isMacOS,
+        isLinux,
+        recommendedProvider,
+    };
+}
+/**
+ * Test if RVF backend is available (always true — pure-TS fallback)
+ */
+async function testRvf() {
+    return true;
+}
+/** better-sqlite3 removed — sql.js is the only SQLite backend */
+/**
+ * Test if sql.js is available and working
+ */
+async function testSqlJs() {
+    try {
+        const initSqlJs = (await import('sql.js')).default;
+        const SQL = await initSqlJs();
+        const testDb = new SQL.Database();
+        testDb.close();
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+}
+/**
+ * Select best available provider
+ */
+async function selectProvider(preferred, verbose = false) {
+    if (preferred && preferred !== 'auto') {
+        if (verbose) {
+            console.log(`[DatabaseProvider] Using explicitly specified provider: ${preferred}`);
+        }
+        return preferred;
+    }
+    const platformInfo = detectPlatform();
+    if (verbose) {
+        console.log(`[DatabaseProvider] Platform detected: ${platformInfo.os}`);
+        console.log(`[DatabaseProvider] Recommended provider: ${platformInfo.recommendedProvider}`);
+    }
+    // Try RVF first (always available via pure-TS fallback, native when @ruvector/rvf installed)
+    if (await testRvf()) {
+        if (verbose) {
+            console.log('[DatabaseProvider] RVF backend available');
+        }
+        return 'rvf';
+    }
+    // Try sql.js (moflo: sql.js is the only SQLite backend)
+    if (await testSqlJs()) {
+        if (verbose) {
+            console.log('[DatabaseProvider] sql.js available and working');
+        }
+        return 'sql.js';
+    }
+    else if (verbose) {
+        console.log('[DatabaseProvider] sql.js not available, using JSON fallback');
+    }
+    // Final fallback to JSON
+    return 'json';
+}
+/**
+ * Create a database instance with platform-aware provider selection
+ *
+ * @param path - Database file path (:memory: for in-memory)
+ * @param options - Database configuration options
+ * @returns Initialized database backend
+ *
+ * @example
+ * ```typescript
+ * // Auto-select best provider for platform
+ * const db = await createDatabase('./data/memory.db');
+ *
+ * // Force specific provider
+ * const db = await createDatabase('./data/memory.db', {
+ *   provider: 'sql.js'
+ * });
+ *
+ * // With custom options
+ * const db = await createDatabase('./data/memory.db', {
+ *   verbose: true,
+ *   optimize: true,
+ *   autoPersistInterval: 10000
+ * });
+ * ```
+ */
+export async function createDatabase(path, options = {}) {
+    const { provider = 'auto', verbose = false, walMode = true, optimize = true, defaultNamespace = 'default', maxEntries = 1000000, autoPersistInterval = 5000, wasmPath, } = options;
+    // Select provider
+    const selectedProvider = await selectProvider(provider, verbose);
+    if (verbose) {
+        console.log(`[DatabaseProvider] Creating database with provider: ${selectedProvider}`);
+        console.log(`[DatabaseProvider] Database path: ${path}`);
+    }
+    let backend;
+    switch (selectedProvider) {
+        case 'sql.js': {
+            const config = {
+                databasePath: path,
+                optimize,
+                defaultNamespace,
+                maxEntries,
+                verbose,
+                autoPersistInterval,
+                wasmPath,
+            };
+            backend = new SqlJsBackend(config);
+            break;
+        }
+        case 'rvf': {
+            const { RvfBackend } = await import('./rvf-backend.js');
+            backend = new RvfBackend({
+                databasePath: path.replace(/\.(db|json)$/, '.rvf'),
+                dimensions: 1536,
+                verbose,
+                defaultNamespace,
+                autoPersistInterval,
+            });
+            break;
+        }
+        case 'json': {
+            // Simple JSON file backend (minimal implementation)
+            backend = new JsonBackend(path, verbose);
+            break;
+        }
+        default:
+            throw new Error(`Unknown database provider: ${selectedProvider}`);
+    }
+    // Initialize the backend
+    await backend.initialize();
+    if (verbose) {
+        console.log(`[DatabaseProvider] Database initialized successfully`);
+    }
+    return backend;
+}
+/**
+ * Get platform information
+ */
+export function getPlatformInfo() {
+    return detectPlatform();
+}
+/**
+ * Check which providers are available
+ */
+export async function getAvailableProviders() {
+    return {
+        rvf: true,
+        betterSqlite3: false, // Removed — sql.js is the only SQLite backend
+        sqlJs: await testSqlJs(),
+        json: true,
+    };
+}
+// ===== JSON Fallback Backend =====
+/**
+ * Simple JSON file backend for when no SQLite is available
+ */
+class JsonBackend {
+    entries = new Map();
+    path;
+    verbose;
+    initialized = false;
+    constructor(path, verbose = false) {
+        this.path = path;
+        this.verbose = verbose;
+    }
+    async initialize() {
+        if (this.initialized)
+            return;
+        // Load from file if exists
+        if (this.path !== ':memory:' && existsSync(this.path)) {
+            try {
+                const fs = await import('node:fs/promises');
+                const data = await fs.readFile(this.path, 'utf-8');
+                const entries = JSON.parse(data);
+                for (const entry of entries) {
+                    // Convert embedding array back to Float32Array
+                    if (entry.embedding) {
+                        entry.embedding = new Float32Array(entry.embedding);
+                    }
+                    this.entries.set(entry.id, entry);
+                }
+                if (this.verbose) {
+                    console.log(`[JsonBackend] Loaded ${this.entries.size} entries from ${this.path}`);
+                }
+            }
+            catch (error) {
+                if (this.verbose) {
+                    console.error('[JsonBackend] Error loading file:', error);
+                }
+            }
+        }
+        this.initialized = true;
+    }
+    async shutdown() {
+        await this.persist();
+        this.initialized = false;
+    }
+    async store(entry) {
+        this.entries.set(entry.id, entry);
+        await this.persist();
+    }
+    async get(id) {
+        return this.entries.get(id) || null;
+    }
+    async getByKey(namespace, key) {
+        for (const entry of this.entries.values()) {
+            if (entry.namespace === namespace && entry.key === key) {
+                return entry;
+            }
+        }
+        return null;
+    }
+    async update(id, updateData) {
+        const entry = this.entries.get(id);
+        if (!entry)
+            return null;
+        const updated = { ...entry, ...updateData, updatedAt: Date.now(), version: entry.version + 1 };
+        this.entries.set(id, updated);
+        await this.persist();
+        return updated;
+    }
+    async delete(id) {
+        const result = this.entries.delete(id);
+        await this.persist();
+        return result;
+    }
+    async query(query) {
+        let results = Array.from(this.entries.values());
+        if (query.namespace) {
+            results = results.filter((e) => e.namespace === query.namespace);
+        }
+        if (query.key) {
+            results = results.filter((e) => e.key === query.key);
+        }
+        if (query.tags && query.tags.length > 0) {
+            results = results.filter((e) => query.tags.every((tag) => e.tags.includes(tag)));
+        }
+        return results.slice(0, query.limit);
+    }
+    async search(embedding, options) {
+        // Simple brute-force search
+        const results = [];
+        for (const entry of this.entries.values()) {
+            if (!entry.embedding)
+                continue;
+            const similarity = this.cosineSimilarity(embedding, entry.embedding);
+            if (options.threshold && similarity < options.threshold)
+                continue;
+            results.push({ entry, score: similarity, distance: 1 - similarity });
+        }
+        results.sort((a, b) => b.score - a.score);
+        return results.slice(0, options.k);
+    }
+    async bulkInsert(entries) {
+        for (const entry of entries) {
+            this.entries.set(entry.id, entry);
+        }
+        await this.persist();
+    }
+    async bulkDelete(ids) {
+        let count = 0;
+        for (const id of ids) {
+            if (this.entries.delete(id))
+                count++;
+        }
+        await this.persist();
+        return count;
+    }
+    async count(namespace) {
+        if (!namespace)
+            return this.entries.size;
+        let count = 0;
+        for (const entry of this.entries.values()) {
+            if (entry.namespace === namespace)
+                count++;
+        }
+        return count;
+    }
+    async listNamespaces() {
+        const namespaces = new Set();
+        for (const entry of this.entries.values()) {
+            namespaces.add(entry.namespace);
+        }
+        return Array.from(namespaces);
+    }
+    async clearNamespace(namespace) {
+        let count = 0;
+        for (const [id, entry] of this.entries.entries()) {
+            if (entry.namespace === namespace) {
+                this.entries.delete(id);
+                count++;
+            }
+        }
+        await this.persist();
+        return count;
+    }
+    async getStats() {
+        return {
+            totalEntries: this.entries.size,
+            entriesByNamespace: {},
+            entriesByType: {},
+            memoryUsage: 0,
+            avgQueryTime: 0,
+            avgSearchTime: 0,
+        };
+    }
+    async healthCheck() {
+        return {
+            status: 'healthy',
+            components: {
+                storage: { status: 'healthy', latency: 0 },
+                index: { status: 'healthy', latency: 0 },
+                cache: { status: 'healthy', latency: 0 },
+            },
+            timestamp: Date.now(),
+            issues: [],
+            recommendations: ['Consider using SQLite backend for better performance'],
+        };
+    }
+    async persist() {
+        if (this.path === ':memory:')
+            return;
+        const fs = await import('node:fs/promises');
+        const entries = Array.from(this.entries.values()).map((e) => ({
+            ...e,
+            // Convert Float32Array to regular array for JSON serialization
+            embedding: e.embedding ? Array.from(e.embedding) : undefined,
+        }));
+        await fs.writeFile(this.path, JSON.stringify(entries, null, 2));
+    }
+    cosineSimilarity(a, b) {
+        let dot = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dot += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+}
+//# sourceMappingURL=database-provider.js.map

package/src/@claude-flow/memory/dist/database-provider.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Database Provider Tests - Cross-Platform Compatibility
+ *
+ * Tests for platform-aware database selection and fallback mechanisms
+ */
+export {};
+//# sourceMappingURL=database-provider.test.d.ts.map

package/src/@claude-flow/memory/dist/database-provider.test.js

@@ -0,0 +1,287 @@
+/**
+ * Database Provider Tests - Cross-Platform Compatibility
+ *
+ * Tests for platform-aware database selection and fallback mechanisms
+ */
+import { describe, it, expect, afterEach } from 'vitest';
+import { createDatabase, getPlatformInfo, getAvailableProviders } from './database-provider.js';
+import { createDefaultEntry } from './types.js';
+import { unlinkSync, existsSync } from 'node:fs';
+describe('DatabaseProvider', () => {
+    const testDbPath = './test-database-provider.db';
+    afterEach(() => {
+        // Cleanup test database
+        if (existsSync(testDbPath)) {
+            try {
+                unlinkSync(testDbPath);
+            }
+            catch (error) {
+                // Ignore cleanup errors
+            }
+        }
+    });
+    describe('Platform Detection', () => {
+        it('should detect platform information', () => {
+            const info = getPlatformInfo();
+            expect(info).toHaveProperty('os');
+            expect(info).toHaveProperty('isWindows');
+            expect(info).toHaveProperty('isMacOS');
+            expect(info).toHaveProperty('isLinux');
+            expect(info).toHaveProperty('recommendedProvider');
+            // Should recommend sql.js on Windows, better-sqlite3 on Unix
+            if (info.isWindows) {
+                expect(info.recommendedProvider).toBe('sql.js');
+            }
+            else {
+                expect(info.recommendedProvider).toBe('sql.js');
+            }
+        });
+    });
+    describe('Provider Availability', () => {
+        it('should check available providers', async () => {
+            const available = await getAvailableProviders();
+            expect(available).toHaveProperty('betterSqlite3');
+            expect(available).toHaveProperty('sqlJs');
+            expect(available).toHaveProperty('json');
+            // JSON backend should always be available
+            expect(available.json).toBe(true);
+        });
+    });
+    describe('Automatic Provider Selection', () => {
+        it('should create database with auto provider selection', async () => {
+            const db = await createDatabase(':memory:');
+            expect(db).toBeDefined();
+            await expect(db.count()).resolves.toBe(0);
+            await db.shutdown();
+        });
+        it('should create persistent database with auto provider', async () => {
+            const db = await createDatabase(testDbPath);
+            expect(db).toBeDefined();
+            // Store test entry
+            const entry = createDefaultEntry({
+                key: 'test-key',
+                content: 'test content',
+                namespace: 'test',
+            });
+            await db.store(entry);
+            const count = await db.count();
+            // RVF backend may include internal metadata entries
+            expect(count).toBeGreaterThanOrEqual(1);
+            await db.shutdown();
+        });
+    });
+    describe('Explicit Provider Selection', () => {
+        it('should create database with sql.js provider', async () => {
+            const available = await getAvailableProviders();
+            if (!available.sqlJs) {
+                console.log('sql.js not available, skipping test');
+                return;
+            }
+            const db = await createDatabase(':memory:', {
+                provider: 'sql.js',
+                verbose: false,
+            });
+            expect(db).toBeDefined();
+            // Test basic operations
+            const entry = createDefaultEntry({
+                key: 'sqljs-test',
+                content: 'testing sql.js backend',
+                namespace: 'test',
+            });
+            await db.store(entry);
+            await expect(db.count()).resolves.toBe(1);
+            const retrieved = await db.get(entry.id);
+            expect(retrieved).toBeDefined();
+            expect(retrieved?.key).toBe('sqljs-test');
+            await db.shutdown();
+        });
+        it('should create database with better-sqlite3 provider', async () => {
+            const available = await getAvailableProviders();
+            if (!available.betterSqlite3) {
+                console.log('better-sqlite3 not available, skipping test');
+                return;
+            }
+            const db = await createDatabase(':memory:', {
+                provider: 'sql.js',
+                verbose: false,
+            });
+            expect(db).toBeDefined();
+            // Test basic operations
+            const entry = createDefaultEntry({
+                key: 'sqlite-test',
+                content: 'testing better-sqlite3 backend',
+                namespace: 'test',
+            });
+            await db.store(entry);
+            await expect(db.count()).resolves.toBe(1);
+            const retrieved = await db.get(entry.id);
+            expect(retrieved).toBeDefined();
+            expect(retrieved?.key).toBe('sqlite-test');
+            await db.shutdown();
+        });
+        it('should create database with JSON provider', async () => {
+            const db = await createDatabase(testDbPath, {
+                provider: 'json',
+                verbose: false,
+            });
+            expect(db).toBeDefined();
+            // Test basic operations
+            const entry = createDefaultEntry({
+                key: 'json-test',
+                content: 'testing JSON backend',
+                namespace: 'test',
+            });
+            await db.store(entry);
+            await expect(db.count()).resolves.toBe(1);
+            const retrieved = await db.get(entry.id);
+            expect(retrieved).toBeDefined();
+            expect(retrieved?.key).toBe('json-test');
+            await db.shutdown();
+        });
+    });
+    describe('Cross-Platform Functionality', () => {
+        it('should handle CRUD operations consistently across providers', async () => {
+            const available = await getAvailableProviders();
+            const providers = [];
+            if (available.betterSqlite3)
+                providers.push('sql.js');
+            if (available.sqlJs)
+                providers.push('sql.js');
+            providers.push('json'); // Always available
+            for (const provider of providers) {
+                const db = await createDatabase(':memory:', { provider });
+                // Create
+                const entry = createDefaultEntry({
+                    key: `test-${provider}`,
+                    content: `testing ${provider}`,
+                    namespace: 'cross-platform',
+                    tags: ['test', provider],
+                });
+                await db.store(entry);
+                // Read
+                const retrieved = await db.get(entry.id);
+                expect(retrieved).toBeDefined();
+                expect(retrieved?.key).toBe(`test-${provider}`);
+                expect(retrieved?.tags).toContain(provider);
+                // Update
+                const updated = await db.update(entry.id, {
+                    content: `updated ${provider}`,
+                });
+                expect(updated).toBeDefined();
+                expect(updated?.content).toBe(`updated ${provider}`);
+                // Query
+                const results = await db.query({
+                    type: 'hybrid',
+                    namespace: 'cross-platform',
+                    limit: 10,
+                });
+                expect(results.length).toBe(1);
+                expect(results[0].key).toBe(`test-${provider}`);
+                // Delete
+                const deleted = await db.delete(entry.id);
+                expect(deleted).toBe(true);
+                await expect(db.count()).resolves.toBe(0);
+                await db.shutdown();
+            }
+        });
+        it('should handle namespace operations across providers', async () => {
+            const db = await createDatabase(':memory:');
+            // Create entries in different namespaces
+            const entries = [
+                createDefaultEntry({ key: 'entry1', content: 'content1', namespace: 'ns1' }),
+                createDefaultEntry({ key: 'entry2', content: 'content2', namespace: 'ns1' }),
+                createDefaultEntry({ key: 'entry3', content: 'content3', namespace: 'ns2' }),
+            ];
+            for (const entry of entries) {
+                await db.store(entry);
+            }
+            // List namespaces
+            const namespaces = await db.listNamespaces();
+            expect(namespaces).toContain('ns1');
+            expect(namespaces).toContain('ns2');
+            // Count by namespace
+            await expect(db.count('ns1')).resolves.toBe(2);
+            await expect(db.count('ns2')).resolves.toBe(1);
+            // Clear namespace
+            const cleared = await db.clearNamespace('ns1');
+            expect(cleared).toBe(2);
+            await expect(db.count('ns1')).resolves.toBe(0);
+            await expect(db.count('ns2')).resolves.toBe(1);
+            await db.shutdown();
+        });
+        it('should handle bulk operations across providers', async () => {
+            const db = await createDatabase(':memory:');
+            // Bulk insert
+            const entries = Array.from({ length: 10 }, (_, i) => createDefaultEntry({
+                key: `bulk-${i}`,
+                content: `content ${i}`,
+                namespace: 'bulk-test',
+            }));
+            await db.bulkInsert(entries);
+            await expect(db.count('bulk-test')).resolves.toBe(10);
+            // Bulk delete
+            const idsToDelete = entries.slice(0, 5).map((e) => e.id);
+            const deletedCount = await db.bulkDelete(idsToDelete);
+            expect(deletedCount).toBe(5);
+            await expect(db.count('bulk-test')).resolves.toBe(5);
+            await db.shutdown();
+        });
+    });
+    describe('Health Check', () => {
+        it('should perform health check', async () => {
+            const db = await createDatabase(':memory:');
+            const health = await db.healthCheck();
+            expect(health).toHaveProperty('status');
+            expect(health).toHaveProperty('components');
+            expect(health).toHaveProperty('timestamp');
+            expect(health).toHaveProperty('issues');
+            expect(health).toHaveProperty('recommendations');
+            expect(health.components).toHaveProperty('storage');
+            expect(health.components).toHaveProperty('index');
+            expect(health.components).toHaveProperty('cache');
+            await db.shutdown();
+        });
+    });
+    describe('Statistics', () => {
+        it('should provide backend statistics', async () => {
+            const db = await createDatabase(':memory:');
+            // Add some test data
+            const entries = [
+                createDefaultEntry({ key: 'stat1', content: 'content1', namespace: 'stats', type: 'semantic' }),
+                createDefaultEntry({ key: 'stat2', content: 'content2', namespace: 'stats', type: 'episodic' }),
+                createDefaultEntry({ key: 'stat3', content: 'content3', namespace: 'stats', type: 'semantic' }),
+            ];
+            for (const entry of entries) {
+                await db.store(entry);
+            }
+            const stats = await db.getStats();
+            expect(stats).toHaveProperty('totalEntries');
+            expect(stats).toHaveProperty('entriesByNamespace');
+            expect(stats).toHaveProperty('entriesByType');
+            expect(stats).toHaveProperty('memoryUsage');
+            expect(stats).toHaveProperty('avgQueryTime');
+            expect(stats.totalEntries).toBe(3);
+            await db.shutdown();
+        });
+    });
+    describe('Error Handling', () => {
+        it('should handle missing entries gracefully', async () => {
+            const db = await createDatabase(':memory:');
+            const entry = await db.get('non-existent-id');
+            expect(entry).toBeNull();
+            const byKey = await db.getByKey('non-existent-ns', 'non-existent-key');
+            expect(byKey).toBeNull();
+            await db.shutdown();
+        });
+        it('should handle empty queries', async () => {
+            const db = await createDatabase(':memory:');
+            const results = await db.query({
+                type: 'hybrid',
+                limit: 10,
+            });
+            expect(results).toEqual([]);
+            await db.shutdown();
+        });
+    });
+});
+//# sourceMappingURL=database-provider.test.js.map