moflo 4.8.19 → 4.8.20

package/src/@claude-flow/memory/dist/sqljs-backend.js

@@ -0,0 +1,600 @@
+/**
+ * SqlJsBackend - Pure JavaScript SQLite for Windows compatibility
+ *
+ * Pure JavaScript/WASM SQLite backend that works everywhere
+ * without native compilation.
+ *
+ * @module v3/memory/sqljs-backend
+ */
+import { EventEmitter } from 'node:events';
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import initSqlJs from 'sql.js';
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG = {
+    databasePath: ':memory:',
+    optimize: true,
+    defaultNamespace: 'default',
+    maxEntries: 1000000,
+    verbose: false,
+    autoPersistInterval: 5000, // 5 seconds
+};
+/**
+ * SqlJs Backend for Cross-Platform Memory Storage
+ *
+ * Provides:
+ * - Pure JavaScript/WASM implementation (no native compilation)
+ * - Windows, macOS, Linux compatibility
+ * - Same SQL interface as native SQLite
+ * - In-memory with periodic disk persistence
+ */
+export class SqlJsBackend extends EventEmitter {
+    config;
+    db = null;
+    initialized = false;
+    persistTimer = null;
+    SQL = null;
+    // Performance tracking
+    stats = {
+        queryCount: 0,
+        totalQueryTime: 0,
+        writeCount: 0,
+        totalWriteTime: 0,
+    };
+    constructor(config = {}) {
+        super();
+        this.config = { ...DEFAULT_CONFIG, ...config };
+    }
+    /**
+     * Initialize the SqlJs backend
+     */
+    async initialize() {
+        if (this.initialized)
+            return;
+        // Load sql.js WASM
+        this.SQL = await initSqlJs({
+            locateFile: this.config.wasmPath
+                ? () => this.config.wasmPath
+                : (file) => `https://sql.js.org/dist/${file}`,
+        });
+        // Load existing database if exists and not in-memory
+        if (this.config.databasePath !== ':memory:' && existsSync(this.config.databasePath)) {
+            const buffer = readFileSync(this.config.databasePath);
+            this.db = new this.SQL.Database(new Uint8Array(buffer));
+            if (this.config.verbose) {
+                console.log(`[SqlJsBackend] Loaded database from ${this.config.databasePath}`);
+            }
+        }
+        else {
+            // Create new database
+            this.db = new this.SQL.Database();
+            if (this.config.verbose) {
+                console.log('[SqlJsBackend] Created new in-memory database');
+            }
+        }
+        // Create schema
+        this.createSchema();
+        // Set up auto-persist if enabled
+        if (this.config.autoPersistInterval > 0 && this.config.databasePath !== ':memory:') {
+            this.persistTimer = setInterval(() => {
+                this.persist().catch((err) => {
+                    this.emit('error', { operation: 'auto-persist', error: err });
+                });
+            }, this.config.autoPersistInterval);
+        }
+        this.initialized = true;
+        this.emit('initialized');
+    }
+    /**
+     * Shutdown the backend
+     */
+    async shutdown() {
+        if (!this.initialized || !this.db)
+            return;
+        // Stop auto-persist timer
+        if (this.persistTimer) {
+            clearInterval(this.persistTimer);
+            this.persistTimer = null;
+        }
+        // Final persist before shutdown
+        if (this.config.databasePath !== ':memory:') {
+            await this.persist();
+        }
+        this.db.close();
+        this.db = null;
+        this.initialized = false;
+        this.emit('shutdown');
+    }
+    /**
+     * Create database schema
+     */
+    createSchema() {
+        if (!this.db)
+            return;
+        // Main entries table
+        this.db.run(`
+      CREATE TABLE IF NOT EXISTS memory_entries (
+        id TEXT PRIMARY KEY,
+        key TEXT NOT NULL,
+        content TEXT NOT NULL,
+        embedding BLOB,
+        type TEXT NOT NULL,
+        namespace TEXT NOT NULL,
+        tags TEXT NOT NULL,
+        metadata TEXT NOT NULL,
+        owner_id TEXT,
+        access_level TEXT NOT NULL,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        expires_at INTEGER,
+        version INTEGER NOT NULL DEFAULT 1,
+        "references" TEXT NOT NULL,
+        access_count INTEGER NOT NULL DEFAULT 0,
+        last_accessed_at INTEGER NOT NULL
+      )
+    `);
+        // Indexes for performance
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_namespace ON memory_entries(namespace)');
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_key ON memory_entries(key)');
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_type ON memory_entries(type)');
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_created_at ON memory_entries(created_at)');
+        this.db.run('CREATE INDEX IF NOT EXISTS idx_expires_at ON memory_entries(expires_at)');
+        this.db.run('CREATE UNIQUE INDEX IF NOT EXISTS idx_namespace_key ON memory_entries(namespace, key)');
+        if (this.config.verbose) {
+            console.log('[SqlJsBackend] Schema created successfully');
+        }
+    }
+    /**
+     * Store a memory entry
+     */
+    async store(entry) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        const stmt = `
+      INSERT OR REPLACE INTO memory_entries (
+        id, key, content, embedding, type, namespace, tags, metadata,
+        owner_id, access_level, created_at, updated_at, expires_at,
+        version, "references", access_count, last_accessed_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `;
+        const embeddingBuffer = entry.embedding
+            ? Buffer.from(entry.embedding.buffer)
+            : null;
+        this.db.run(stmt, [
+            entry.id,
+            entry.key,
+            entry.content,
+            embeddingBuffer,
+            entry.type,
+            entry.namespace,
+            JSON.stringify(entry.tags),
+            JSON.stringify(entry.metadata),
+            entry.ownerId || null,
+            entry.accessLevel,
+            entry.createdAt,
+            entry.updatedAt,
+            entry.expiresAt || null,
+            entry.version,
+            JSON.stringify(entry.references),
+            entry.accessCount,
+            entry.lastAccessedAt,
+        ]);
+        const duration = performance.now() - startTime;
+        this.stats.writeCount++;
+        this.stats.totalWriteTime += duration;
+        this.emit('entry:stored', { entry, duration });
+    }
+    /**
+     * Retrieve a memory entry by ID
+     */
+    async get(id) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE id = ?');
+        const row = stmt.getAsObject([id]);
+        stmt.free();
+        const duration = performance.now() - startTime;
+        this.stats.queryCount++;
+        this.stats.totalQueryTime += duration;
+        if (!row || Object.keys(row).length === 0) {
+            return null;
+        }
+        const entry = this.rowToEntry(row);
+        // Update access tracking
+        this.updateAccessTracking(id);
+        this.emit('entry:retrieved', { id, duration });
+        return entry;
+    }
+    /**
+     * Retrieve a memory entry by key within a namespace
+     */
+    async getByKey(namespace, key) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        const stmt = this.db.prepare('SELECT * FROM memory_entries WHERE namespace = ? AND key = ?');
+        const row = stmt.getAsObject([namespace, key]);
+        stmt.free();
+        const duration = performance.now() - startTime;
+        this.stats.queryCount++;
+        this.stats.totalQueryTime += duration;
+        if (!row || Object.keys(row).length === 0) {
+            return null;
+        }
+        const entry = this.rowToEntry(row);
+        // Update access tracking
+        this.updateAccessTracking(entry.id);
+        this.emit('entry:retrieved', { namespace, key, duration });
+        return entry;
+    }
+    /**
+     * Update a memory entry
+     */
+    async update(id, updateData) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        // Get existing entry
+        const existing = await this.get(id);
+        if (!existing)
+            return null;
+        // Merge updates
+        const updated = {
+            ...existing,
+            ...updateData,
+            updatedAt: Date.now(),
+            version: existing.version + 1,
+        };
+        // Store updated entry
+        await this.store(updated);
+        const duration = performance.now() - startTime;
+        this.emit('entry:updated', { id, update: updateData, duration });
+        return updated;
+    }
+    /**
+     * Delete a memory entry
+     */
+    async delete(id) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        this.db.run('DELETE FROM memory_entries WHERE id = ?', [id]);
+        const duration = performance.now() - startTime;
+        this.stats.writeCount++;
+        this.stats.totalWriteTime += duration;
+        this.emit('entry:deleted', { id, duration });
+        return true;
+    }
+    /**
+     * Query memory entries
+     */
+    async query(query) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        let sql = 'SELECT * FROM memory_entries WHERE 1=1';
+        const params = [];
+        // Namespace filter
+        if (query.namespace) {
+            sql += ' AND namespace = ?';
+            params.push(query.namespace);
+        }
+        // Type filter
+        if (query.memoryType) {
+            sql += ' AND type = ?';
+            params.push(query.memoryType);
+        }
+        // Owner filter
+        if (query.ownerId) {
+            sql += ' AND owner_id = ?';
+            params.push(query.ownerId);
+        }
+        // Access level filter
+        if (query.accessLevel) {
+            sql += ' AND access_level = ?';
+            params.push(query.accessLevel);
+        }
+        // Key filters
+        if (query.key) {
+            sql += ' AND key = ?';
+            params.push(query.key);
+        }
+        else if (query.keyPrefix) {
+            sql += ' AND key LIKE ?';
+            params.push(query.keyPrefix + '%');
+        }
+        // Time range filters
+        if (query.createdAfter) {
+            sql += ' AND created_at >= ?';
+            params.push(query.createdAfter);
+        }
+        if (query.createdBefore) {
+            sql += ' AND created_at <= ?';
+            params.push(query.createdBefore);
+        }
+        if (query.updatedAfter) {
+            sql += ' AND updated_at >= ?';
+            params.push(query.updatedAfter);
+        }
+        if (query.updatedBefore) {
+            sql += ' AND updated_at <= ?';
+            params.push(query.updatedBefore);
+        }
+        // Expiration filter
+        if (!query.includeExpired) {
+            sql += ' AND (expires_at IS NULL OR expires_at > ?)';
+            params.push(Date.now());
+        }
+        // Ordering and pagination
+        sql += ' ORDER BY created_at DESC';
+        if (query.limit) {
+            sql += ' LIMIT ?';
+            params.push(query.limit);
+        }
+        if (query.offset) {
+            sql += ' OFFSET ?';
+            params.push(query.offset);
+        }
+        const stmt = this.db.prepare(sql);
+        if (params.length > 0) {
+            stmt.bind(params);
+        }
+        const results = [];
+        while (stmt.step()) {
+            const row = stmt.getAsObject();
+            const entry = this.rowToEntry(row);
+            // Tag filtering (post-query since tags are JSON)
+            if (query.tags && query.tags.length > 0) {
+                const hasAllTags = query.tags.every((tag) => entry.tags.includes(tag));
+                if (!hasAllTags)
+                    continue;
+            }
+            // Metadata filtering (post-query since metadata is JSON)
+            if (query.metadata) {
+                const matchesMetadata = Object.entries(query.metadata).every(([key, value]) => entry.metadata[key] === value);
+                if (!matchesMetadata)
+                    continue;
+            }
+            results.push(entry);
+        }
+        stmt.free();
+        const duration = performance.now() - startTime;
+        this.stats.queryCount++;
+        this.stats.totalQueryTime += duration;
+        this.emit('query:executed', { query, resultCount: results.length, duration });
+        return results;
+    }
+    /**
+     * Semantic vector search (limited without vector index)
+     */
+    async search(embedding, options) {
+        this.ensureInitialized();
+        // Get all entries with embeddings
+        const entries = await this.query({
+            type: 'hybrid',
+            limit: options.filters?.limit || 1000,
+        });
+        // Calculate cosine similarity for each entry
+        const results = [];
+        for (const entry of entries) {
+            if (!entry.embedding)
+                continue;
+            const similarity = this.cosineSimilarity(embedding, entry.embedding);
+            if (options.threshold && similarity < options.threshold) {
+                continue;
+            }
+            results.push({
+                entry,
+                score: similarity,
+                distance: 1 - similarity,
+            });
+        }
+        // Sort by score descending
+        results.sort((a, b) => b.score - a.score);
+        // Return top k results
+        return results.slice(0, options.k);
+    }
+    /**
+     * Bulk insert entries
+     */
+    async bulkInsert(entries) {
+        this.ensureInitialized();
+        for (const entry of entries) {
+            await this.store(entry);
+        }
+        this.emit('bulk:inserted', { count: entries.length });
+    }
+    /**
+     * Bulk delete entries
+     */
+    async bulkDelete(ids) {
+        this.ensureInitialized();
+        let count = 0;
+        for (const id of ids) {
+            const success = await this.delete(id);
+            if (success)
+                count++;
+        }
+        this.emit('bulk:deleted', { count });
+        return count;
+    }
+    /**
+     * Get entry count
+     */
+    async count(namespace) {
+        this.ensureInitialized();
+        let sql = 'SELECT COUNT(*) as count FROM memory_entries';
+        const params = [];
+        if (namespace) {
+            sql += ' WHERE namespace = ?';
+            params.push(namespace);
+        }
+        const stmt = this.db.prepare(sql);
+        const row = stmt.getAsObject(params);
+        stmt.free();
+        return row.count || 0;
+    }
+    /**
+     * List all namespaces
+     */
+    async listNamespaces() {
+        this.ensureInitialized();
+        const stmt = this.db.prepare('SELECT DISTINCT namespace FROM memory_entries');
+        const namespaces = [];
+        while (stmt.step()) {
+            const row = stmt.getAsObject();
+            namespaces.push(row.namespace);
+        }
+        stmt.free();
+        return namespaces;
+    }
+    /**
+     * Clear all entries in a namespace
+     */
+    async clearNamespace(namespace) {
+        this.ensureInitialized();
+        const countBefore = await this.count(namespace);
+        this.db.run('DELETE FROM memory_entries WHERE namespace = ?', [namespace]);
+        this.emit('namespace:cleared', { namespace, count: countBefore });
+        return countBefore;
+    }
+    /**
+     * Get backend statistics
+     */
+    async getStats() {
+        this.ensureInitialized();
+        const total = await this.count();
+        // Count by namespace
+        const entriesByNamespace = {};
+        const namespaces = await this.listNamespaces();
+        for (const ns of namespaces) {
+            entriesByNamespace[ns] = await this.count(ns);
+        }
+        // Count by type
+        const entriesByType = {};
+        const types = ['episodic', 'semantic', 'procedural', 'working', 'cache'];
+        for (const type of types) {
+            const stmt = this.db.prepare('SELECT COUNT(*) as count FROM memory_entries WHERE type = ?');
+            const row = stmt.getAsObject([type]);
+            stmt.free();
+            entriesByType[type] = row.count || 0;
+        }
+        return {
+            totalEntries: total,
+            entriesByNamespace,
+            entriesByType,
+            memoryUsage: this.estimateMemoryUsage(),
+            avgQueryTime: this.stats.queryCount > 0 ? this.stats.totalQueryTime / this.stats.queryCount : 0,
+            avgSearchTime: 0, // Not tracked separately
+        };
+    }
+    /**
+     * Perform health check
+     */
+    async healthCheck() {
+        const issues = [];
+        const recommendations = [];
+        // Storage health
+        const storageStart = performance.now();
+        const storageHealthy = this.db !== null;
+        const storageLatency = performance.now() - storageStart;
+        if (!storageHealthy) {
+            issues.push('Database not initialized');
+        }
+        // Index health (sql.js doesn't have native vector index)
+        const indexHealth = {
+            status: 'healthy',
+            latency: 0,
+            message: 'No vector index (brute-force search)',
+        };
+        recommendations.push('Consider using AgentDB with HNSW for faster vector search');
+        // Cache health (not applicable for sql.js)
+        const cacheHealth = {
+            status: 'healthy',
+            latency: 0,
+            message: 'No separate cache layer',
+        };
+        const status = issues.length === 0 ? 'healthy' : 'degraded';
+        return {
+            status,
+            components: {
+                storage: {
+                    status: storageHealthy ? 'healthy' : 'unhealthy',
+                    latency: storageLatency,
+                },
+                index: indexHealth,
+                cache: cacheHealth,
+            },
+            timestamp: Date.now(),
+            issues,
+            recommendations,
+        };
+    }
+    /**
+     * Persist changes to disk (sql.js is in-memory, needs explicit save)
+     */
+    async persist() {
+        if (!this.db || this.config.databasePath === ':memory:') {
+            return;
+        }
+        const data = this.db.export();
+        const buffer = Buffer.from(data);
+        writeFileSync(this.config.databasePath, buffer);
+        if (this.config.verbose) {
+            console.log(`[SqlJsBackend] Persisted ${buffer.length} bytes to ${this.config.databasePath}`);
+        }
+        this.emit('persisted', { size: buffer.length, path: this.config.databasePath });
+    }
+    // ===== Helper Methods =====
+    ensureInitialized() {
+        if (!this.initialized || !this.db) {
+            throw new Error('SqlJsBackend not initialized. Call initialize() first.');
+        }
+    }
+    rowToEntry(row) {
+        return {
+            id: row.id,
+            key: row.key,
+            content: row.content,
+            embedding: row.embedding
+                ? new Float32Array(new Uint8Array(row.embedding).buffer)
+                : undefined,
+            type: row.type,
+            namespace: row.namespace,
+            tags: JSON.parse(row.tags),
+            metadata: JSON.parse(row.metadata),
+            ownerId: row.owner_id,
+            accessLevel: row.access_level,
+            createdAt: row.created_at,
+            updatedAt: row.updated_at,
+            expiresAt: row.expires_at,
+            version: row.version,
+            references: JSON.parse(row.references),
+            accessCount: row.access_count,
+            lastAccessedAt: row.last_accessed_at,
+        };
+    }
+    updateAccessTracking(id) {
+        if (!this.db)
+            return;
+        this.db.run('UPDATE memory_entries SET access_count = access_count + 1, last_accessed_at = ? WHERE id = ?', [Date.now(), id]);
+    }
+    cosineSimilarity(a, b) {
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        if (normA === 0 || normB === 0)
+            return 0;
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    estimateMemoryUsage() {
+        if (!this.db)
+            return 0;
+        // Export to get size
+        const data = this.db.export();
+        return data.length;
+    }
+}
+//# sourceMappingURL=sqljs-backend.js.map