@siftd/connect-agent 0.1.0 → 0.2.2

package/dist/core/memory-advanced.js

@@ -0,0 +1,571 @@
+/**
+ * Advanced Memory System
+ *
+ * Features:
+ * - Hierarchical memory types (episodic, semantic, procedural)
+ * - Vector embeddings for semantic search
+ * - Importance scoring with temporal decay
+ * - Memory associations/links
+ * - Reflection and consolidation
+ */
+import Database from 'better-sqlite3';
+import { LocalIndex } from 'vectra';
+import * as fs from 'fs';
+// Simple local embedding using character n-grams (no API needed)
+class LocalEmbedding {
+    dimensions = 384;
+    async embed(text) {
+        return this.computeEmbedding(text);
+    }
+    async embedBatch(texts) {
+        return texts.map(t => this.computeEmbedding(t));
+    }
+    computeEmbedding(text) {
+        const embedding = new Array(this.dimensions).fill(0);
+        const normalized = text.toLowerCase();
+        // Character trigrams
+        for (let i = 0; i < normalized.length - 2; i++) {
+            const trigram = normalized.slice(i, i + 3);
+            const hash = this.hashString(trigram);
+            const idx = Math.abs(hash) % this.dimensions;
+            embedding[idx] += 1;
+        }
+        // Word unigrams
+        const words = normalized.split(/\s+/);
+        for (const word of words) {
+            const hash = this.hashString(word);
+            const idx = Math.abs(hash) % this.dimensions;
+            embedding[idx] += 2;
+        }
+        // Normalize
+        const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
+        if (magnitude > 0) {
+            for (let i = 0; i < embedding.length; i++) {
+                embedding[i] /= magnitude;
+            }
+        }
+        return embedding;
+    }
+    hashString(str) {
+        let hash = 5381;
+        for (let i = 0; i < str.length; i++) {
+            hash = ((hash << 5) + hash) + str.charCodeAt(i);
+        }
+        return hash;
+    }
+}
+// Voyage AI embedding service
+class VoyageEmbedding {
+    apiKey;
+    model = 'voyage-2';
+    constructor(apiKey) {
+        this.apiKey = apiKey;
+    }
+    async embed(text) {
+        const results = await this.embedBatch([text]);
+        return results[0];
+    }
+    async embedBatch(texts) {
+        const response = await fetch('https://api.voyageai.com/v1/embeddings', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${this.apiKey}`
+            },
+            body: JSON.stringify({
+                model: this.model,
+                input: texts
+            })
+        });
+        if (!response.ok) {
+            throw new Error(`Voyage API error: ${response.status}`);
+        }
+        const data = await response.json();
+        return data.data.map(d => d.embedding);
+    }
+}
+export class AdvancedMemoryStore {
+    db;
+    vectorIndex = null;
+    vectorPath;
+    embedder;
+    config;
+    constructor(options = {}) {
+        const dbPath = options.dbPath || 'memories_advanced.db';
+        this.vectorPath = options.vectorPath || './memory_vectors';
+        this.db = new Database(dbPath);
+        if (options.voyageApiKey) {
+            this.embedder = new VoyageEmbedding(options.voyageApiKey);
+        }
+        else {
+            this.embedder = new LocalEmbedding();
+        }
+        this.config = {
+            maxMemories: options.maxMemories || 10000,
+            decayInterval: 3600000,
+            consolidationInterval: 86400000,
+            minImportanceThreshold: 0.1
+        };
+        this.init();
+    }
+    async init() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS memories (
+        id TEXT PRIMARY KEY,
+        type TEXT NOT NULL,
+        content TEXT NOT NULL,
+        summary TEXT,
+        source TEXT NOT NULL,
+        timestamp TEXT NOT NULL,
+        lastAccessed TEXT NOT NULL,
+        importance REAL NOT NULL DEFAULT 0.5,
+        accessCount INTEGER NOT NULL DEFAULT 0,
+        decayRate REAL NOT NULL DEFAULT 0.1,
+        associations TEXT NOT NULL DEFAULT '[]',
+        tags TEXT NOT NULL DEFAULT '[]',
+        hasEmbedding INTEGER NOT NULL DEFAULT 0
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_memory_type ON memories(type);
+      CREATE INDEX IF NOT EXISTS idx_memory_importance ON memories(importance);
+      CREATE INDEX IF NOT EXISTS idx_memory_timestamp ON memories(timestamp);
+      CREATE INDEX IF NOT EXISTS idx_memory_tags ON memories(tags);
+
+      CREATE TABLE IF NOT EXISTS reflections (
+        id TEXT PRIMARY KEY,
+        timestamp TEXT NOT NULL,
+        insights TEXT NOT NULL,
+        memoriesProcessed INTEGER NOT NULL,
+        consolidations INTEGER NOT NULL
+      );
+    `);
+        await this.initVectorIndex();
+    }
+    async initVectorIndex() {
+        if (!fs.existsSync(this.vectorPath)) {
+            fs.mkdirSync(this.vectorPath, { recursive: true });
+        }
+        this.vectorIndex = new LocalIndex(this.vectorPath);
+        if (!await this.vectorIndex.isIndexCreated()) {
+            await this.vectorIndex.createIndex();
+        }
+    }
+    async remember(content, options = {}) {
+        const id = `mem_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+        const now = new Date().toISOString();
+        const type = options.type || this.inferMemoryType(content);
+        const importance = options.importance ?? this.calculateInitialImportance(content, type);
+        const memory = {
+            id,
+            type,
+            content,
+            summary: options.summary,
+            source: options.source || 'user',
+            timestamp: now,
+            lastAccessed: now,
+            importance,
+            accessCount: 1,
+            decayRate: this.getDecayRateForType(type),
+            associations: options.associations || [],
+            tags: options.tags || this.extractTags(content),
+            hasEmbedding: false
+        };
+        const stmt = this.db.prepare(`
+      INSERT INTO memories (id, type, content, summary, source, timestamp, lastAccessed,
+                           importance, accessCount, decayRate, associations, tags, hasEmbedding)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+        stmt.run(memory.id, memory.type, memory.content, memory.summary || null, memory.source, memory.timestamp, memory.lastAccessed, memory.importance, memory.accessCount, memory.decayRate, JSON.stringify(memory.associations), JSON.stringify(memory.tags), 0);
+        await this.addEmbedding(id, content);
+        await this.createAssociations(id, content);
+        await this.pruneIfNeeded();
+        return id;
+    }
+    async search(query, options = {}) {
+        const limit = options.limit || 10;
+        if (!this.vectorIndex) {
+            return this.textSearch(query, options);
+        }
+        const queryEmbedding = await this.embedder.embed(query);
+        const results = await this.vectorIndex.queryItems(queryEmbedding, limit * 2);
+        const memories = [];
+        for (const result of results) {
+            const memory = this.getById(result.item.metadata.id);
+            if (memory) {
+                if (options.type && memory.type !== options.type)
+                    continue;
+                if (options.minImportance && memory.importance < options.minImportance)
+                    continue;
+                this.recordAccess(memory.id);
+                memories.push(memory);
+                if (memories.length >= limit)
+                    break;
+            }
+        }
+        if (options.includeAssociations) {
+            const associated = new Set();
+            for (const mem of memories) {
+                for (const assocId of mem.associations) {
+                    if (!memories.find(m => m.id === assocId)) {
+                        associated.add(assocId);
+                    }
+                }
+            }
+            for (const assocId of associated) {
+                const mem = this.getById(assocId);
+                if (mem)
+                    memories.push(mem);
+            }
+        }
+        return memories;
+    }
+    textSearch(query, options) {
+        const limit = options.limit || 10;
+        const conditions = ['1=1'];
+        const params = [];
+        if (options.type) {
+            conditions.push('type = ?');
+            params.push(options.type);
+        }
+        if (options.minImportance) {
+            conditions.push('importance >= ?');
+            params.push(options.minImportance);
+        }
+        conditions.push('(content LIKE ? OR tags LIKE ?)');
+        params.push(`%${query}%`, `%${query}%`);
+        const stmt = this.db.prepare(`
+      SELECT * FROM memories
+      WHERE ${conditions.join(' AND ')}
+      ORDER BY importance DESC, accessCount DESC
+      LIMIT ?
+    `);
+        const rows = stmt.all(...params, limit);
+        return rows.map(row => this.rowToMemory(row));
+    }
+    getById(id) {
+        const stmt = this.db.prepare('SELECT * FROM memories WHERE id = ?');
+        const row = stmt.get(id);
+        return row ? this.rowToMemory(row) : null;
+    }
+    getByType(type, limit = 50) {
+        const stmt = this.db.prepare(`
+      SELECT * FROM memories
+      WHERE type = ?
+      ORDER BY importance DESC, lastAccessed DESC
+      LIMIT ?
+    `);
+        const rows = stmt.all(type, limit);
+        return rows.map(row => this.rowToMemory(row));
+    }
+    recall(id) {
+        const memory = this.getById(id);
+        if (memory) {
+            this.recordAccess(id);
+            return this.getById(id);
+        }
+        return null;
+    }
+    async forget(id) {
+        const stmt = this.db.prepare('DELETE FROM memories WHERE id = ?');
+        const result = stmt.run(id);
+        if (this.vectorIndex) {
+            try {
+                await this.vectorIndex.deleteItem(id);
+            }
+            catch {
+                // May not exist
+            }
+        }
+        this.removeAssociations(id);
+        return result.changes > 0;
+    }
+    updateImportance(id, importance) {
+        const stmt = this.db.prepare('UPDATE memories SET importance = ? WHERE id = ?');
+        stmt.run(Math.max(0, Math.min(1, importance)), id);
+    }
+    associate(id1, id2) {
+        const mem1 = this.getById(id1);
+        const mem2 = this.getById(id2);
+        if (mem1 && mem2) {
+            if (!mem1.associations.includes(id2)) {
+                mem1.associations.push(id2);
+                this.db.prepare('UPDATE memories SET associations = ? WHERE id = ?')
+                    .run(JSON.stringify(mem1.associations), id1);
+            }
+            if (!mem2.associations.includes(id1)) {
+                mem2.associations.push(id1);
+                this.db.prepare('UPDATE memories SET associations = ? WHERE id = ?')
+                    .run(JSON.stringify(mem2.associations), id2);
+            }
+        }
+    }
+    applyDecay() {
+        const now = Date.now();
+        const memories = this.list();
+        let decayed = 0;
+        for (const memory of memories) {
+            const age = now - new Date(memory.lastAccessed).getTime();
+            const hoursSinceAccess = age / 3600000;
+            const decayFactor = Math.exp(-memory.decayRate * hoursSinceAccess / 24);
+            const newImportance = memory.importance * decayFactor;
+            const accessBoost = Math.min(0.3, memory.accessCount * 0.02);
+            const finalImportance = Math.min(1, newImportance + accessBoost);
+            if (Math.abs(finalImportance - memory.importance) > 0.01) {
+                this.updateImportance(memory.id, finalImportance);
+                decayed++;
+            }
+            if (finalImportance < this.config.minImportanceThreshold && memory.type !== 'procedural') {
+                this.forget(memory.id);
+            }
+        }
+        return decayed;
+    }
+    async reflect() {
+        const recentMemories = this.getRecent(50);
+        const insights = [];
+        let consolidations = 0;
+        const tagGroups = new Map();
+        for (const mem of recentMemories) {
+            for (const tag of mem.tags) {
+                if (!tagGroups.has(tag)) {
+                    tagGroups.set(tag, []);
+                }
+                tagGroups.get(tag).push(mem);
+            }
+        }
+        for (const [tag, mems] of tagGroups) {
+            if (mems.length >= 3) {
+                insights.push(`Recurring theme: "${tag}" appears in ${mems.length} recent memories`);
+                if (mems.length >= 5) {
+                    const summary = this.consolidateMemories(mems);
+                    if (summary) {
+                        await this.remember(summary, {
+                            type: 'semantic',
+                            source: 'reflection',
+                            importance: 0.8,
+                            tags: [tag, 'consolidated']
+                        });
+                        consolidations++;
+                    }
+                }
+            }
+        }
+        for (const mem of recentMemories) {
+            if (mem.associations.length === 0 && mem.importance > 0.6) {
+                insights.push(`Important isolated memory: "${mem.content.slice(0, 50)}..." - consider linking`);
+            }
+        }
+        const reflectionId = `ref_${Date.now()}`;
+        this.db.prepare(`
+      INSERT INTO reflections (id, timestamp, insights, memoriesProcessed, consolidations)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(reflectionId, new Date().toISOString(), JSON.stringify(insights), recentMemories.length, consolidations);
+        return {
+            insights,
+            consolidations,
+            memoriesProcessed: recentMemories.length
+        };
+    }
+    stats() {
+        const total = this.db.prepare('SELECT COUNT(*) as count FROM memories').get();
+        const byType = {
+            episodic: 0,
+            semantic: 0,
+            procedural: 0,
+            working: 0
+        };
+        const types = this.db.prepare('SELECT type, COUNT(*) as count FROM memories GROUP BY type').all();
+        for (const t of types) {
+            byType[t.type] = t.count;
+        }
+        const avgImportance = this.db.prepare('SELECT AVG(importance) as avg FROM memories').get();
+        const memories = this.list();
+        let totalAssociations = 0;
+        for (const m of memories) {
+            totalAssociations += m.associations.length;
+        }
+        const oldest = this.db.prepare('SELECT timestamp FROM memories ORDER BY timestamp ASC LIMIT 1').get();
+        const newest = this.db.prepare('SELECT timestamp FROM memories ORDER BY timestamp DESC LIMIT 1').get();
+        return {
+            total: total.count,
+            byType,
+            avgImportance: avgImportance.avg || 0,
+            totalAssociations: totalAssociations / 2,
+            oldestMemory: oldest?.timestamp || null,
+            newestMemory: newest?.timestamp || null
+        };
+    }
+    list(limit = 100) {
+        const stmt = this.db.prepare(`
+      SELECT * FROM memories
+      ORDER BY importance DESC, lastAccessed DESC
+      LIMIT ?
+    `);
+        const rows = stmt.all(limit);
+        return rows.map(row => this.rowToMemory(row));
+    }
+    getRecent(limit = 20) {
+        const stmt = this.db.prepare(`
+      SELECT * FROM memories
+      ORDER BY timestamp DESC
+      LIMIT ?
+    `);
+        const rows = stmt.all(limit);
+        return rows.map(row => this.rowToMemory(row));
+    }
+    async addEmbedding(id, content) {
+        if (!this.vectorIndex)
+            return;
+        try {
+            const embedding = await this.embedder.embed(content);
+            await this.vectorIndex.insertItem({
+                vector: embedding,
+                metadata: { id }
+            });
+            this.db.prepare('UPDATE memories SET hasEmbedding = 1 WHERE id = ?').run(id);
+        }
+        catch (error) {
+            console.error('Failed to create embedding:', error);
+        }
+    }
+    async createAssociations(id, content) {
+        const similar = await this.search(content, { limit: 5 });
+        for (const mem of similar) {
+            if (mem.id !== id) {
+                this.associate(id, mem.id);
+            }
+        }
+    }
+    removeAssociations(id) {
+        const stmt = this.db.prepare('SELECT id, associations FROM memories WHERE associations LIKE ?');
+        const rows = stmt.all(`%${id}%`);
+        for (const row of rows) {
+            const associations = JSON.parse(row.associations);
+            const filtered = associations.filter(a => a !== id);
+            this.db.prepare('UPDATE memories SET associations = ? WHERE id = ?')
+                .run(JSON.stringify(filtered), row.id);
+        }
+    }
+    recordAccess(id) {
+        const stmt = this.db.prepare(`
+      UPDATE memories
+      SET lastAccessed = ?, accessCount = accessCount + 1
+      WHERE id = ?
+    `);
+        stmt.run(new Date().toISOString(), id);
+    }
+    inferMemoryType(content) {
+        const lower = content.toLowerCase();
+        if (lower.includes('to ') && (lower.includes('run ') || lower.includes('use ') || lower.includes('do '))) {
+            return 'procedural';
+        }
+        if (lower.includes('step ') || lower.includes('first ') || lower.includes('then ')) {
+            return 'procedural';
+        }
+        if (lower.match(/\b(today|yesterday|on \w+day|last \w+|this morning)\b/)) {
+            return 'episodic';
+        }
+        if (lower.match(/\b(happened|did|said|asked|told)\b/)) {
+            return 'episodic';
+        }
+        return 'semantic';
+    }
+    calculateInitialImportance(content, type) {
+        let importance = 0.5;
+        if (type === 'procedural')
+            importance += 0.2;
+        if (type === 'semantic')
+            importance += 0.1;
+        const lower = content.toLowerCase();
+        if (lower.includes('important') || lower.includes('remember'))
+            importance += 0.15;
+        if (lower.includes('always') || lower.includes('never'))
+            importance += 0.1;
+        if (lower.includes('password') || lower.includes('key') || lower.includes('secret'))
+            importance += 0.2;
+        if (lower.includes('prefer') || lower.includes('like') || lower.includes('favorite'))
+            importance += 0.1;
+        return Math.min(1, importance);
+    }
+    getDecayRateForType(type) {
+        switch (type) {
+            case 'procedural': return 0.02;
+            case 'semantic': return 0.05;
+            case 'episodic': return 0.15;
+            case 'working': return 0.5;
+            default: return 0.1;
+        }
+    }
+    extractTags(content) {
+        const tags = [];
+        const lower = content.toLowerCase();
+        const patterns = [
+            /\b(preference|setting|config|password|key|api|token)\b/g,
+            /\b(project|file|folder|code|script)\b/g,
+            /\b(schedule|reminder|task|todo)\b/g,
+            /\b(user|name|email|phone)\b/g
+        ];
+        for (const pattern of patterns) {
+            const matches = lower.match(pattern);
+            if (matches) {
+                tags.push(...matches);
+            }
+        }
+        return [...new Set(tags)];
+    }
+    consolidateMemories(memories) {
+        if (memories.length < 3)
+            return null;
+        const contents = memories.map(m => m.content);
+        const commonWords = this.findCommonWords(contents);
+        if (commonWords.length > 0) {
+            return `Consolidated insight: Topics frequently discussed include: ${commonWords.slice(0, 5).join(', ')}. Based on ${memories.length} related memories.`;
+        }
+        return null;
+    }
+    findCommonWords(texts) {
+        const wordCounts = new Map();
+        const stopWords = new Set(['the', 'a', 'an', 'is', 'are', 'was', 'were', 'to', 'for', 'of', 'and', 'in', 'on', 'at']);
+        for (const text of texts) {
+            const words = text.toLowerCase().split(/\s+/);
+            const seen = new Set();
+            for (const word of words) {
+                if (word.length > 3 && !stopWords.has(word) && !seen.has(word)) {
+                    seen.add(word);
+                    wordCounts.set(word, (wordCounts.get(word) || 0) + 1);
+                }
+            }
+        }
+        return Array.from(wordCounts.entries())
+            .filter(([_, count]) => count >= 2)
+            .sort((a, b) => b[1] - a[1])
+            .map(([word]) => word);
+    }
+    async pruneIfNeeded() {
+        const count = this.db.prepare('SELECT COUNT(*) as count FROM memories').get().count;
+        if (count > this.config.maxMemories) {
+            const toRemove = count - this.config.maxMemories;
+            const stmt = this.db.prepare(`
+        SELECT id FROM memories
+        WHERE type != 'procedural'
+        ORDER BY importance ASC, accessCount ASC
+        LIMIT ?
+      `);
+            const rows = stmt.all(toRemove);
+            for (const row of rows) {
+                await this.forget(row.id);
+            }
+        }
+    }
+    rowToMemory(row) {
+        return {
+            ...row,
+            associations: JSON.parse(row.associations),
+            tags: JSON.parse(row.tags),
+            hasEmbedding: Boolean(row.hasEmbedding)
+        };
+    }
+    close() {
+        this.db.close();
+    }
+}

package/dist/core/scheduler.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Task Scheduler
+ * Schedule tasks for future execution with notifications
+ */
+export interface ScheduledTask {
+    id: string;
+    task: string;
+    schedule: ScheduleSpec;
+    chatId: string;
+    enabled: boolean;
+    created: string;
+    lastRun?: string;
+    nextRun?: string;
+    runCount: number;
+}
+export type ScheduleSpec = {
+    type: 'once';
+    at: string;
+} | {
+    type: 'delay';
+    ms: number;
+} | {
+    type: 'cron';
+    pattern: string;
+} | {
+    type: 'interval';
+    ms: number;
+};
+export type NotifyCallback = (chatId: string, message: string) => Promise<void>;
+export declare class TaskScheduler {
+    private tasks;
+    private activeTimers;
+    private notifyCallback?;
+    private taskExecutor?;
+    private dataPath;
+    constructor(dataPath?: string);
+    /**
+     * Set the notification callback for sending messages
+     */
+    setNotifyCallback(callback: NotifyCallback): void;
+    /**
+     * Set the task executor callback
+     */
+    setTaskExecutor(executor: (task: string, chatId: string) => Promise<string>): void;
+    /**
+     * Parse natural language schedule
+     */
+    parseSchedule(when: string): ScheduleSpec | null;
+    /**
+     * Schedule a task
+     */
+    schedule(task: string, when: string | ScheduleSpec, chatId: string): string;
+    /**
+     * Calculate next run time
+     */
+    private calculateNextRun;
+    /**
+     * Activate a task's timer
+     */
+    private activateTask;
+    /**
+     * Deactivate a task's timer
+     */
+    private deactivateTask;
+    /**
+     * Cancel a task
+     */
+    cancel(taskId: string): boolean;
+    /**
+     * Pause a task
+     */
+    pause(taskId: string): boolean;
+    /**
+     * Resume a task
+     */
+    resume(taskId: string): boolean;
+    /**
+     * List all tasks
+     */
+    list(): ScheduledTask[];
+    /**
+     * Get a specific task
+     */
+    get(taskId: string): ScheduledTask | undefined;
+    /**
+     * Format schedule for display
+     */
+    formatSchedule(schedule: ScheduleSpec): string;
+    /**
+     * Save tasks to disk
+     */
+    private saveTasks;
+    /**
+     * Load tasks from disk
+     */
+    private loadTasks;
+    /**
+     * Stop all timers (for shutdown)
+     */
+    shutdown(): void;
+}