@siftd/connect-agent 0.2.10 → 0.2.12

package/dist/core/memory-interface.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Memory Store Interface
+ *
+ * Common interface for memory backends (SQLite, Postgres, etc.)
+ */
+export type MemoryType = 'episodic' | 'semantic' | 'procedural' | 'working';
+export interface Memory {
+    id: string;
+    type: MemoryType;
+    content: string;
+    summary?: string;
+    source: string;
+    timestamp: string;
+    lastAccessed: string;
+    importance: number;
+    accessCount: number;
+    decayRate: number;
+    associations: string[];
+    tags: string[];
+    hasEmbedding: boolean;
+}
+export interface MemorySearchOptions {
+    type?: MemoryType;
+    limit?: number;
+    minImportance?: number;
+}
+export interface MemoryStore {
+    /**
+     * Store a new memory
+     */
+    remember(content: string, options?: {
+        type?: MemoryType;
+        source?: string;
+        importance?: number;
+        tags?: string[];
+    }): Promise<string>;
+    /**
+     * Search memories using semantic similarity
+     */
+    search(query: string, options?: MemorySearchOptions): Promise<Memory[]>;
+    /**
+     * Get a specific memory by ID
+     */
+    get(id: string): Promise<Memory | null>;
+    /**
+     * Delete a memory
+     */
+    forget(id: string): Promise<boolean>;
+    /**
+     * Boost importance of a memory
+     */
+    reinforce(id: string, boost?: number): Promise<void>;
+    /**
+     * Get memory statistics
+     */
+    stats(): {
+        total: number;
+        byType: Record<string, number>;
+        avgImportance: number;
+        totalAssociations: number;
+    };
+    /**
+     * Close the store and release resources
+     */
+    close(): void | Promise<void>;
+}

package/dist/core/memory-interface.js

@@ -0,0 +1,6 @@
+/**
+ * Memory Store Interface
+ *
+ * Common interface for memory backends (SQLite, Postgres, etc.)
+ */
+export {};

package/dist/core/memory-postgres.d.ts

@@ -0,0 +1,109 @@
+/**
+ * PostgreSQL Memory Store with pgvector
+ *
+ * Cloud-native memory backend for Railway deployments.
+ * Uses pgvector extension for efficient vector similarity search.
+ *
+ * Features:
+ * - All AdvancedMemoryStore features
+ * - Scalable Postgres backend
+ * - pgvector for vector similarity
+ * - Shared across multiple agent instances
+ */
+export type MemoryType = 'episodic' | 'semantic' | 'procedural' | 'working';
+export interface Memory {
+    id: string;
+    type: MemoryType;
+    content: string;
+    summary?: string;
+    source: string;
+    timestamp: string;
+    lastAccessed: string;
+    importance: number;
+    accessCount: number;
+    decayRate: number;
+    associations: string[];
+    tags: string[];
+    hasEmbedding: boolean;
+}
+interface PostgresMemoryConfig {
+    connectionString: string;
+    userId?: string;
+    dimensions?: number;
+}
+export declare class PostgresMemoryStore {
+    private pool;
+    private userId;
+    private dimensions;
+    private initialized;
+    constructor(config: PostgresMemoryConfig);
+    /**
+     * Initialize database schema
+     */
+    initialize(): Promise<void>;
+    /**
+     * Store a memory
+     */
+    remember(content: string, options?: {
+        type?: MemoryType;
+        source?: string;
+        importance?: number;
+        tags?: string[];
+    }): Promise<string>;
+    /**
+     * Search memories using vector similarity
+     */
+    search(query: string, options?: {
+        type?: MemoryType;
+        limit?: number;
+        minImportance?: number;
+    }): Promise<Memory[]>;
+    /**
+     * Get memory by ID
+     */
+    get(id: string): Promise<Memory | null>;
+    /**
+     * Delete a memory
+     */
+    forget(id: string): Promise<boolean>;
+    /**
+     * Update memory importance
+     */
+    reinforce(id: string, boost?: number): Promise<void>;
+    /**
+     * Get memory statistics
+     */
+    stats(): {
+        total: number;
+        byType: Record<string, number>;
+        avgImportance: number;
+        totalAssociations: number;
+    };
+    private _cachedStats;
+    /**
+     * Update stats cache (call periodically)
+     */
+    updateStatsCache(): Promise<void>;
+    /**
+     * Apply temporal decay to all memories
+     */
+    applyDecay(): Promise<void>;
+    /**
+     * Cleanup old low-importance memories
+     */
+    cleanup(options?: {
+        maxAge?: number;
+        minImportance?: number;
+    }): Promise<number>;
+    /**
+     * Close the connection pool
+     */
+    close(): Promise<void>;
+    private inferType;
+    private calculateImportance;
+}
+/**
+ * Check if Postgres is available and configured
+ */
+export declare function isPostgresConfigured(): boolean;
+export {};

package/dist/core/memory-postgres.js

@@ -0,0 +1,374 @@
+/**
+ * PostgreSQL Memory Store with pgvector
+ *
+ * Cloud-native memory backend for Railway deployments.
+ * Uses pgvector extension for efficient vector similarity search.
+ *
+ * Features:
+ * - All AdvancedMemoryStore features
+ * - Scalable Postgres backend
+ * - pgvector for vector similarity
+ * - Shared across multiple agent instances
+ */
+import pg from 'pg';
+const { Pool } = pg;
+// Simple local embedding using character n-grams (same as SQLite version)
+function computeEmbedding(text, dimensions = 384) {
+    const embedding = new Array(dimensions).fill(0);
+    const normalized = text.toLowerCase();
+    // Character trigrams
+    for (let i = 0; i < normalized.length - 2; i++) {
+        const trigram = normalized.slice(i, i + 3);
+        const hash = hashString(trigram);
+        const idx = Math.abs(hash) % dimensions;
+        embedding[idx] += 1;
+    }
+    // Word unigrams
+    const words = normalized.split(/\s+/);
+    for (const word of words) {
+        const hash = hashString(word);
+        const idx = Math.abs(hash) % dimensions;
+        embedding[idx] += 2;
+    }
+    // Normalize
+    const magnitude = Math.sqrt(embedding.reduce((sum, val) => sum + val * val, 0));
+    if (magnitude > 0) {
+        for (let i = 0; i < embedding.length; i++) {
+            embedding[i] /= magnitude;
+        }
+    }
+    return embedding;
+}
+function hashString(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = ((hash << 5) - hash) + char;
+        hash = hash & hash;
+    }
+    return hash;
+}
+export class PostgresMemoryStore {
+    pool;
+    userId;
+    dimensions;
+    initialized = false;
+    constructor(config) {
+        this.pool = new Pool({
+            connectionString: config.connectionString,
+            ssl: { rejectUnauthorized: false } // Required for Railway
+        });
+        this.userId = config.userId || 'default';
+        this.dimensions = config.dimensions || 384;
+    }
+    /**
+     * Initialize database schema
+     */
+    async initialize() {
+        if (this.initialized)
+            return;
+        const client = await this.pool.connect();
+        try {
+            // Enable pgvector extension
+            await client.query('CREATE EXTENSION IF NOT EXISTS vector');
+            // Create memories table
+            await client.query(`
+        CREATE TABLE IF NOT EXISTS memories (
+          id TEXT PRIMARY KEY,
+          user_id TEXT NOT NULL,
+          type TEXT NOT NULL,
+          content TEXT NOT NULL,
+          summary TEXT,
+          source TEXT,
+          timestamp TIMESTAMPTZ DEFAULT NOW(),
+          last_accessed TIMESTAMPTZ DEFAULT NOW(),
+          importance REAL DEFAULT 0.5,
+          access_count INTEGER DEFAULT 0,
+          decay_rate REAL DEFAULT 0.01,
+          associations TEXT[] DEFAULT '{}',
+          tags TEXT[] DEFAULT '{}',
+          embedding vector(${this.dimensions}),
+          created_at TIMESTAMPTZ DEFAULT NOW()
+        )
+      `);
+            // Create indexes
+            await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_memories_user_id ON memories(user_id)
+      `);
+            await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type)
+      `);
+            await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_memories_importance ON memories(importance DESC)
+      `);
+            // Create vector similarity index (IVFFlat for performance)
+            await client.query(`
+        CREATE INDEX IF NOT EXISTS idx_memories_embedding
+        ON memories USING ivfflat (embedding vector_cosine_ops)
+        WITH (lists = 100)
+      `).catch(() => {
+                // IVFFlat requires at least 100 rows, fall back to exact search
+                console.log('[POSTGRES] IVFFlat index skipped (will use exact search)');
+            });
+            this.initialized = true;
+            console.log('[POSTGRES] Memory store initialized');
+        }
+        finally {
+            client.release();
+        }
+    }
+    /**
+     * Store a memory
+     */
+    async remember(content, options) {
+        await this.initialize();
+        const id = `mem_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+        const type = options?.type || this.inferType(content);
+        const importance = options?.importance ?? this.calculateImportance(content);
+        const embedding = computeEmbedding(content, this.dimensions);
+        await this.pool.query(`
+      INSERT INTO memories (id, user_id, type, content, source, importance, tags, embedding)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+    `, [
+            id,
+            this.userId,
+            type,
+            content,
+            options?.source || 'user',
+            importance,
+            options?.tags || [],
+            `[${embedding.join(',')}]`
+        ]);
+        console.log(`[POSTGRES] Stored memory ${id} (${type}, importance=${importance.toFixed(2)})`);
+        return id;
+    }
+    /**
+     * Search memories using vector similarity
+     */
+    async search(query, options) {
+        await this.initialize();
+        const embedding = computeEmbedding(query, this.dimensions);
+        const limit = options?.limit || 10;
+        const minImportance = options?.minImportance || 0;
+        let sql = `
+      SELECT
+        id, type, content, summary, source, timestamp, last_accessed,
+        importance, access_count, decay_rate, associations, tags,
+        1 - (embedding <=> $1::vector) as similarity
+      FROM memories
+      WHERE user_id = $2
+        AND importance >= $3
+    `;
+        const params = [
+            `[${embedding.join(',')}]`,
+            this.userId,
+            minImportance
+        ];
+        if (options?.type) {
+            sql += ` AND type = $${params.length + 1}`;
+            params.push(options.type);
+        }
+        sql += ` ORDER BY similarity DESC LIMIT $${params.length + 1}`;
+        params.push(limit);
+        const result = await this.pool.query(sql, params);
+        // Update access counts for retrieved memories
+        if (result.rows.length > 0) {
+            const ids = result.rows.map(r => r.id);
+            await this.pool.query(`
+        UPDATE memories
+        SET access_count = access_count + 1,
+            last_accessed = NOW()
+        WHERE id = ANY($1)
+      `, [ids]);
+        }
+        return result.rows.map(row => ({
+            id: row.id,
+            type: row.type,
+            content: row.content,
+            summary: row.summary,
+            source: row.source,
+            timestamp: row.timestamp.toISOString(),
+            lastAccessed: row.last_accessed.toISOString(),
+            importance: row.importance,
+            accessCount: row.access_count,
+            decayRate: row.decay_rate,
+            associations: row.associations || [],
+            tags: row.tags || [],
+            hasEmbedding: true
+        }));
+    }
+    /**
+     * Get memory by ID
+     */
+    async get(id) {
+        await this.initialize();
+        const result = await this.pool.query(`
+      SELECT * FROM memories WHERE id = $1 AND user_id = $2
+    `, [id, this.userId]);
+        if (result.rows.length === 0)
+            return null;
+        const row = result.rows[0];
+        return {
+            id: row.id,
+            type: row.type,
+            content: row.content,
+            summary: row.summary,
+            source: row.source,
+            timestamp: row.timestamp.toISOString(),
+            lastAccessed: row.last_accessed.toISOString(),
+            importance: row.importance,
+            accessCount: row.access_count,
+            decayRate: row.decay_rate,
+            associations: row.associations || [],
+            tags: row.tags || [],
+            hasEmbedding: true
+        };
+    }
+    /**
+     * Delete a memory
+     */
+    async forget(id) {
+        await this.initialize();
+        const result = await this.pool.query(`
+      DELETE FROM memories WHERE id = $1 AND user_id = $2
+    `, [id, this.userId]);
+        return (result.rowCount ?? 0) > 0;
+    }
+    /**
+     * Update memory importance
+     */
+    async reinforce(id, boost = 0.1) {
+        await this.initialize();
+        await this.pool.query(`
+      UPDATE memories
+      SET importance = LEAST(1.0, importance + $1)
+      WHERE id = $2 AND user_id = $3
+    `, [boost, id, this.userId]);
+    }
+    /**
+     * Get memory statistics
+     */
+    stats() {
+        // This is synchronous in the interface, but we need async for Postgres
+        // Return cached stats or default values
+        return this._cachedStats || {
+            total: 0,
+            byType: { episodic: 0, semantic: 0, procedural: 0, working: 0 },
+            avgImportance: 0.5,
+            totalAssociations: 0
+        };
+    }
+    _cachedStats = null;
+    /**
+     * Update stats cache (call periodically)
+     */
+    async updateStatsCache() {
+        await this.initialize();
+        const result = await this.pool.query(`
+      SELECT
+        COUNT(*) as total,
+        AVG(importance) as avg_importance,
+        SUM(array_length(associations, 1)) as total_associations
+      FROM memories
+      WHERE user_id = $1
+    `, [this.userId]);
+        const typeResult = await this.pool.query(`
+      SELECT type, COUNT(*) as count
+      FROM memories
+      WHERE user_id = $1
+      GROUP BY type
+    `, [this.userId]);
+        const byType = {
+            episodic: 0, semantic: 0, procedural: 0, working: 0
+        };
+        for (const row of typeResult.rows) {
+            byType[row.type] = parseInt(row.count);
+        }
+        this._cachedStats = {
+            total: parseInt(result.rows[0].total) || 0,
+            byType,
+            avgImportance: parseFloat(result.rows[0].avg_importance) || 0.5,
+            totalAssociations: parseInt(result.rows[0].total_associations) || 0
+        };
+    }
+    /**
+     * Apply temporal decay to all memories
+     */
+    async applyDecay() {
+        await this.initialize();
+        await this.pool.query(`
+      UPDATE memories
+      SET importance = GREATEST(0.1, importance * (1 - decay_rate))
+      WHERE user_id = $1
+        AND importance > 0.1
+        AND last_accessed < NOW() - INTERVAL '1 day'
+    `, [this.userId]);
+    }
+    /**
+     * Cleanup old low-importance memories
+     */
+    async cleanup(options) {
+        await this.initialize();
+        const maxAgeDays = (options?.maxAge || 30 * 24 * 60 * 60 * 1000) / (1000 * 60 * 60 * 24);
+        const minImportance = options?.minImportance || 0.2;
+        const result = await this.pool.query(`
+      DELETE FROM memories
+      WHERE user_id = $1
+        AND importance < $2
+        AND last_accessed < NOW() - INTERVAL '1 day' * $3
+    `, [this.userId, minImportance, maxAgeDays]);
+        const count = result.rowCount ?? 0;
+        if (count > 0) {
+            console.log(`[POSTGRES] Cleaned up ${count} old memories`);
+        }
+        return count;
+    }
+    /**
+     * Close the connection pool
+     */
+    async close() {
+        await this.pool.end();
+        console.log('[POSTGRES] Connection pool closed');
+    }
+    // Helper methods
+    inferType(content) {
+        const lower = content.toLowerCase();
+        // Procedural indicators
+        if (lower.includes('how to') || lower.includes('steps to') ||
+            lower.includes('process for') || lower.includes('when doing')) {
+            return 'procedural';
+        }
+        // Episodic indicators (past tense, specific events)
+        if (lower.includes('yesterday') || lower.includes('last week') ||
+            lower.includes('we did') || lower.includes('happened')) {
+            return 'episodic';
+        }
+        // Default to semantic (facts)
+        return 'semantic';
+    }
+    calculateImportance(content) {
+        let importance = 0.5;
+        // Boost for longer, detailed content
+        if (content.length > 200)
+            importance += 0.1;
+        if (content.length > 500)
+            importance += 0.1;
+        // Boost for structured content
+        if (content.includes(':') || content.includes('-') || content.includes('•')) {
+            importance += 0.1;
+        }
+        // Boost for preference indicators
+        const lower = content.toLowerCase();
+        if (lower.includes('prefer') || lower.includes('always') ||
+            lower.includes('never') || lower.includes('important')) {
+            importance += 0.15;
+        }
+        return Math.min(1.0, importance);
+    }
+}
+/**
+ * Check if Postgres is available and configured
+ */
+export function isPostgresConfigured() {
+    return !!process.env.DATABASE_URL;
+}

package/dist/heartbeat.js

@@ -10,7 +10,7 @@ import { hostname } from 'os';
 import { createHash } from 'crypto';
 import { getServerUrl, getAgentToken, getUserId, isCloudMode } from './config.js';
 const HEARTBEAT_INTERVAL = 10000; // 10 seconds
-const VERSION = '0.2.10'; // Should match package.json
+const VERSION = '0.2.12'; // Should match package.json
 const state = {
     intervalId: null,
     runnerId: null,

package/dist/orchestrator.d.ts

@@ -11,6 +11,7 @@ export declare class MasterOrchestrator {
     private model;
     private maxTokens;
     private memory;
+    private postgresMemory;
     private scheduler;
     private indexer;
     private jobs;
@@ -19,6 +20,8 @@ export declare class MasterOrchestrator {
     private workspaceDir;
     private claudePath;
     private initialized;
+    private usingPostgres;
+    private voyageApiKey?;
     private bashTool;
     private webTools;
     private workerTools;
@@ -149,5 +152,5 @@ export declare class MasterOrchestrator {
     /**
      * Shutdown cleanly
      */
-    shutdown(): void;
+    shutdown(): Promise<void>;
 }

package/dist/orchestrator.js

@@ -8,6 +8,7 @@ import Anthropic from '@anthropic-ai/sdk';
 import { spawn, execSync } from 'child_process';
 import { existsSync } from 'fs';
 import { AdvancedMemoryStore } from './core/memory-advanced.js';
+import { PostgresMemoryStore, isPostgresConfigured } from './core/memory-postgres.js';
 import { TaskScheduler } from './core/scheduler.js';
 import { SystemIndexer } from './core/system-indexer.js';
 import { FileWatcher } from './core/file-watcher.js';
@@ -75,6 +76,7 @@ export class MasterOrchestrator {
     model;
     maxTokens;
     memory;
+    postgresMemory = null;
     scheduler;
     indexer;
     jobs = new Map();
@@ -83,6 +85,8 @@ export class MasterOrchestrator {
     workspaceDir;
     claudePath;
     initialized = false;
+    usingPostgres = false;
+    voyageApiKey;
     // New tools from whatsapp-claude
     bashTool;
     webTools;
@@ -96,17 +100,19 @@ export class MasterOrchestrator {
         this.maxTokens = options.maxTokens || 4096;
         this.userId = options.userId;
         this.workspaceDir = options.workspaceDir || process.env.HOME || '/tmp';
+        this.voyageApiKey = options.voyageApiKey;
         // Find claude binary - critical for worker delegation
         this.claudePath = this.findClaudeBinary();
         console.log(`[ORCHESTRATOR] Claude binary: ${this.claudePath}`);
-        // Initialize memory with user-specific paths
+        // Memory initialization is deferred to initialize() for async Postgres support
+        // For now, create SQLite as fallback (will be replaced if Postgres is configured)
         this.memory = new AdvancedMemoryStore({
             dbPath: `memories_${options.userId}.db`,
             vectorPath: `./memory_vectors_${options.userId}`,
             voyageApiKey: options.voyageApiKey
         });
         this.scheduler = new TaskScheduler(`scheduled_${options.userId}.json`);
-        // Create indexer (will index on first initialize call)
+        // Create indexer (will be re-initialized after memory setup)
         this.indexer = new SystemIndexer(this.memory);
         // Initialize new tools
         this.bashTool = new BashTool(this.workspaceDir);
@@ -129,14 +135,39 @@ export class MasterOrchestrator {
         if (this.initialized)
             return;
         console.log('[ORCHESTRATOR] Initializing...');
-        // Index the filesystem into memory
-        try {
-            const result = await this.indexer.indexHome();
-            console.log(`[ORCHESTRATOR] Filesystem indexed: ${result.projects} projects, ${result.directories} directories`);
+        // Check for Postgres and switch memory backend if configured
+        if (isPostgresConfigured()) {
+            try {
+                console.log('[ORCHESTRATOR] DATABASE_URL detected, using PostgreSQL + pgvector');
+                this.postgresMemory = new PostgresMemoryStore({
+                    connectionString: process.env.DATABASE_URL,
+                    userId: this.userId
+                });
+                await this.postgresMemory.initialize();
+                this.memory = this.postgresMemory;
+                this.usingPostgres = true;
+                // Re-create indexer with Postgres memory
+                // Note: SystemIndexer expects AdvancedMemoryStore, so we skip indexing for Postgres
+                console.log('[ORCHESTRATOR] Memory backend: PostgreSQL + pgvector');
+            }
+            catch (error) {
+                console.error('[ORCHESTRATOR] PostgreSQL initialization failed, falling back to SQLite:', error);
+                this.usingPostgres = false;
+            }
         }
-        catch (error) {
-            console.error('[ORCHESTRATOR] Filesystem indexing failed:', error);
-            // Continue anyway - indexing is a nice-to-have
+        else {
+            console.log('[ORCHESTRATOR] Memory backend: SQLite + vectra (local)');
+        }
+        // Index the filesystem into memory (only for SQLite - Postgres typically runs in cloud)
+        if (!this.usingPostgres) {
+            try {
+                const result = await this.indexer.indexHome();
+                console.log(`[ORCHESTRATOR] Filesystem indexed: ${result.projects} projects, ${result.directories} directories`);
+            }
+            catch (error) {
+                console.error('[ORCHESTRATOR] Filesystem indexing failed:', error);
+                // Continue anyway - indexing is a nice-to-have
+            }
         }
         this.initialized = true;
     }
@@ -1337,9 +1368,15 @@ This enables parallel workers to coordinate.`;
     /**
      * Shutdown cleanly
      */
-    shutdown() {
+    async shutdown() {
         this.fileWatcher.stopAll();
         this.scheduler.shutdown();
-        this.memory.close();
+        // Close memory store (Postgres is async, SQLite is sync)
+        if (this.usingPostgres && this.postgresMemory) {
+            await this.postgresMemory.close();
+        }
+        else if ('close' in this.memory) {
+            this.memory.close();
+        }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@siftd/connect-agent",
-  "version": "0.2.10",
+  "version": "0.2.12",
   "description": "Master orchestrator agent - control Claude Code remotely via web",
   "type": "module",
   "main": "dist/index.js",
@@ -43,6 +43,7 @@
     "conf": "^13.0.1",
     "node-cron": "^3.0.3",
     "ora": "^8.1.1",
+    "pg": "^8.13.1",
     "vectra": "^0.9.0",
     "ws": "^8.18.3"
   },
@@ -50,6 +51,7 @@
     "@types/better-sqlite3": "^7.6.12",
     "@types/node": "^22.10.2",
     "@types/node-cron": "^3.0.11",
+    "@types/pg": "^8.11.10",
     "@types/ws": "^8.18.1",
     "typescript": "^5.7.2"
   }