family-ai-agent 1.0.0

package/src/memory/knowledge-base/index.ts

@@ -0,0 +1,357 @@
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import { query, transaction } from '../../database/client.js';
+import { getOpenRouterClient } from '../../llm/openrouter-client.js';
+import { createLogger, logMemoryOperation } from '../../utils/logger.js';
+import { MemoryError } from '../../utils/errors.js';
+
+const logger = createLogger('KnowledgeBase');
+
+export interface Document {
+  id: string;
+  userId?: string;
+  filename: string;
+  fileType?: string;
+  fileSize?: number;
+  content?: string;
+  metadata: Record<string, unknown>;
+  createdAt: Date;
+}
+
+export interface DocumentChunk {
+  id: string;
+  documentId: string;
+  chunkIndex: number;
+  content: string;
+  metadata: Record<string, unknown>;
+}
+
+export interface SearchResult {
+  documentId: string;
+  chunkId: string;
+  content: string;
+  similarity: number;
+  metadata: Record<string, unknown>;
+  filename: string;
+}
+
+export class KnowledgeBase {
+  private client = getOpenRouterClient();
+  private textSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize: 1000,
+    chunkOverlap: 200,
+    separators: ['\n\n', '\n', '. ', ' ', ''],
+  });
+
+  // Add a document to the knowledge base
+  async addDocument(
+    content: string,
+    filename: string,
+    options: {
+      userId?: string;
+      fileType?: string;
+      fileSize?: number;
+      metadata?: Record<string, unknown>;
+    } = {}
+  ): Promise<string> {
+    try {
+      // Split content into chunks
+      const chunks = await this.textSplitter.splitText(content);
+      logger.debug('Document split into chunks', {
+        filename,
+        chunkCount: chunks.length,
+      });
+
+      // Generate embeddings for all chunks
+      const embeddings = await this.client.embed(chunks);
+
+      // Store document and chunks in transaction
+      const documentId = await transaction(async (client) => {
+        // Insert document
+        const docResult = await client.query<{ id: string }>(
+          `INSERT INTO documents (user_id, filename, file_type, file_size, content, metadata)
+           VALUES ($1, $2, $3, $4, $5, $6)
+           RETURNING id`,
+          [
+            options.userId ?? null,
+            filename,
+            options.fileType ?? null,
+            options.fileSize ?? null,
+            content,
+            JSON.stringify(options.metadata ?? {}),
+          ]
+        );
+
+        const docId = docResult.rows[0]?.id;
+        if (!docId) {
+          throw new MemoryError('Failed to insert document');
+        }
+
+        // Insert chunks with embeddings
+        for (let i = 0; i < chunks.length; i++) {
+          const chunk = chunks[i]!;
+          const embedding = embeddings[i]!;
+
+          await client.query(
+            `INSERT INTO document_chunks (document_id, chunk_index, content, embedding, metadata)
+             VALUES ($1, $2, $3, $4, $5)`,
+            [
+              docId,
+              i,
+              chunk,
+              `[${embedding.join(',')}]`,
+              JSON.stringify({ chunkIndex: i, totalChunks: chunks.length }),
+            ]
+          );
+        }
+
+        return docId;
+      });
+
+      logMemoryOperation('write', 'knowledge_base', {
+        documentId,
+        filename,
+        chunkCount: chunks.length,
+      });
+
+      return documentId;
+    } catch (error) {
+      logger.error('Failed to add document', { error, filename });
+      throw new MemoryError(
+        `Failed to add document: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Search the knowledge base
+  async search(
+    queryText: string,
+    options: {
+      userId?: string;
+      limit?: number;
+      minSimilarity?: number;
+    } = {}
+  ): Promise<SearchResult[]> {
+    const { userId, limit = 5, minSimilarity = 0.7 } = options;
+
+    try {
+      // Generate query embedding
+      const queryEmbedding = await this.client.embedQuery(queryText);
+
+      // Build query
+      let sql = `
+        SELECT
+          dc.id as chunk_id,
+          dc.document_id,
+          dc.content,
+          dc.metadata as chunk_metadata,
+          d.filename,
+          d.metadata as doc_metadata,
+          1 - (dc.embedding <=> $1::vector) as similarity
+        FROM document_chunks dc
+        JOIN documents d ON d.id = dc.document_id
+        WHERE 1 - (dc.embedding <=> $1::vector) >= $2
+      `;
+      const params: unknown[] = [`[${queryEmbedding.join(',')}]`, minSimilarity];
+      let paramIndex = 3;
+
+      if (userId) {
+        sql += ` AND d.user_id = $${paramIndex}`;
+        params.push(userId);
+        paramIndex++;
+      }
+
+      sql += ` ORDER BY similarity DESC LIMIT $${paramIndex}`;
+      params.push(limit);
+
+      const result = await query<{
+        chunk_id: string;
+        document_id: string;
+        content: string;
+        chunk_metadata: Record<string, unknown>;
+        filename: string;
+        doc_metadata: Record<string, unknown>;
+        similarity: number;
+      }>(sql, params);
+
+      const searchResults: SearchResult[] = result.rows.map((row) => ({
+        chunkId: row.chunk_id,
+        documentId: row.document_id,
+        content: row.content,
+        similarity: row.similarity,
+        metadata: { ...row.doc_metadata, ...row.chunk_metadata },
+        filename: row.filename,
+      }));
+
+      logMemoryOperation('search', 'knowledge_base', {
+        queryLength: queryText.length,
+        resultsCount: searchResults.length,
+      });
+
+      return searchResults;
+    } catch (error) {
+      logger.error('Failed to search knowledge base', { error });
+      throw new MemoryError(
+        `Failed to search knowledge base: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Get document by ID
+  async getDocument(id: string): Promise<Document | null> {
+    try {
+      const result = await query<{
+        id: string;
+        user_id: string | null;
+        filename: string;
+        file_type: string | null;
+        file_size: number | null;
+        content: string | null;
+        metadata: Record<string, unknown>;
+        created_at: Date;
+      }>(
+        `SELECT id, user_id, filename, file_type, file_size, content, metadata, created_at
+         FROM documents WHERE id = $1`,
+        [id]
+      );
+
+      if (result.rows.length === 0) {
+        return null;
+      }
+
+      const row = result.rows[0]!;
+      return {
+        id: row.id,
+        userId: row.user_id ?? undefined,
+        filename: row.filename,
+        fileType: row.file_type ?? undefined,
+        fileSize: row.file_size ?? undefined,
+        content: row.content ?? undefined,
+        metadata: row.metadata,
+        createdAt: row.created_at,
+      };
+    } catch (error) {
+      logger.error('Failed to get document', { error, id });
+      throw new MemoryError(
+        `Failed to get document: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // List documents for a user
+  async listDocuments(
+    userId: string,
+    options: { limit?: number; offset?: number } = {}
+  ): Promise<Document[]> {
+    const { limit = 20, offset = 0 } = options;
+
+    try {
+      const result = await query<{
+        id: string;
+        user_id: string | null;
+        filename: string;
+        file_type: string | null;
+        file_size: number | null;
+        metadata: Record<string, unknown>;
+        created_at: Date;
+      }>(
+        `SELECT id, user_id, filename, file_type, file_size, metadata, created_at
+         FROM documents
+         WHERE user_id = $1
+         ORDER BY created_at DESC
+         LIMIT $2 OFFSET $3`,
+        [userId, limit, offset]
+      );
+
+      return result.rows.map((row) => ({
+        id: row.id,
+        userId: row.user_id ?? undefined,
+        filename: row.filename,
+        fileType: row.file_type ?? undefined,
+        fileSize: row.file_size ?? undefined,
+        metadata: row.metadata,
+        createdAt: row.created_at,
+      }));
+    } catch (error) {
+      logger.error('Failed to list documents', { error });
+      throw new MemoryError(
+        `Failed to list documents: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Delete document and its chunks
+  async deleteDocument(id: string): Promise<void> {
+    try {
+      await query('DELETE FROM documents WHERE id = $1', [id]);
+      logMemoryOperation('write', 'knowledge_base', {
+        documentId: id,
+        action: 'delete',
+      });
+    } catch (error) {
+      logger.error('Failed to delete document', { error, id });
+      throw new MemoryError(
+        `Failed to delete document: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Get chunks for a document
+  async getDocumentChunks(documentId: string): Promise<DocumentChunk[]> {
+    try {
+      const result = await query<{
+        id: string;
+        document_id: string;
+        chunk_index: number;
+        content: string;
+        metadata: Record<string, unknown>;
+      }>(
+        `SELECT id, document_id, chunk_index, content, metadata
+         FROM document_chunks
+         WHERE document_id = $1
+         ORDER BY chunk_index ASC`,
+        [documentId]
+      );
+
+      return result.rows.map((row) => ({
+        id: row.id,
+        documentId: row.document_id,
+        chunkIndex: row.chunk_index,
+        content: row.content,
+        metadata: row.metadata,
+      }));
+    } catch (error) {
+      logger.error('Failed to get document chunks', { error, documentId });
+      throw new MemoryError(
+        `Failed to get document chunks: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Get document count for user
+  async getDocumentCount(userId: string): Promise<number> {
+    try {
+      const result = await query<{ count: string }>(
+        'SELECT COUNT(*) as count FROM documents WHERE user_id = $1',
+        [userId]
+      );
+      return parseInt(result.rows[0]?.count ?? '0', 10);
+    } catch (error) {
+      logger.error('Failed to get document count', { error });
+      throw new MemoryError(
+        `Failed to get document count: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+}
+
+// Singleton instance
+let kbInstance: KnowledgeBase | null = null;
+
+export function getKnowledgeBase(): KnowledgeBase {
+  if (!kbInstance) {
+    kbInstance = new KnowledgeBase();
+  }
+  return kbInstance;
+}
+
+export default KnowledgeBase;

package/src/memory/longterm/vector-store.ts

@@ -0,0 +1,364 @@
+import { query, transaction } from '../../database/client.js';
+import { getOpenRouterClient } from '../../llm/openrouter-client.js';
+import { createLogger, logMemoryOperation } from '../../utils/logger.js';
+import { MemoryError } from '../../utils/errors.js';
+import type { RetrievedMemory } from '../../core/agents/types.js';
+
+const logger = createLogger('VectorStore');
+
+export type MemoryType = 'semantic' | 'episodic' | 'procedural';
+
+export interface MemoryEntry {
+  id: string;
+  userId?: string;
+  memoryType: MemoryType;
+  content: string;
+  importance: number;
+  metadata?: Record<string, unknown>;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+export interface SearchOptions {
+  userId?: string;
+  memoryType?: MemoryType;
+  limit?: number;
+  minSimilarity?: number;
+}
+
+export class VectorStore {
+  private client = getOpenRouterClient();
+
+  // Store a memory with embedding
+  async store(
+    content: string,
+    memoryType: MemoryType,
+    options: {
+      userId?: string;
+      importance?: number;
+      metadata?: Record<string, unknown>;
+    } = {}
+  ): Promise<string> {
+    try {
+      // Generate embedding
+      const embedding = await this.client.embedQuery(content);
+
+      // Insert into database
+      const result = await query<{ id: string }>(
+        `INSERT INTO long_term_memories
+         (user_id, memory_type, content, embedding, importance, metadata)
+         VALUES ($1, $2, $3, $4, $5, $6)
+         RETURNING id`,
+        [
+          options.userId ?? null,
+          memoryType,
+          content,
+          `[${embedding.join(',')}]`,
+          options.importance ?? 0.5,
+          JSON.stringify(options.metadata ?? {}),
+        ]
+      );
+
+      const memoryId = result.rows[0]?.id;
+      if (!memoryId) {
+        throw new MemoryError('Failed to store memory');
+      }
+
+      logMemoryOperation('write', 'long_term', {
+        memoryId,
+        memoryType,
+        contentLength: content.length,
+      });
+
+      return memoryId;
+    } catch (error) {
+      logger.error('Failed to store memory', { error });
+      throw new MemoryError(
+        `Failed to store memory: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Search for similar memories
+  async search(
+    queryText: string,
+    options: SearchOptions = {}
+  ): Promise<RetrievedMemory[]> {
+    const {
+      userId,
+      memoryType,
+      limit = 5,
+      minSimilarity = 0.7,
+    } = options;
+
+    try {
+      // Generate query embedding
+      const queryEmbedding = await this.client.embedQuery(queryText);
+
+      // Build the query with filters
+      let sql = `
+        SELECT
+          id,
+          memory_type,
+          content,
+          metadata,
+          1 - (embedding <=> $1::vector) as similarity
+        FROM long_term_memories
+        WHERE 1 - (embedding <=> $1::vector) >= $2
+      `;
+      const params: unknown[] = [`[${queryEmbedding.join(',')}]`, minSimilarity];
+      let paramIndex = 3;
+
+      if (userId) {
+        sql += ` AND user_id = $${paramIndex}`;
+        params.push(userId);
+        paramIndex++;
+      }
+
+      if (memoryType) {
+        sql += ` AND memory_type = $${paramIndex}`;
+        params.push(memoryType);
+        paramIndex++;
+      }
+
+      sql += ` ORDER BY similarity DESC LIMIT $${paramIndex}`;
+      params.push(limit);
+
+      const result = await query<{
+        id: string;
+        memory_type: MemoryType;
+        content: string;
+        metadata: Record<string, unknown>;
+        similarity: number;
+      }>(sql, params);
+
+      // Update access count for retrieved memories
+      if (result.rows.length > 0) {
+        const ids = result.rows.map((r) => r.id);
+        await query(
+          `UPDATE long_term_memories
+           SET access_count = access_count + 1, last_accessed = NOW()
+           WHERE id = ANY($1)`,
+          [ids]
+        );
+      }
+
+      const memories: RetrievedMemory[] = result.rows.map((row) => ({
+        id: row.id,
+        type: row.memory_type,
+        content: row.content,
+        relevanceScore: row.similarity,
+        metadata: row.metadata,
+      }));
+
+      logMemoryOperation('search', 'long_term', {
+        queryLength: queryText.length,
+        resultsCount: memories.length,
+      });
+
+      return memories;
+    } catch (error) {
+      logger.error('Failed to search memories', { error });
+      throw new MemoryError(
+        `Failed to search memories: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Get memory by ID
+  async get(id: string): Promise<MemoryEntry | null> {
+    try {
+      const result = await query<{
+        id: string;
+        user_id: string | null;
+        memory_type: MemoryType;
+        content: string;
+        importance: number;
+        metadata: Record<string, unknown>;
+        created_at: Date;
+        updated_at: Date;
+      }>(
+        `SELECT id, user_id, memory_type, content, importance, metadata, created_at, updated_at
+         FROM long_term_memories WHERE id = $1`,
+        [id]
+      );
+
+      if (result.rows.length === 0) {
+        return null;
+      }
+
+      const row = result.rows[0]!;
+      return {
+        id: row.id,
+        userId: row.user_id ?? undefined,
+        memoryType: row.memory_type,
+        content: row.content,
+        importance: row.importance,
+        metadata: row.metadata,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+      };
+    } catch (error) {
+      logger.error('Failed to get memory', { error, id });
+      throw new MemoryError(
+        `Failed to get memory: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Update memory content
+  async update(
+    id: string,
+    content: string,
+    options: { importance?: number; metadata?: Record<string, unknown> } = {}
+  ): Promise<void> {
+    try {
+      // Re-generate embedding for new content
+      const embedding = await this.client.embedQuery(content);
+
+      await query(
+        `UPDATE long_term_memories
+         SET content = $2, embedding = $3, importance = COALESCE($4, importance),
+             metadata = COALESCE($5, metadata), updated_at = NOW()
+         WHERE id = $1`,
+        [
+          id,
+          content,
+          `[${embedding.join(',')}]`,
+          options.importance ?? null,
+          options.metadata ? JSON.stringify(options.metadata) : null,
+        ]
+      );
+
+      logMemoryOperation('write', 'long_term', { memoryId: id, action: 'update' });
+    } catch (error) {
+      logger.error('Failed to update memory', { error, id });
+      throw new MemoryError(
+        `Failed to update memory: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Delete memory
+  async delete(id: string): Promise<void> {
+    try {
+      await query('DELETE FROM long_term_memories WHERE id = $1', [id]);
+      logMemoryOperation('write', 'long_term', { memoryId: id, action: 'delete' });
+    } catch (error) {
+      logger.error('Failed to delete memory', { error, id });
+      throw new MemoryError(
+        `Failed to delete memory: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Bulk store memories
+  async bulkStore(
+    entries: Array<{
+      content: string;
+      memoryType: MemoryType;
+      userId?: string;
+      importance?: number;
+      metadata?: Record<string, unknown>;
+    }>
+  ): Promise<string[]> {
+    try {
+      // Generate embeddings for all entries
+      const embeddings = await this.client.embed(entries.map((e) => e.content));
+
+      // Use transaction for bulk insert
+      const ids = await transaction(async (client) => {
+        const results: string[] = [];
+
+        for (let i = 0; i < entries.length; i++) {
+          const entry = entries[i]!;
+          const embedding = embeddings[i]!;
+
+          const result = await client.query<{ id: string }>(
+            `INSERT INTO long_term_memories
+             (user_id, memory_type, content, embedding, importance, metadata)
+             VALUES ($1, $2, $3, $4, $5, $6)
+             RETURNING id`,
+            [
+              entry.userId ?? null,
+              entry.memoryType,
+              entry.content,
+              `[${embedding.join(',')}]`,
+              entry.importance ?? 0.5,
+              JSON.stringify(entry.metadata ?? {}),
+            ]
+          );
+
+          if (result.rows[0]) {
+            results.push(result.rows[0].id);
+          }
+        }
+
+        return results;
+      });
+
+      logMemoryOperation('write', 'long_term', {
+        action: 'bulk_store',
+        count: ids.length,
+      });
+
+      return ids;
+    } catch (error) {
+      logger.error('Failed to bulk store memories', { error });
+      throw new MemoryError(
+        `Failed to bulk store memories: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+
+  // Get recent memories for a user
+  async getRecent(userId: string, limit: number = 10): Promise<MemoryEntry[]> {
+    try {
+      const result = await query<{
+        id: string;
+        user_id: string | null;
+        memory_type: MemoryType;
+        content: string;
+        importance: number;
+        metadata: Record<string, unknown>;
+        created_at: Date;
+        updated_at: Date;
+      }>(
+        `SELECT id, user_id, memory_type, content, importance, metadata, created_at, updated_at
+         FROM long_term_memories
+         WHERE user_id = $1
+         ORDER BY created_at DESC
+         LIMIT $2`,
+        [userId, limit]
+      );
+
+      return result.rows.map((row) => ({
+        id: row.id,
+        userId: row.user_id ?? undefined,
+        memoryType: row.memory_type,
+        content: row.content,
+        importance: row.importance,
+        metadata: row.metadata,
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+      }));
+    } catch (error) {
+      logger.error('Failed to get recent memories', { error });
+      throw new MemoryError(
+        `Failed to get recent memories: ${error instanceof Error ? error.message : 'Unknown error'}`
+      );
+    }
+  }
+}
+
+// Singleton instance
+let storeInstance: VectorStore | null = null;
+
+export function getVectorStore(): VectorStore {
+  if (!storeInstance) {
+    storeInstance = new VectorStore();
+  }
+  return storeInstance;
+}
+
+export default VectorStore;