family-ai-agent 1.0.6 → 1.0.8

package/src/database/vector/index.ts

@@ -0,0 +1,292 @@
+/**
+ * Vector Search Engine
+ *
+ * Unified interface for vector similarity search.
+ * PostgreSQL uses pgvector, SQLite uses in-memory HNSW.
+ */
+
+import { HNSWIndex, type SearchResult } from './hnsw-index.js';
+import { EmbeddingCache } from './embedding-cache.js';
+import { cosineSimilarity, parseEmbedding, serializeEmbedding } from './similarity.js';
+import type { DatabaseAdapter, VectorSearchOptions, VectorSearchResult, VectorSearchEngine } from '../types.js';
+import { createLogger } from '../../utils/logger.js';
+
+const logger = createLogger('VectorSearch');
+
+// Re-export components
+export { HNSWIndex, type SearchResult } from './hnsw-index.js';
+export { EmbeddingCache, type EmbeddingCacheStats } from './embedding-cache.js';
+export * from './similarity.js';
+
+/**
+ * PostgreSQL vector search engine using pgvector
+ */
+export class PgVectorSearchEngine implements VectorSearchEngine {
+  private adapter: DatabaseAdapter;
+  private tableName: string;
+  private embeddingColumn: string;
+  private dimension: number;
+
+  constructor(
+    adapter: DatabaseAdapter,
+    options: {
+      tableName: string;
+      embeddingColumn?: string;
+      dimension?: number;
+    }
+  ) {
+    this.adapter = adapter;
+    this.tableName = options.tableName;
+    this.embeddingColumn = options.embeddingColumn ?? 'embedding';
+    this.dimension = options.dimension ?? 1536;
+  }
+
+  async initialize(): Promise<void> {
+    // pgvector is initialized at adapter level
+    logger.debug('PgVectorSearchEngine initialized', { table: this.tableName });
+  }
+
+  async store(id: string, embedding: number[], metadata?: Record<string, unknown>): Promise<void> {
+    // This is handled by the memory layer, not directly by vector engine
+    logger.debug('Store called on PgVectorSearchEngine', { id });
+  }
+
+  async search(queryEmbedding: number[], options: VectorSearchOptions = {}): Promise<VectorSearchResult[]> {
+    const { limit = 5, minSimilarity = 0.7, userId, memoryType } = options;
+    const embeddingStr = `[${queryEmbedding.join(',')}]`;
+
+    let sql = `
+      SELECT id, content, memory_type, metadata,
+             1 - (${this.embeddingColumn} <=> $1::vector) as similarity
+      FROM ${this.tableName}
+      WHERE 1 - (${this.embeddingColumn} <=> $1::vector) >= $2
+    `;
+    const params: unknown[] = [embeddingStr, minSimilarity];
+    let paramIdx = 3;
+
+    if (userId) {
+      sql += ` AND user_id = $${paramIdx}`;
+      params.push(userId);
+      paramIdx++;
+    }
+
+    if (memoryType) {
+      sql += ` AND memory_type = $${paramIdx}`;
+      params.push(memoryType);
+      paramIdx++;
+    }
+
+    sql += ` ORDER BY similarity DESC LIMIT $${paramIdx}`;
+    params.push(limit);
+
+    const result = await this.adapter.query<{
+      id: string;
+      content: string;
+      memory_type: string;
+      metadata: Record<string, unknown>;
+      similarity: number;
+    }>(sql, params);
+
+    return result.rows.map((row) => ({
+      id: row.id,
+      content: row.content,
+      similarity: row.similarity,
+      memoryType: row.memory_type,
+      metadata: row.metadata,
+    }));
+  }
+
+  async remove(id: string): Promise<void> {
+    // Handled by memory layer
+    logger.debug('Remove called on PgVectorSearchEngine', { id });
+  }
+
+  async rebuildIndex(): Promise<void> {
+    // pgvector indexes are managed by PostgreSQL
+    logger.debug('Rebuild index called on PgVectorSearchEngine');
+  }
+
+  getStats(): { size: number; dimension: number } {
+    return { size: 0, dimension: this.dimension }; // Would need a count query
+  }
+}
+
+/**
+ * SQLite vector search engine using in-memory HNSW
+ */
+export class SqliteVectorSearchEngine implements VectorSearchEngine {
+  private adapter: DatabaseAdapter;
+  private tableName: string;
+  private index: HNSWIndex;
+  private cache: EmbeddingCache;
+  private dimension: number;
+  private initialized: boolean = false;
+
+  constructor(
+    adapter: DatabaseAdapter,
+    options: {
+      tableName: string;
+      dimension?: number;
+      cacheSize?: number;
+    }
+  ) {
+    this.adapter = adapter;
+    this.tableName = options.tableName;
+    this.dimension = options.dimension ?? 1536;
+    this.index = new HNSWIndex({
+      dimension: this.dimension,
+      similarity: 'cosine',
+      M: 16,
+      efConstruction: 200,
+    });
+    this.cache = new EmbeddingCache({
+      maxSize: options.cacheSize ?? 10000,
+      dimension: this.dimension,
+    });
+  }
+
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    // Load all embeddings from database into memory
+    const result = await this.adapter.query<{
+      id: string;
+      embedding: string;
+      memory_type: string;
+      content: string;
+    }>(`SELECT id, embedding, memory_type, content FROM ${this.tableName}`);
+
+    for (const row of result.rows) {
+      const embedding = parseEmbedding(row.embedding);
+      if (embedding && embedding.length === this.dimension) {
+        this.index.add(row.id, embedding);
+        this.cache.set(row.id, embedding);
+      }
+    }
+
+    this.initialized = true;
+    logger.info('SqliteVectorSearchEngine initialized', {
+      table: this.tableName,
+      vectorCount: this.index.size,
+    });
+  }
+
+  async store(id: string, embedding: number[], metadata?: Record<string, unknown>): Promise<void> {
+    this.index.add(id, embedding);
+    this.cache.set(id, embedding, metadata);
+  }
+
+  async search(queryEmbedding: number[], options: VectorSearchOptions = {}): Promise<VectorSearchResult[]> {
+    const { limit = 5, minSimilarity = 0.7, userId, memoryType } = options;
+
+    // Use HNSW for initial candidates (get more than needed for filtering)
+    const candidates = this.index.search(queryEmbedding, limit * 3);
+
+    // Filter candidates that don't meet similarity threshold
+    const filtered = candidates.filter((c) => c.similarity >= minSimilarity);
+
+    if (filtered.length === 0) {
+      return [];
+    }
+
+    // Get full records from database
+    const ids = filtered.map((c) => c.id);
+    const placeholders = ids.map((_, i) => `$${i + 1}`).join(',');
+
+    let sql = `SELECT id, content, memory_type, metadata FROM ${this.tableName} WHERE id IN (${placeholders})`;
+    const params: unknown[] = [...ids];
+
+    if (userId) {
+      sql = sql.replace('WHERE', `WHERE user_id = $${params.length + 1} AND`);
+      params.push(userId);
+    }
+
+    if (memoryType) {
+      sql = sql.replace('WHERE', `WHERE memory_type = $${params.length + 1} AND`);
+      params.push(memoryType);
+    }
+
+    const result = await this.adapter.query<{
+      id: string;
+      content: string;
+      memory_type: string;
+      metadata: string | Record<string, unknown>;
+    }>(sql, params);
+
+    // Create a map for quick lookup
+    const recordMap = new Map<string, typeof result.rows[0]>();
+    for (const row of result.rows) {
+      recordMap.set(row.id, row);
+    }
+
+    // Build results with similarity scores
+    const results: VectorSearchResult[] = [];
+    for (const candidate of filtered) {
+      const record = recordMap.get(candidate.id);
+      if (record) {
+        results.push({
+          id: record.id,
+          content: record.content,
+          similarity: candidate.similarity,
+          memoryType: record.memory_type,
+          metadata: typeof record.metadata === 'string'
+            ? JSON.parse(record.metadata)
+            : record.metadata,
+        });
+      }
+    }
+
+    // Sort by similarity and limit
+    return results
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, limit);
+  }
+
+  async remove(id: string): Promise<void> {
+    this.index.remove(id);
+    this.cache.delete(id);
+  }
+
+  async rebuildIndex(): Promise<void> {
+    this.initialized = false;
+    this.index.clear();
+    this.cache.clear();
+    await this.initialize();
+  }
+
+  getStats(): { size: number; dimension: number } {
+    const indexStats = this.index.getStats();
+    return {
+      size: indexStats.size,
+      dimension: indexStats.dimension,
+    };
+  }
+
+  getCacheStats() {
+    return this.cache.getStats();
+  }
+}
+
+/**
+ * Create vector search engine based on adapter type
+ */
+export function createVectorSearchEngine(
+  adapter: DatabaseAdapter,
+  options: {
+    tableName: string;
+    embeddingColumn?: string;
+    dimension?: number;
+    cacheSize?: number;
+  }
+): VectorSearchEngine {
+  if (adapter.type === 'postgresql') {
+    return new PgVectorSearchEngine(adapter, options);
+  }
+  return new SqliteVectorSearchEngine(adapter, options);
+}
+
+export default {
+  PgVectorSearchEngine,
+  SqliteVectorSearchEngine,
+  createVectorSearchEngine,
+};

package/src/database/vector/similarity.ts

@@ -0,0 +1,198 @@
+/**
+ * Vector Similarity Functions
+ *
+ * Mathematical utilities for computing vector similarity.
+ * Used for in-memory vector search in SQLite mode.
+ */
+
+/**
+ * Compute cosine similarity between two vectors
+ * Returns a value between -1 and 1, where 1 means identical direction
+ */
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+
+  if (normA === 0 || normB === 0) {
+    return 0;
+  }
+
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * Compute Euclidean distance between two vectors
+ */
+export function euclideanDistance(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i]! - b[i]!;
+    sum += diff * diff;
+  }
+
+  return Math.sqrt(sum);
+}
+
+/**
+ * Compute dot product of two vectors
+ */
+export function dotProduct(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    sum += a[i]! * b[i]!;
+  }
+
+  return sum;
+}
+
+/**
+ * Compute L2 norm (magnitude) of a vector
+ */
+export function l2Norm(v: number[]): number {
+  let sum = 0;
+  for (let i = 0; i < v.length; i++) {
+    sum += v[i]! * v[i]!;
+  }
+  return Math.sqrt(sum);
+}
+
+/**
+ * Normalize a vector to unit length
+ */
+export function normalize(v: number[]): number[] {
+  const norm = l2Norm(v);
+  if (norm === 0) {
+    return v.slice();
+  }
+  return v.map((x) => x / norm);
+}
+
+/**
+ * Parse embedding from JSON string or array
+ */
+export function parseEmbedding(value: string | number[] | null | undefined): number[] | null {
+  if (!value) return null;
+
+  if (Array.isArray(value)) {
+    return value;
+  }
+
+  try {
+    // Try parsing as JSON array
+    const parsed = JSON.parse(value);
+    if (Array.isArray(parsed)) {
+      return parsed;
+    }
+    return null;
+  } catch {
+    // Try parsing as PostgreSQL vector format: [0.1, 0.2, ...]
+    const match = value.match(/^\[(.+)\]$/);
+    if (match) {
+      try {
+        return match[1]!.split(',').map((s) => parseFloat(s.trim()));
+      } catch {
+        return null;
+      }
+    }
+    return null;
+  }
+}
+
+/**
+ * Serialize embedding to JSON string
+ */
+export function serializeEmbedding(embedding: number[]): string {
+  return JSON.stringify(embedding);
+}
+
+/**
+ * Compute hash of embedding for caching
+ */
+export function hashEmbedding(embedding: number[]): string {
+  // Simple hash using first and last few elements + length
+  const first = embedding.slice(0, 4).map((n) => n.toFixed(4)).join(',');
+  const last = embedding.slice(-4).map((n) => n.toFixed(4)).join(',');
+  return `${embedding.length}:${first}:${last}`;
+}
+
+/**
+ * Quantize embedding to reduce memory usage
+ * Converts float32 to int8 (-128 to 127)
+ */
+export function quantize(embedding: number[]): Int8Array {
+  const quantized = new Int8Array(embedding.length);
+  for (let i = 0; i < embedding.length; i++) {
+    // Scale from [-1, 1] to [-128, 127]
+    quantized[i] = Math.round(Math.max(-128, Math.min(127, embedding[i]! * 127)));
+  }
+  return quantized;
+}
+
+/**
+ * Dequantize int8 back to float
+ */
+export function dequantize(quantized: Int8Array): number[] {
+  const result = new Array<number>(quantized.length);
+  for (let i = 0; i < quantized.length; i++) {
+    result[i] = quantized[i]! / 127;
+  }
+  return result;
+}
+
+/**
+ * Cosine similarity for quantized vectors (faster)
+ */
+export function quantizedCosineSimilarity(a: Int8Array, b: Int8Array): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+
+  if (normA === 0 || normB === 0) {
+    return 0;
+  }
+
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+export default {
+  cosineSimilarity,
+  euclideanDistance,
+  dotProduct,
+  l2Norm,
+  normalize,
+  parseEmbedding,
+  serializeEmbedding,
+  hashEmbedding,
+  quantize,
+  dequantize,
+  quantizedCosineSimilarity,
+};