@aitytech/agentkits-memory 1.0.0 → 2.0.0

package/src/search/hybrid-search.ts

@@ -0,0 +1,968 @@
+/**
+ * Hybrid Search Module
+ *
+ * Combines FTS5 keyword search with vector semantic search
+ * for improved recall (15-20% better than either alone).
+ *
+ * Features:
+ * - SQLite FTS5 full-text search with trigram tokenizer (CJK support)
+ * - Score fusion (α*keyword + β*semantic)
+ * - 3-layer search workflow for token efficiency
+ * - Token economics tracking
+ *
+ * CJK Language Support:
+ * Uses trigram tokenizer which works for Japanese, Chinese, Korean
+ * by matching substrings instead of requiring word boundaries.
+ *
+ * @module @aitytech/agentkits-memory/search
+ */
+
+import type { Database as BetterDatabase } from 'better-sqlite3';
+import type { MemoryEntry, SearchResult, EmbeddingGenerator } from '../types.js';
+
+/**
+ * Hybrid search configuration
+ */
+export interface HybridSearchConfig {
+  /** Weight for keyword/FTS5 score (0-1, default: 0.3) */
+  keywordWeight: number;
+
+  /** Weight for semantic/vector score (0-1, default: 0.7) */
+  semanticWeight: number;
+
+  /** Minimum combined score threshold (0-1, default: 0.1) */
+  minScore: number;
+
+  /** Enable BM25 scoring for FTS5 (default: true) */
+  useBM25: boolean;
+
+  /** Maximum results per search layer (default: 100) */
+  maxResultsPerLayer: number;
+
+  /**
+   * FTS5 tokenizer to use (default: 'trigram')
+   * - 'trigram': Best for CJK languages (Japanese, Chinese, Korean)
+   * - 'unicode61': Standard tokenizer, English/Latin only
+   * - 'porter': Stemming for English
+   */
+  tokenizer: 'trigram' | 'unicode61' | 'porter';
+
+  /** Fall back to LIKE search if FTS5 unavailable (default: true) */
+  fallbackToLike: boolean;
+}
+
+/**
+ * Compact search result (Layer 1)
+ * Minimal data for initial filtering - saves tokens
+ */
+export interface CompactSearchResult {
+  /** Entry ID */
+  id: string;
+
+  /** Entry key */
+  key: string;
+
+  /** Namespace */
+  namespace: string;
+
+  /** Combined relevance score (0-1) */
+  score: number;
+
+  /** Keyword match score */
+  keywordScore: number;
+
+  /** Semantic similarity score */
+  semanticScore: number;
+
+  /** Preview snippet (first 100 chars) */
+  snippet: string;
+
+  /** Estimated token count */
+  estimatedTokens: number;
+}
+
+/**
+ * Timeline result (Layer 2)
+ * Context around search results
+ */
+export interface TimelineResult {
+  /** The target entry */
+  entry: CompactSearchResult;
+
+  /** Related entries before (chronologically) */
+  before: CompactSearchResult[];
+
+  /** Related entries after (chronologically) */
+  after: CompactSearchResult[];
+
+  /** Total context window tokens */
+  totalTokens: number;
+}
+
+/**
+ * Token economics for search operations
+ */
+export interface TokenEconomics {
+  /** Tokens saved by using compact results */
+  tokensSaved: number;
+
+  /** Tokens that would be used with full results */
+  fullResultTokens: number;
+
+  /** Actual tokens used */
+  actualTokens: number;
+
+  /** Savings percentage */
+  savingsPercent: number;
+
+  /** Layer breakdown */
+  layers: {
+    compact: number;
+    timeline: number;
+    full: number;
+  };
+}
+
+/**
+ * Full search result with economics
+ */
+export interface HybridSearchResult {
+  /** Search results */
+  results: SearchResult[];
+
+  /** Compact results (layer 1) */
+  compact: CompactSearchResult[];
+
+  /** Token economics */
+  economics: TokenEconomics;
+
+  /** Search timing */
+  timing: {
+    keywordMs: number;
+    semanticMs: number;
+    fusionMs: number;
+    totalMs: number;
+  };
+}
+
+/**
+ * Default hybrid search configuration
+ */
+const DEFAULT_CONFIG: HybridSearchConfig = {
+  keywordWeight: 0.3,
+  semanticWeight: 0.7,
+  minScore: 0.1,
+  useBM25: true,
+  maxResultsPerLayer: 100,
+  tokenizer: 'trigram', // Best for CJK languages
+  fallbackToLike: true,
+};
+
+/**
+ * Estimate token count for text (rough approximation)
+ * Uses ~4 chars per token as average for English text
+ */
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Hybrid Search Engine
+ *
+ * Provides enterprise-grade search combining keyword and semantic search
+ * with token-efficient 3-layer retrieval workflow.
+ *
+ * Supports CJK languages (Japanese, Chinese, Korean) via trigram tokenizer.
+ */
+export class HybridSearchEngine {
+  private db: BetterDatabase;
+  private config: HybridSearchConfig;
+  private embeddingGenerator?: EmbeddingGenerator;
+  private ftsInitialized = false;
+  private ftsAvailable = false;
+  /** The actual tokenizer being used (may differ from config if tokenizer not available) */
+  private activeTokenizer: 'trigram' | 'unicode61' | 'porter' | null = null;
+
+  constructor(
+    db: BetterDatabase,
+    config: Partial<HybridSearchConfig> = {},
+    embeddingGenerator?: EmbeddingGenerator
+  ) {
+    this.db = db;
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    this.embeddingGenerator = embeddingGenerator;
+  }
+
+  /**
+   * Check if FTS5 is available in this SQLite build
+   */
+  private checkFts5Available(): boolean {
+    try {
+      // Try to create a minimal FTS5 table
+      this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS _fts5_check USING fts5(test)`);
+      this.db.exec(`DROP TABLE IF EXISTS _fts5_check`);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Check if a specific tokenizer is available
+   */
+  private checkTokenizerAvailable(tokenizer: string): boolean {
+    try {
+      this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS _tokenizer_check USING fts5(test, ${tokenizer})`);
+      this.db.exec(`DROP TABLE IF EXISTS _tokenizer_check`);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Get the best available tokenizer for FTS5
+   * Tries trigram first (best for CJK), then unicode61, then porter
+   * Also sets the activeTokenizer field
+   */
+  private getBestTokenizer(): string {
+    // Try tokenizers in order of preference for CJK support
+    if (this.config.tokenizer === 'trigram' && this.checkTokenizerAvailable("tokenize='trigram'")) {
+      this.activeTokenizer = 'trigram';
+      return "tokenize='trigram'";
+    }
+    if (this.config.tokenizer === 'porter' && this.checkTokenizerAvailable("tokenize='porter unicode61'")) {
+      this.activeTokenizer = 'porter';
+      return "tokenize='porter unicode61'";
+    }
+    // Default to unicode61 which should always be available
+    this.activeTokenizer = 'unicode61';
+    return "tokenize='unicode61'";
+  }
+
+  /**
+   * Initialize FTS5 virtual table
+   * Note: For best CJK support, use better-sqlite3 which includes trigram tokenizer.
+   */
+  async initialize(): Promise<void> {
+    if (this.ftsInitialized) return;
+
+    // Check if FTS5 is available
+    this.ftsAvailable = this.checkFts5Available();
+
+    if (!this.ftsAvailable) {
+      console.warn(
+        '[HybridSearch] FTS5 not available in this SQLite build. ' +
+        'Falling back to LIKE search.'
+      );
+      this.ftsInitialized = true;
+      return;
+    }
+
+    try {
+      // Get the best available tokenizer
+      const tokenizer = this.getBestTokenizer();
+
+      // Create FTS5 virtual table for full-text search
+      // Uses content= to sync with main table
+      // trigram tokenizer provides substring matching for CJK languages
+      this.db.exec(`
+        CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+          key,
+          content,
+          namespace,
+          tags,
+          content=memory_entries,
+          content_rowid=rowid,
+          ${tokenizer}
+        )
+      `);
+
+      // Create triggers to keep FTS in sync with main table
+      this.db.exec(`
+        CREATE TRIGGER IF NOT EXISTS memory_fts_insert AFTER INSERT ON memory_entries BEGIN
+          INSERT INTO memory_fts(rowid, key, content, namespace, tags)
+          VALUES (NEW.rowid, NEW.key, NEW.content, NEW.namespace, NEW.tags);
+        END
+      `);
+
+      this.db.exec(`
+        CREATE TRIGGER IF NOT EXISTS memory_fts_delete AFTER DELETE ON memory_entries BEGIN
+          INSERT INTO memory_fts(memory_fts, rowid, key, content, namespace, tags)
+          VALUES ('delete', OLD.rowid, OLD.key, OLD.content, OLD.namespace, OLD.tags);
+        END
+      `);
+
+      this.db.exec(`
+        CREATE TRIGGER IF NOT EXISTS memory_fts_update AFTER UPDATE ON memory_entries BEGIN
+          INSERT INTO memory_fts(memory_fts, rowid, key, content, namespace, tags)
+          VALUES ('delete', OLD.rowid, OLD.key, OLD.content, OLD.namespace, OLD.tags);
+          INSERT INTO memory_fts(rowid, key, content, namespace, tags)
+          VALUES (NEW.rowid, NEW.key, NEW.content, NEW.namespace, NEW.tags);
+        END
+      `);
+
+      // Rebuild FTS index from existing data
+      await this.rebuildFtsIndex();
+    } catch (error) {
+      console.warn('[HybridSearch] Failed to initialize FTS5:', error);
+      this.ftsAvailable = false;
+    }
+
+    this.ftsInitialized = true;
+  }
+
+  /**
+   * Check if FTS5 is available and initialized
+   */
+  isFtsAvailable(): boolean {
+    return this.ftsAvailable;
+  }
+
+  /**
+   * Get the active tokenizer being used
+   * Returns null if FTS5 is not available
+   */
+  getActiveTokenizer(): 'trigram' | 'unicode61' | 'porter' | null {
+    return this.activeTokenizer;
+  }
+
+  /**
+   * Check if CJK search is fully supported (requires trigram tokenizer)
+   * If not, CJK queries will fall back to LIKE search
+   */
+  isCjkOptimized(): boolean {
+    return this.ftsAvailable && this.activeTokenizer === 'trigram';
+  }
+
+  /**
+   * Rebuild FTS index from existing memory entries
+   * Uses the FTS5 'rebuild' command for content-synced tables
+   */
+  async rebuildFtsIndex(): Promise<void> {
+    if (!this.ftsAvailable) return;
+
+    try {
+      // For content-synced FTS5 tables (using content=memory_entries),
+      // use the 'rebuild' command which re-reads from the content table
+      this.db.exec(`INSERT INTO memory_fts(memory_fts) VALUES('rebuild')`);
+    } catch (error) {
+      console.warn('[HybridSearch] Failed to rebuild FTS index:', error);
+    }
+  }
+
+  /**
+   * Layer 1: Compact Search
+   *
+   * Returns minimal data for initial filtering.
+   * ~10x token savings vs full results.
+   */
+  async searchCompact(
+    query: string,
+    options: {
+      limit?: number;
+      namespace?: string;
+      includeKeyword?: boolean;
+      includeSemantic?: boolean;
+    } = {}
+  ): Promise<CompactSearchResult[]> {
+    const limit = options.limit || this.config.maxResultsPerLayer;
+    const includeKeyword = options.includeKeyword ?? true;
+    const includeSemantic = options.includeSemantic ?? !!this.embeddingGenerator;
+
+    const results: Map<string, CompactSearchResult> = new Map();
+
+    // Keyword search with FTS5
+    if (includeKeyword) {
+      const keywordResults = await this.keywordSearch(query, limit, options.namespace);
+      for (const result of keywordResults) {
+        results.set(result.id, result);
+      }
+    }
+
+    // Semantic search with embeddings
+    if (includeSemantic && this.embeddingGenerator) {
+      const semanticResults = await this.semanticSearchCompact(query, limit, options.namespace);
+      for (const result of semanticResults) {
+        const existing = results.get(result.id);
+        if (existing) {
+          // Merge scores using fusion
+          existing.semanticScore = result.semanticScore;
+          existing.score = this.fuseScores(existing.keywordScore, result.semanticScore);
+        } else {
+          results.set(result.id, result);
+        }
+      }
+    }
+
+    // Sort by combined score and limit
+    return Array.from(results.values())
+      .filter((r) => r.score >= this.config.minScore)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit);
+  }
+
+  /**
+   * Layer 2: Timeline Search
+   *
+   * Returns context around matched entries.
+   * Useful for understanding temporal relationships.
+   */
+  async searchTimeline(
+    entryIds: string[],
+    contextWindow: number = 3
+  ): Promise<TimelineResult[]> {
+    const results: TimelineResult[] = [];
+
+    for (const id of entryIds) {
+      // Get the target entry
+      const targetRow = this.db.prepare(`
+        SELECT id, key, namespace, content, created_at
+        FROM memory_entries WHERE id = ?
+      `).get(id) as {
+        id: string;
+        key: string;
+        namespace: string;
+        content: string;
+        created_at: number;
+      } | undefined;
+
+      if (!targetRow) continue;
+
+      const targetCompact: CompactSearchResult = {
+        id: targetRow.id,
+        key: targetRow.key,
+        namespace: targetRow.namespace,
+        score: 1.0,
+        keywordScore: 0,
+        semanticScore: 0,
+        snippet: targetRow.content.substring(0, 100),
+        estimatedTokens: estimateTokens(targetRow.content),
+      };
+
+      // Get entries before
+      const beforeRows = this.db.prepare(`
+        SELECT id, key, namespace, content, created_at
+        FROM memory_entries
+        WHERE namespace = ? AND created_at < ?
+        ORDER BY created_at DESC
+        LIMIT ?
+      `).all(targetRow.namespace, targetRow.created_at, contextWindow) as typeof targetRow[];
+
+      const before: CompactSearchResult[] = beforeRows.map(row => ({
+        id: row.id,
+        key: row.key,
+        namespace: row.namespace,
+        score: 0.5,
+        keywordScore: 0,
+        semanticScore: 0,
+        snippet: row.content.substring(0, 100),
+        estimatedTokens: estimateTokens(row.content),
+      }));
+
+      // Get entries after
+      const afterRows = this.db.prepare(`
+        SELECT id, key, namespace, content, created_at
+        FROM memory_entries
+        WHERE namespace = ? AND created_at > ?
+        ORDER BY created_at ASC
+        LIMIT ?
+      `).all(targetRow.namespace, targetRow.created_at, contextWindow) as typeof targetRow[];
+
+      const after: CompactSearchResult[] = afterRows.map(row => ({
+        id: row.id,
+        key: row.key,
+        namespace: row.namespace,
+        score: 0.5,
+        keywordScore: 0,
+        semanticScore: 0,
+        snippet: row.content.substring(0, 100),
+        estimatedTokens: estimateTokens(row.content),
+      }));
+
+      const totalTokens =
+        targetCompact.estimatedTokens +
+        before.reduce((sum, r) => sum + r.estimatedTokens, 0) +
+        after.reduce((sum, r) => sum + r.estimatedTokens, 0);
+
+      results.push({
+        entry: targetCompact,
+        before: before.reverse(), // Chronological order
+        after,
+        totalTokens,
+      });
+    }
+
+    return results;
+  }
+
+  /**
+   * Layer 3: Full Search
+   *
+   * Returns complete entry data for selected IDs.
+   * Only fetch what you need after filtering.
+   */
+  async getFull(ids: string[]): Promise<MemoryEntry[]> {
+    if (ids.length === 0) return [];
+
+    const placeholders = ids.map(() => '?').join(', ');
+    const rows = this.db.prepare(`
+      SELECT * FROM memory_entries WHERE id IN (${placeholders})
+    `).all(...ids) as Record<string, unknown>[];
+
+    const entries = rows.map(row => this.rowToEntry(row));
+
+    // Sort by original order
+    const orderMap = new Map(ids.map((id, i) => [id, i]));
+    entries.sort((a, b) => (orderMap.get(a.id) || 0) - (orderMap.get(b.id) || 0));
+
+    return entries;
+  }
+
+  /**
+   * Full hybrid search with token economics
+   *
+   * Combines all three layers with detailed metrics.
+   */
+  async search(
+    query: string,
+    options: {
+      limit?: number;
+      namespace?: string;
+      fetchFull?: boolean;
+    } = {}
+  ): Promise<HybridSearchResult> {
+    const startTime = performance.now();
+    const limit = options.limit || 10;
+
+    // Layer 1: Compact search
+    const keywordStart = performance.now();
+    const compact = await this.searchCompact(query, {
+      limit: this.config.maxResultsPerLayer,
+      namespace: options.namespace,
+    });
+    const keywordTime = performance.now() - keywordStart;
+
+    // Calculate token economics
+    const compactTokens = compact.reduce((sum, r) => sum + r.estimatedTokens, 0);
+
+    // Layer 3: Fetch full results if requested
+    const semanticStart = performance.now();
+    let results: SearchResult[] = [];
+    let fullTokens = 0;
+
+    if (options.fetchFull !== false) {
+      const topIds = compact.slice(0, limit).map((r) => r.id);
+      const fullEntries = await this.getFull(topIds);
+
+      results = fullEntries.map((entry, i) => ({
+        entry,
+        score: compact[i]?.score || 0,
+        distance: 1 - (compact[i]?.score || 0),
+      }));
+
+      fullTokens = fullEntries.reduce((sum, e) => sum + estimateTokens(e.content), 0);
+    }
+    const semanticTime = performance.now() - semanticStart;
+
+    const totalTime = performance.now() - startTime;
+
+    // Calculate savings
+    const fullResultTokens = compact.reduce((sum, r) => sum + r.estimatedTokens, 0);
+    const actualTokens = options.fetchFull !== false ? fullTokens : compactTokens / 10;
+    const tokensSaved = fullResultTokens - actualTokens;
+    const savingsPercent = fullResultTokens > 0 ? (tokensSaved / fullResultTokens) * 100 : 0;
+
+    return {
+      results,
+      compact: compact.slice(0, limit),
+      economics: {
+        tokensSaved: Math.max(0, tokensSaved),
+        fullResultTokens,
+        actualTokens,
+        savingsPercent: Math.max(0, savingsPercent),
+        layers: {
+          compact: compact.length,
+          timeline: 0,
+          full: results.length,
+        },
+      },
+      timing: {
+        keywordMs: keywordTime,
+        semanticMs: semanticTime,
+        fusionMs: 0,
+        totalMs: totalTime,
+      },
+    };
+  }
+
+  /**
+   * Check if text contains CJK characters
+   * CJK requires special handling (LIKE or trigram tokenizer)
+   */
+  private containsCJK(text: string): boolean {
+    // Unicode ranges for CJK characters
+    // - CJK Unified Ideographs: \u4E00-\u9FFF
+    // - Hiragana: \u3040-\u309F
+    // - Katakana: \u30A0-\u30FF
+    // - Hangul: \uAC00-\uD7AF
+    // - CJK Extension: \u3400-\u4DBF
+    return /[\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\u3400-\u4DBF]/.test(text);
+  }
+
+  /**
+   * Keyword search using FTS5 (with LIKE fallback)
+   *
+   * For CJK languages, automatically falls back to LIKE search
+   * unless trigram tokenizer is available.
+   */
+  private async keywordSearch(
+    query: string,
+    limit: number,
+    namespace?: string
+  ): Promise<CompactSearchResult[]> {
+    // Use LIKE fallback if FTS5 not available
+    if (!this.ftsAvailable) {
+      return this.likeSearch(query, limit, namespace);
+    }
+
+    // For CJK queries, use LIKE fallback unless trigram tokenizer is actually active
+    // (unicode61 tokenizer doesn't work with CJK - no word boundaries)
+    if (this.containsCJK(query) && this.activeTokenizer !== 'trigram') {
+      return this.likeSearch(query, limit, namespace);
+    }
+
+    // Trigram tokenizer requires at least 3 characters to match
+    // For short CJK queries (< 3 chars), fall back to LIKE search
+    if (this.activeTokenizer === 'trigram' && this.containsCJK(query)) {
+      const cjkChars = query.match(/[\u4E00-\u9FFF\u3040-\u309F\u30A0-\u30FF\uAC00-\uD7AF\u3400-\u4DBF]/g);
+      if (cjkChars && cjkChars.length < 3) {
+        return this.likeSearch(query, limit, namespace);
+      }
+    }
+
+    // Sanitize query for FTS5
+    const sanitizedQuery = this.sanitizeFtsQuery(query);
+    if (!sanitizedQuery) return [];
+
+    try {
+      let rows: {
+        id: string;
+        key: string;
+        namespace: string;
+        content: string;
+        rank: number;
+      }[];
+
+      if (namespace) {
+        rows = this.db.prepare(`
+          SELECT
+            m.id, m.key, m.namespace, m.content,
+            bm25(memory_fts) as rank
+          FROM memory_fts f
+          JOIN memory_entries m ON f.rowid = m.rowid
+          WHERE memory_fts MATCH ? AND m.namespace = ?
+          ORDER BY rank
+          LIMIT ?
+        `).all(sanitizedQuery, namespace, limit) as typeof rows;
+      } else {
+        rows = this.db.prepare(`
+          SELECT
+            m.id, m.key, m.namespace, m.content,
+            bm25(memory_fts) as rank
+          FROM memory_fts f
+          JOIN memory_entries m ON f.rowid = m.rowid
+          WHERE memory_fts MATCH ?
+          ORDER BY rank
+          LIMIT ?
+        `).all(sanitizedQuery, limit) as typeof rows;
+      }
+
+      return rows.map(row => {
+        // Normalize BM25 score (negative, closer to 0 is better)
+        // Convert to 0-1 scale where 1 is best
+        const keywordScore = Math.min(1, Math.max(0, 1 + row.rank / 10));
+
+        return {
+          id: row.id,
+          key: row.key,
+          namespace: row.namespace,
+          score: keywordScore * this.config.keywordWeight,
+          keywordScore,
+          semanticScore: 0,
+          snippet: row.content.substring(0, 100),
+          estimatedTokens: estimateTokens(row.content),
+        };
+      });
+    } catch (error) {
+      // Fall back to LIKE search on error
+      if (this.config.fallbackToLike) {
+        return this.likeSearch(query, limit, namespace);
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * LIKE-based search fallback (works without FTS5)
+   *
+   * Less efficient but supports all languages.
+   */
+  private likeSearch(
+    query: string,
+    limit: number,
+    namespace?: string
+  ): CompactSearchResult[] {
+    // Handle empty query
+    const trimmedQuery = query.trim();
+    if (!trimmedQuery) return [];
+
+    const searchPattern = `%${trimmedQuery}%`;
+    let rows: {
+      id: string;
+      key: string;
+      namespace: string;
+      content: string;
+    }[];
+
+    if (namespace) {
+      rows = this.db.prepare(`
+        SELECT id, key, namespace, content
+        FROM memory_entries
+        WHERE (content LIKE ? OR key LIKE ? OR tags LIKE ?)
+          AND namespace = ?
+        ORDER BY created_at DESC
+        LIMIT ?
+      `).all(searchPattern, searchPattern, searchPattern, namespace, limit) as typeof rows;
+    } else {
+      rows = this.db.prepare(`
+        SELECT id, key, namespace, content
+        FROM memory_entries
+        WHERE content LIKE ? OR key LIKE ? OR tags LIKE ?
+        ORDER BY created_at DESC
+        LIMIT ?
+      `).all(searchPattern, searchPattern, searchPattern, limit) as typeof rows;
+    }
+
+    return rows.map(row => {
+      // Simple scoring based on match position
+      const lowerContent = row.content.toLowerCase();
+      const lowerQuery = query.toLowerCase();
+      const matchIndex = lowerContent.indexOf(lowerQuery);
+      const keywordScore = matchIndex >= 0 ? Math.max(0.3, 1 - matchIndex / 1000) : 0.5;
+
+      return {
+        id: row.id,
+        key: row.key,
+        namespace: row.namespace,
+        score: keywordScore * this.config.keywordWeight,
+        keywordScore,
+        semanticScore: 0,
+        snippet: row.content.substring(0, 100),
+        estimatedTokens: estimateTokens(row.content),
+      };
+    });
+  }
+
+  /**
+   * Semantic search returning compact results
+   */
+  private async semanticSearchCompact(
+    query: string,
+    limit: number,
+    namespace?: string
+  ): Promise<CompactSearchResult[]> {
+    if (!this.embeddingGenerator) return [];
+
+    // Generate query embedding
+    const queryEmbedding = await this.embeddingGenerator(query);
+
+    // Get all entries with embeddings
+    let rows: {
+      id: string;
+      key: string;
+      namespace: string;
+      content: string;
+      embedding: Buffer;
+    }[];
+
+    if (namespace) {
+      rows = this.db.prepare(`
+        SELECT id, key, namespace, content, embedding
+        FROM memory_entries
+        WHERE embedding IS NOT NULL AND namespace = ?
+      `).all(namespace) as typeof rows;
+    } else {
+      rows = this.db.prepare(`
+        SELECT id, key, namespace, content, embedding
+        FROM memory_entries
+        WHERE embedding IS NOT NULL
+      `).all() as typeof rows;
+    }
+
+    const candidates: Array<{
+      id: string;
+      key: string;
+      namespace: string;
+      content: string;
+      similarity: number;
+    }> = [];
+
+    for (const row of rows) {
+      if (row.embedding) {
+        const embedding = new Float32Array(row.embedding.buffer.slice(
+          row.embedding.byteOffset,
+          row.embedding.byteOffset + row.embedding.byteLength
+        ));
+        const similarity = this.cosineSimilarity(queryEmbedding, embedding);
+
+        candidates.push({
+          id: row.id,
+          key: row.key,
+          namespace: row.namespace,
+          content: row.content,
+          similarity,
+        });
+      }
+    }
+
+    // Sort by similarity and take top results
+    candidates.sort((a, b) => b.similarity - a.similarity);
+
+    return candidates.slice(0, limit).map((c) => ({
+      id: c.id,
+      key: c.key,
+      namespace: c.namespace,
+      score: c.similarity * this.config.semanticWeight,
+      keywordScore: 0,
+      semanticScore: c.similarity,
+      snippet: c.content.substring(0, 100),
+      estimatedTokens: estimateTokens(c.content),
+    }));
+  }
+
+  /**
+   * Fuse keyword and semantic scores
+   */
+  private fuseScores(keywordScore: number, semanticScore: number): number {
+    return (
+      keywordScore * this.config.keywordWeight +
+      semanticScore * this.config.semanticWeight
+    );
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   */
+  private cosineSimilarity(a: Float32Array, b: Float32Array): number {
+    if (a.length !== b.length) return 0;
+
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+    return denominator === 0 ? 0 : dotProduct / denominator;
+  }
+
+  /**
+   * Sanitize query for FTS5
+   * Preserves CJK characters (Japanese, Chinese, Korean) and basic alphanumerics
+   */
+  private sanitizeFtsQuery(query: string): string {
+    // For trigram tokenizer, preserve CJK characters
+    // Remove only FTS5 special operators: AND, OR, NOT, *, ^, :, ", (, )
+    // Keep Unicode letters, digits, and spaces
+    const sanitized = query
+      // Remove FTS5 special characters and operators, but preserve Unicode letters
+      .replace(/[*^:"()]/g, ' ')
+      .replace(/\bAND\b|\bOR\b|\bNOT\b|\bNEAR\b/gi, ' ')
+      .trim();
+
+    if (!sanitized) return '';
+
+    // For trigram tokenizer with CJK, we can pass the text directly
+    // The trigram tokenizer handles the text as-is
+    if (this.activeTokenizer === 'trigram') {
+      // With trigram, wrap the entire query in quotes for phrase matching
+      return `"${sanitized}"`;
+    }
+
+    // For other tokenizers, split into terms and wrap each
+    return sanitized
+      .split(/\s+/)
+      .filter((term) => term.length > 0)
+      .map((term) => `"${term}"`)
+      .join(' OR ');
+  }
+
+  /**
+   * Convert database row to MemoryEntry
+   */
+  private rowToEntry(row: Record<string, unknown>): MemoryEntry {
+    let embedding: Float32Array | undefined;
+    if (row.embedding) {
+      const embeddingData = row.embedding as Buffer;
+      embedding = new Float32Array(
+        embeddingData.buffer.slice(
+          embeddingData.byteOffset,
+          embeddingData.byteOffset + embeddingData.byteLength
+        )
+      );
+    }
+
+    return {
+      id: row.id as string,
+      key: row.key as string,
+      content: row.content as string,
+      embedding,
+      type: row.type as MemoryEntry['type'],
+      namespace: row.namespace as string,
+      tags: JSON.parse((row.tags as string) || '[]'),
+      metadata: JSON.parse((row.metadata as string) || '{}'),
+      sessionId: row.session_id as string | undefined,
+      ownerId: row.owner_id as string | undefined,
+      accessLevel: row.access_level as MemoryEntry['accessLevel'],
+      createdAt: row.created_at as number,
+      updatedAt: row.updated_at as number,
+      expiresAt: row.expires_at as number | undefined,
+      version: row.version as number,
+      references: JSON.parse((row.references as string) || '[]'),
+      accessCount: row.access_count as number,
+      lastAccessedAt: row.last_accessed_at as number,
+    };
+  }
+
+  /**
+   * Get configuration
+   */
+  getConfig(): HybridSearchConfig {
+    return { ...this.config };
+  }
+
+  /**
+   * Update configuration
+   */
+  updateConfig(config: Partial<HybridSearchConfig>): void {
+    this.config = { ...this.config, ...config };
+  }
+}
+
+/**
+ * Create a hybrid search engine
+ */
+export function createHybridSearchEngine(
+  db: BetterDatabase,
+  config?: Partial<HybridSearchConfig>,
+  embeddingGenerator?: EmbeddingGenerator
+): HybridSearchEngine {
+  return new HybridSearchEngine(db, config, embeddingGenerator);
+}
+
+export default HybridSearchEngine;