claude-brain 0.3.0

package/src/retrieval/bm25/index.ts

@@ -0,0 +1,300 @@
+/**
+ * BM25 Sparse Search Engine
+ * Uses MiniSearch for fast keyword-based retrieval
+ */
+
+import type { Logger } from 'pino'
+import MiniSearch from 'minisearch'
+import { tokenize, tokenizeForSearch, type TokenizerOptions } from './tokenizer'
+
+export interface BM25Document {
+  id: string
+  content: string
+  metadata: Record<string, unknown>
+  collection: string
+}
+
+export interface BM25SearchResult {
+  id: string
+  content: string
+  metadata: Record<string, unknown>
+  collection: string
+  score: number
+}
+
+export interface BM25Config {
+  /** Fields to index */
+  fields: string[]
+  /** Fields to store */
+  storeFields: string[]
+  /** BM25 k1 parameter (term frequency saturation) */
+  k1?: number
+  /** BM25 b parameter (document length normalization) */
+  b?: number
+  /** Boost for exact matches */
+  boostExact?: number
+}
+
+const DEFAULT_CONFIG: BM25Config = {
+  fields: ['content'],
+  storeFields: ['content', 'metadata', 'collection'],
+  k1: 1.2,
+  b: 0.75,
+  boostExact: 2.0
+}
+
+export class BM25Engine {
+  private logger: Logger
+  private config: BM25Config
+  private index: MiniSearch<BM25Document>
+  private documentCount: number = 0
+  private tokenizerOptions: TokenizerOptions
+
+  constructor(logger: Logger, config: Partial<BM25Config> = {}) {
+    this.logger = logger.child({ component: 'bm25-engine' })
+    this.config = { ...DEFAULT_CONFIG, ...config }
+    this.tokenizerOptions = {
+      minLength: 2,
+      removeStopwords: true,
+      stemming: true,
+      splitCamelCase: true
+    }
+
+    this.index = this.createIndex()
+  }
+
+  /**
+   * Create a new MiniSearch index
+   */
+  private createIndex(): MiniSearch<BM25Document> {
+    return new MiniSearch<BM25Document>({
+      fields: this.config.fields,
+      storeFields: this.config.storeFields,
+      idField: 'id',
+      tokenize: (text: string) => tokenize(text, this.tokenizerOptions),
+      processTerm: (term: string) => term.toLowerCase()
+    })
+  }
+
+  /**
+   * Build index from documents
+   */
+  async buildIndex(documents: BM25Document[]): Promise<void> {
+    this.logger.info({ count: documents.length }, 'Building BM25 index')
+
+    // Create fresh index
+    this.index = this.createIndex()
+    this.documentCount = 0
+
+    // Add documents in batches
+    const batchSize = 1000
+    for (let i = 0; i < documents.length; i += batchSize) {
+      const batch = documents.slice(i, i + batchSize)
+      await this.addDocuments(batch)
+    }
+
+    this.logger.info({
+      indexed: this.documentCount,
+      terms: this.index.termCount
+    }, 'BM25 index built')
+  }
+
+  /**
+   * Add documents to index
+   */
+  async addDocuments(documents: BM25Document[]): Promise<void> {
+    try {
+      this.index.addAll(documents)
+      this.documentCount += documents.length
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to add documents to BM25 index')
+      throw error
+    }
+  }
+
+  /**
+   * Add single document
+   */
+  addDocument(document: BM25Document): void {
+    try {
+      this.index.add(document)
+      this.documentCount++
+    } catch (error) {
+      this.logger.error({ error, id: document.id }, 'Failed to add document to BM25 index')
+      throw error
+    }
+  }
+
+  /**
+   * Remove document from index
+   */
+  removeDocument(document: BM25Document): void {
+    try {
+      this.index.remove(document)
+      this.documentCount--
+    } catch (error) {
+      this.logger.error({ error, id: document.id }, 'Failed to remove document from BM25 index')
+      // Don't throw - document might not exist
+    }
+  }
+
+  /**
+   * Update document in index
+   */
+  updateDocument(document: BM25Document): void {
+    this.removeDocument(document)
+    this.addDocument(document)
+  }
+
+  /**
+   * Search the index
+   */
+  search(
+    query: string,
+    options: {
+      limit?: number
+      filter?: (result: BM25SearchResult) => boolean
+      collection?: string
+    } = {}
+  ): BM25SearchResult[] {
+    const { limit = 20, filter, collection } = options
+
+    if (!query.trim()) {
+      return []
+    }
+
+    this.logger.debug({
+      query: query.slice(0, 50),
+      limit
+    }, 'BM25 search')
+
+    try {
+      // Tokenize query
+      const queryTokens = tokenizeForSearch(query, this.tokenizerOptions)
+
+      // Search with MiniSearch options
+      const results = this.index.search(query, {
+        prefix: true, // Allow prefix matching
+        fuzzy: 0.2, // Allow minor typos
+        combineWith: 'OR', // Match any term
+        boost: {
+          content: this.config.boostExact!
+        }
+      })
+
+      // Transform and filter results
+      let searchResults: BM25SearchResult[] = results.map(result => ({
+        id: result.id,
+        content: (result as any).content || '',
+        metadata: (result as any).metadata || {},
+        collection: (result as any).collection || '',
+        score: result.score
+      }))
+
+      // Filter by collection if specified
+      if (collection) {
+        searchResults = searchResults.filter(r => r.collection === collection)
+      }
+
+      // Apply custom filter
+      if (filter) {
+        searchResults = searchResults.filter(filter)
+      }
+
+      // Normalize scores to 0-1 range
+      searchResults = this.normalizeScores(searchResults)
+
+      // Apply limit
+      return searchResults.slice(0, limit)
+
+    } catch (error) {
+      this.logger.error({ error, query }, 'BM25 search failed')
+      return []
+    }
+  }
+
+  /**
+   * Normalize scores to 0-1 range
+   */
+  private normalizeScores(results: BM25SearchResult[]): BM25SearchResult[] {
+    if (results.length === 0) return []
+    if (results.length === 1) {
+      return [{ ...results[0], score: 1.0 }]
+    }
+
+    const maxScore = Math.max(...results.map(r => r.score))
+    if (maxScore === 0) {
+      return results.map(r => ({ ...r, score: 0 }))
+    }
+
+    return results.map(r => ({
+      ...r,
+      score: r.score / maxScore
+    }))
+  }
+
+  /**
+   * Get suggestions for autocomplete
+   */
+  suggest(query: string, limit: number = 5): string[] {
+    try {
+      const results = this.index.autoSuggest(query, { limit })
+      return results.map(r => r.suggestion)
+    } catch (error) {
+      this.logger.error({ error, query }, 'BM25 suggest failed')
+      return []
+    }
+  }
+
+  /**
+   * Get index statistics
+   */
+  getStats(): {
+    documentCount: number
+    termCount: number
+  } {
+    return {
+      documentCount: this.documentCount,
+      termCount: this.index.termCount
+    }
+  }
+
+  /**
+   * Clear the index
+   */
+  clear(): void {
+    this.index = this.createIndex()
+    this.documentCount = 0
+    this.logger.info('BM25 index cleared')
+  }
+
+  /**
+   * Export index for persistence
+   */
+  exportIndex(): string {
+    return JSON.stringify(this.index.toJSON())
+  }
+
+  /**
+   * Import index from persisted data
+   */
+  importIndex(data: string): void {
+    try {
+      const parsed = JSON.parse(data)
+      this.index = MiniSearch.loadJSON(data, {
+        fields: this.config.fields,
+        storeFields: this.config.storeFields,
+        idField: 'id'
+      })
+      // Count documents after import
+      this.documentCount = this.index.documentCount
+      this.logger.info({ documentCount: this.documentCount }, 'BM25 index imported')
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to import BM25 index')
+      throw error
+    }
+  }
+}
+
+export { tokenize, tokenizeForSearch, getNGrams } from './tokenizer'
+export type { TokenizerOptions } from './tokenizer'

package/src/retrieval/bm25/tokenizer.ts

@@ -0,0 +1,184 @@
+/**
+ * Text Tokenizer for BM25
+ * Handles text preprocessing for sparse search
+ */
+
+// Common English stopwords to filter
+const STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+  'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
+  'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+  'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought',
+  'used', 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my',
+  'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours',
+  'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her',
+  'hers', 'herself', 'they', 'them', 'their', 'theirs', 'themselves',
+  'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how', 'all',
+  'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such',
+  'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
+  's', 't', 'just', 'don', 'now', 'then', 'here', 'there', 'also'
+])
+
+// Common programming terms to keep
+const KEEP_TERMS = new Set([
+  'api', 'sql', 'css', 'html', 'json', 'xml', 'http', 'https', 'url', 'uri',
+  'jwt', 'oauth', 'rest', 'graphql', 'grpc', 'tcp', 'udp', 'ip', 'dns',
+  'aws', 'gcp', 'azure', 'docker', 'kubernetes', 'k8s', 'npm', 'yarn', 'pnpm',
+  'git', 'github', 'gitlab', 'ci', 'cd', 'devops', 'mlops', 'db', 'orm',
+  'ui', 'ux', 'cli', 'gui', 'ide', 'sdk', 'mcp', 'llm', 'ai', 'ml'
+])
+
+export interface TokenizerOptions {
+  /** Minimum token length to keep */
+  minLength?: number
+  /** Maximum token length */
+  maxLength?: number
+  /** Remove stopwords */
+  removeStopwords?: boolean
+  /** Convert to lowercase */
+  lowercase?: boolean
+  /** Apply stemming (basic) */
+  stemming?: boolean
+  /** Split on camelCase */
+  splitCamelCase?: boolean
+}
+
+const DEFAULT_OPTIONS: TokenizerOptions = {
+  minLength: 2,
+  maxLength: 50,
+  removeStopwords: true,
+  lowercase: true,
+  stemming: true,
+  splitCamelCase: true
+}
+
+/**
+ * Tokenize text for BM25 indexing and search
+ */
+export function tokenize(text: string, options: TokenizerOptions = {}): string[] {
+  const opts = { ...DEFAULT_OPTIONS, ...options }
+
+  if (!text || typeof text !== 'string') {
+    return []
+  }
+
+  // Split on camelCase if enabled
+  let processedText = text
+  if (opts.splitCamelCase) {
+    processedText = text.replace(/([a-z])([A-Z])/g, '$1 $2')
+  }
+
+  // Lowercase if enabled
+  if (opts.lowercase) {
+    processedText = processedText.toLowerCase()
+  }
+
+  // Split into tokens
+  // Match words, numbers, and hyphenated compounds
+  const tokenRegex = /[\w]+(?:[-_][\w]+)*/g
+  const rawTokens = processedText.match(tokenRegex) || []
+
+  // Process tokens
+  let tokens = rawTokens
+    .map(token => {
+      // Keep programming terms intact
+      if (KEEP_TERMS.has(token.toLowerCase())) {
+        return token
+      }
+
+      // Apply basic stemming if enabled
+      if (opts.stemming) {
+        return basicStem(token)
+      }
+
+      return token
+    })
+    .filter(token => {
+      // Length filter
+      if (token.length < opts.minLength! || token.length > opts.maxLength!) {
+        return false
+      }
+
+      // Stopword filter
+      if (opts.removeStopwords && STOPWORDS.has(token.toLowerCase())) {
+        return false
+      }
+
+      // Filter pure numbers (but keep alphanumeric)
+      if (/^\d+$/.test(token)) {
+        return false
+      }
+
+      return true
+    })
+
+  return tokens
+}
+
+/**
+ * Basic Porter-like stemming
+ * Simplified for performance - handles common English suffixes
+ */
+function basicStem(word: string): string {
+  if (word.length < 4) return word
+
+  // Common suffix replacements
+  const suffixRules: [RegExp, string][] = [
+    [/ies$/, 'y'],
+    [/ied$/, 'y'],
+    [/es$/, ''],
+    [/s$/, ''],
+    [/ing$/, ''],
+    [/ed$/, ''],
+    [/tion$/, 't'],
+    [/ness$/, ''],
+    [/ment$/, ''],
+    [/able$/, ''],
+    [/ible$/, ''],
+    [/ful$/, ''],
+    [/less$/, ''],
+    [/ly$/, '']
+  ]
+
+  let stemmed = word
+  for (const [pattern, replacement] of suffixRules) {
+    if (pattern.test(word)) {
+      const candidate = word.replace(pattern, replacement)
+      // Only apply if result is at least 3 chars
+      if (candidate.length >= 3) {
+        stemmed = candidate
+        break
+      }
+    }
+  }
+
+  return stemmed
+}
+
+/**
+ * Get n-grams from tokens
+ */
+export function getNGrams(tokens: string[], n: number = 2): string[] {
+  if (tokens.length < n) return []
+
+  const ngrams: string[] = []
+  for (let i = 0; i <= tokens.length - n; i++) {
+    ngrams.push(tokens.slice(i, i + n).join(' '))
+  }
+
+  return ngrams
+}
+
+/**
+ * Combined tokenizer for search that includes unigrams and bigrams
+ */
+export function tokenizeForSearch(
+  text: string,
+  options: TokenizerOptions = {}
+): string[] {
+  const unigrams = tokenize(text, options)
+  const bigrams = getNGrams(unigrams, 2)
+
+  // Return unique tokens (unigrams + bigrams)
+  return [...new Set([...unigrams, ...bigrams])]
+}

package/src/retrieval/feedback/adaptive.ts

@@ -0,0 +1,223 @@
+/**
+ * Adaptive Learning
+ * Learns optimal retrieval thresholds from feedback
+ */
+
+import type { Logger } from 'pino'
+import type { MemoryFeedback, AdaptiveThresholds, RetrievalMetrics } from '../types'
+import type { FeedbackStore } from './store'
+import { calculateAllMetrics, calculateAverageRating, calculatePositiveRate } from './metrics'
+
+/** Default thresholds before any learning */
+const DEFAULT_THRESHOLDS: AdaptiveThresholds = {
+  denseMinSimilarity: 0.3,
+  denseWeight: 0.7,
+  sparseWeight: 0.3,
+  rrfK: 60,
+  feedbackCount: 0,
+  lastUpdated: new Date().toISOString()
+}
+
+/** Thresholds bounds to prevent extreme values */
+const BOUNDS = {
+  denseMinSimilarity: { min: 0.1, max: 0.9 },
+  denseWeight: { min: 0.3, max: 0.9 },
+  sparseWeight: { min: 0.1, max: 0.7 },
+  rrfK: { min: 20, max: 100 }
+}
+
+export interface AdaptiveLearnerConfig {
+  /** Minimum feedback count before adaptation */
+  minFeedbackForAdaptation: number
+  /** Learning rate for threshold adjustments */
+  learningRate: number
+  /** Target positive feedback rate */
+  targetPositiveRate: number
+}
+
+const DEFAULT_CONFIG: AdaptiveLearnerConfig = {
+  minFeedbackForAdaptation: 10,
+  learningRate: 0.1,
+  targetPositiveRate: 0.7
+}
+
+export class AdaptiveLearner {
+  private logger: Logger
+  private feedbackStore: FeedbackStore
+  private config: AdaptiveLearnerConfig
+  private currentThresholds: AdaptiveThresholds
+
+  constructor(
+    logger: Logger,
+    feedbackStore: FeedbackStore,
+    config: Partial<AdaptiveLearnerConfig> = {}
+  ) {
+    this.logger = logger.child({ component: 'adaptive-learner' })
+    this.feedbackStore = feedbackStore
+    this.config = { ...DEFAULT_CONFIG, ...config }
+    this.currentThresholds = { ...DEFAULT_THRESHOLDS }
+  }
+
+  /**
+   * Get current adaptive thresholds
+   */
+  getThresholds(): AdaptiveThresholds {
+    return { ...this.currentThresholds }
+  }
+
+  /**
+   * Update thresholds based on feedback
+   */
+  async updateThresholds(): Promise<AdaptiveThresholds> {
+    const feedback = await this.feedbackStore.getRecentFeedback(1000)
+
+    if (feedback.length < this.config.minFeedbackForAdaptation) {
+      this.logger.debug({
+        feedbackCount: feedback.length,
+        required: this.config.minFeedbackForAdaptation
+      }, 'Not enough feedback for adaptation')
+      return this.currentThresholds
+    }
+
+    this.logger.info({ feedbackCount: feedback.length }, 'Adapting thresholds')
+
+    const metrics = calculateAllMetrics(feedback)
+    const positiveRate = calculatePositiveRate(feedback)
+    const avgRating = calculateAverageRating(feedback)
+
+    // Analyze feedback by provenance
+    const denseOnly = feedback.filter(f => this.inferProvenance(f) === 'dense')
+    const sparseOnly = feedback.filter(f => this.inferProvenance(f) === 'sparse')
+    const both = feedback.filter(f => this.inferProvenance(f) === 'both')
+
+    const densePositiveRate = denseOnly.length > 0 ? calculatePositiveRate(denseOnly) : 0.5
+    const sparsePositiveRate = sparseOnly.length > 0 ? calculatePositiveRate(sparseOnly) : 0.5
+    const bothPositiveRate = both.length > 0 ? calculatePositiveRate(both) : 0.5
+
+    // Adjust dense/sparse weights based on performance
+    if (denseOnly.length >= 5 && sparseOnly.length >= 5) {
+      // If dense performs better, increase its weight
+      if (densePositiveRate > sparsePositiveRate + 0.1) {
+        this.currentThresholds.denseWeight = this.adjustValue(
+          this.currentThresholds.denseWeight,
+          this.config.learningRate,
+          'increase',
+          BOUNDS.denseWeight
+        )
+        this.currentThresholds.sparseWeight = this.adjustValue(
+          this.currentThresholds.sparseWeight,
+          this.config.learningRate,
+          'decrease',
+          BOUNDS.sparseWeight
+        )
+      }
+      // If sparse performs better, increase its weight
+      else if (sparsePositiveRate > densePositiveRate + 0.1) {
+        this.currentThresholds.sparseWeight = this.adjustValue(
+          this.currentThresholds.sparseWeight,
+          this.config.learningRate,
+          'increase',
+          BOUNDS.sparseWeight
+        )
+        this.currentThresholds.denseWeight = this.adjustValue(
+          this.currentThresholds.denseWeight,
+          this.config.learningRate,
+          'decrease',
+          BOUNDS.denseWeight
+        )
+      }
+    }
+
+    // Adjust minimum similarity based on positive rate
+    if (positiveRate < this.config.targetPositiveRate - 0.1) {
+      // Too many negative results - lower threshold to get more results
+      this.currentThresholds.denseMinSimilarity = this.adjustValue(
+        this.currentThresholds.denseMinSimilarity,
+        this.config.learningRate,
+        'decrease',
+        BOUNDS.denseMinSimilarity
+      )
+    } else if (positiveRate > this.config.targetPositiveRate + 0.1) {
+      // Results are good - can raise threshold to be more selective
+      this.currentThresholds.denseMinSimilarity = this.adjustValue(
+        this.currentThresholds.denseMinSimilarity,
+        this.config.learningRate / 2, // More conservative increase
+        'increase',
+        BOUNDS.denseMinSimilarity
+      )
+    }
+
+    // Adjust RRF K based on metrics
+    // Higher K reduces effect of rank, lower K emphasizes top ranks
+    if (metrics.mrr < 0.5 && both.length > 0) {
+      // If MRR is low and we have combined results, try lower K
+      this.currentThresholds.rrfK = this.adjustValue(
+        this.currentThresholds.rrfK,
+        5, // Adjust by fixed amount
+        'decrease',
+        BOUNDS.rrfK
+      )
+    }
+
+    this.currentThresholds.feedbackCount = feedback.length
+    this.currentThresholds.lastUpdated = new Date().toISOString()
+
+    this.logger.info({
+      thresholds: this.currentThresholds,
+      metrics,
+      positiveRate,
+      avgRating
+    }, 'Thresholds adapted')
+
+    return this.currentThresholds
+  }
+
+  /**
+   * Adjust a value with bounds checking
+   */
+  private adjustValue(
+    current: number,
+    amount: number,
+    direction: 'increase' | 'decrease',
+    bounds: { min: number; max: number }
+  ): number {
+    const delta = direction === 'increase' ? amount : -amount
+    const newValue = current + delta
+    return Math.max(bounds.min, Math.min(bounds.max, newValue))
+  }
+
+  /**
+   * Infer provenance from feedback (heuristic)
+   * In a real implementation, this would be stored in feedback
+   */
+  private inferProvenance(feedback: MemoryFeedback): 'dense' | 'sparse' | 'both' {
+    // This is a heuristic - in production, provenance should be stored with feedback
+    // For now, assume most results are from combined search
+    return 'both'
+  }
+
+  /**
+   * Get adaptation status
+   */
+  getStatus(): {
+    feedbackCount: number
+    canAdapt: boolean
+    currentThresholds: AdaptiveThresholds
+    lastUpdated: string
+  } {
+    return {
+      feedbackCount: this.currentThresholds.feedbackCount,
+      canAdapt: this.currentThresholds.feedbackCount >= this.config.minFeedbackForAdaptation,
+      currentThresholds: this.currentThresholds,
+      lastUpdated: this.currentThresholds.lastUpdated
+    }
+  }
+
+  /**
+   * Reset thresholds to defaults
+   */
+  reset(): void {
+    this.currentThresholds = { ...DEFAULT_THRESHOLDS }
+    this.logger.info('Thresholds reset to defaults')
+  }
+}