claude-brain 0.3.0

package/src/retrieval/feedback/index.ts

@@ -0,0 +1,16 @@
+/**
+ * Feedback Module Exports
+ */
+
+export { FeedbackStore } from './store'
+export {
+  FeedbackMetrics,
+  calculateMRR,
+  calculatePrecisionAtK,
+  calculateNDCG,
+  calculateAllMetrics,
+  calculateAverageRating,
+  calculatePositiveRate,
+  calculateUsageRate
+} from './metrics'
+export { AdaptiveLearner, type AdaptiveLearnerConfig } from './adaptive'

package/src/retrieval/feedback/metrics.ts

@@ -0,0 +1,223 @@
+/**
+ * Retrieval Metrics
+ * Evaluation metrics for retrieval quality: MRR, Precision@K, NDCG
+ */
+
+import type { MemoryFeedback, RetrievalMetrics } from '../types'
+
+/**
+ * Calculate Mean Reciprocal Rank (MRR)
+ *
+ * MRR measures where the first relevant result appears in the ranking.
+ * MRR = (1/|Q|) * sum(1/rank_i) for each query
+ *
+ * @param feedbackByQuery - Feedback grouped by query
+ * @param relevanceThreshold - Minimum rating to consider relevant (default: 4)
+ */
+export function calculateMRR(
+  feedbackByQuery: Map<string, MemoryFeedback[]>,
+  relevanceThreshold: number = 4
+): number {
+  if (feedbackByQuery.size === 0) return 0
+
+  let sumReciprocal = 0
+  let queryCount = 0
+
+  for (const [, feedbacks] of feedbackByQuery) {
+    // Sort by implicit rank (assuming feedback order matches retrieval order)
+    const sorted = [...feedbacks]
+
+    // Find first relevant result
+    let foundRelevant = false
+    for (let rank = 0; rank < sorted.length; rank++) {
+      if (sorted[rank].rating >= relevanceThreshold) {
+        sumReciprocal += 1 / (rank + 1)
+        foundRelevant = true
+        break
+      }
+    }
+
+    if (sorted.length > 0) {
+      queryCount++
+    }
+  }
+
+  return queryCount > 0 ? sumReciprocal / queryCount : 0
+}
+
+/**
+ * Calculate Precision at K
+ *
+ * Precision@K = (# relevant in top K) / K
+ *
+ * @param feedbackByQuery - Feedback grouped by query
+ * @param k - Number of top results to consider
+ * @param relevanceThreshold - Minimum rating to consider relevant
+ */
+export function calculatePrecisionAtK(
+  feedbackByQuery: Map<string, MemoryFeedback[]>,
+  k: number,
+  relevanceThreshold: number = 4
+): number {
+  if (feedbackByQuery.size === 0) return 0
+
+  let sumPrecision = 0
+  let queryCount = 0
+
+  for (const [, feedbacks] of feedbackByQuery) {
+    const topK = feedbacks.slice(0, k)
+    if (topK.length === 0) continue
+
+    const relevantCount = topK.filter(f => f.rating >= relevanceThreshold).length
+    sumPrecision += relevantCount / k
+
+    queryCount++
+  }
+
+  return queryCount > 0 ? sumPrecision / queryCount : 0
+}
+
+/**
+ * Calculate Normalized Discounted Cumulative Gain (NDCG)
+ *
+ * NDCG measures ranking quality, giving higher weight to relevant results at the top.
+ * DCG = sum(rating_i / log2(i + 1))
+ * NDCG = DCG / IDCG (ideal DCG)
+ *
+ * @param feedbackByQuery - Feedback grouped by query
+ * @param k - Number of top results to consider (optional, default all)
+ */
+export function calculateNDCG(
+  feedbackByQuery: Map<string, MemoryFeedback[]>,
+  k?: number
+): number {
+  if (feedbackByQuery.size === 0) return 0
+
+  let sumNDCG = 0
+  let queryCount = 0
+
+  for (const [, feedbacks] of feedbackByQuery) {
+    if (feedbacks.length === 0) continue
+
+    const results = k ? feedbacks.slice(0, k) : feedbacks
+
+    // Calculate DCG
+    let dcg = 0
+    for (let i = 0; i < results.length; i++) {
+      // Use rating as relevance score (1-5)
+      const relevance = results[i].rating
+      dcg += relevance / Math.log2(i + 2) // +2 because log2(1) = 0
+    }
+
+    // Calculate ideal DCG (sort by rating descending)
+    const idealResults = [...results].sort((a, b) => b.rating - a.rating)
+    let idcg = 0
+    for (let i = 0; i < idealResults.length; i++) {
+      const relevance = idealResults[i].rating
+      idcg += relevance / Math.log2(i + 2)
+    }
+
+    // Normalize
+    if (idcg > 0) {
+      sumNDCG += dcg / idcg
+    }
+
+    queryCount++
+  }
+
+  return queryCount > 0 ? sumNDCG / queryCount : 0
+}
+
+/**
+ * Calculate all retrieval metrics from feedback
+ */
+export function calculateAllMetrics(
+  feedback: MemoryFeedback[],
+  kValues: number[] = [3, 5, 10]
+): RetrievalMetrics {
+  // Group feedback by query
+  const feedbackByQuery = new Map<string, MemoryFeedback[]>()
+
+  for (const f of feedback) {
+    const existing = feedbackByQuery.get(f.queryId) || []
+    existing.push(f)
+    feedbackByQuery.set(f.queryId, existing)
+  }
+
+  // Calculate precision at each K
+  const precisionAtK: Record<number, number> = {}
+  for (const k of kValues) {
+    precisionAtK[k] = calculatePrecisionAtK(feedbackByQuery, k)
+  }
+
+  return {
+    mrr: calculateMRR(feedbackByQuery),
+    precisionAtK,
+    ndcg: calculateNDCG(feedbackByQuery),
+    queryCount: feedbackByQuery.size,
+    avgLatencyMs: 0 // Not tracked in feedback
+  }
+}
+
+/**
+ * Calculate average rating from feedback
+ */
+export function calculateAverageRating(feedback: MemoryFeedback[]): number {
+  if (feedback.length === 0) return 0
+  const sum = feedback.reduce((acc, f) => acc + f.rating, 0)
+  return sum / feedback.length
+}
+
+/**
+ * Calculate positive feedback rate (rating >= 4)
+ */
+export function calculatePositiveRate(
+  feedback: MemoryFeedback[],
+  threshold: number = 4
+): number {
+  if (feedback.length === 0) return 0
+  const positive = feedback.filter(f => f.rating >= threshold).length
+  return positive / feedback.length
+}
+
+/**
+ * Calculate usage rate (was_used = true)
+ */
+export function calculateUsageRate(feedback: MemoryFeedback[]): number {
+  if (feedback.length === 0) return 0
+  const used = feedback.filter(f => f.wasUsed).length
+  return used / feedback.length
+}
+
+export class FeedbackMetrics {
+  calculateMRR(feedback: Map<string, MemoryFeedback[]>): number {
+    return calculateMRR(feedback)
+  }
+
+  calculatePrecisionAtK(
+    feedback: Map<string, MemoryFeedback[]>,
+    k: number
+  ): number {
+    return calculatePrecisionAtK(feedback, k)
+  }
+
+  calculateNDCG(feedback: Map<string, MemoryFeedback[]>): number {
+    return calculateNDCG(feedback)
+  }
+
+  calculateAll(feedback: MemoryFeedback[]): RetrievalMetrics {
+    return calculateAllMetrics(feedback)
+  }
+
+  calculateAverageRating(feedback: MemoryFeedback[]): number {
+    return calculateAverageRating(feedback)
+  }
+
+  calculatePositiveRate(feedback: MemoryFeedback[]): number {
+    return calculatePositiveRate(feedback)
+  }
+
+  calculateUsageRate(feedback: MemoryFeedback[]): number {
+    return calculateUsageRate(feedback)
+  }
+}

package/src/retrieval/feedback/store.ts

@@ -0,0 +1,283 @@
+/**
+ * Feedback Store
+ * Stores and retrieves memory feedback for adaptive learning
+ */
+
+import type { Logger } from 'pino'
+import type { CollectionManager } from '@/memory/chroma/collection-manager'
+import type { EmbeddingProvider } from '@/memory/chroma/embeddings'
+import type { MemoryFeedback, RateMemoryInput } from '../types'
+
+export class FeedbackStore {
+  private logger: Logger
+  private collections: CollectionManager
+  private embeddings: EmbeddingProvider
+
+  constructor(
+    logger: Logger,
+    collections: CollectionManager,
+    embeddings: EmbeddingProvider
+  ) {
+    this.logger = logger.child({ component: 'feedback-store' })
+    this.collections = collections
+    this.embeddings = embeddings
+  }
+
+  /**
+   * Store feedback for a memory retrieval
+   */
+  async storeFeedback(input: RateMemoryInput): Promise<string> {
+    const feedbackId = `feedback_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
+    const queryId = `query_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
+    const timestamp = new Date().toISOString()
+
+    this.logger.debug({
+      feedbackId,
+      memoryId: input.memory_id,
+      rating: input.rating
+    }, 'Storing feedback')
+
+    try {
+      const collection = await this.collections.getFeedback()
+
+      // Generate embedding for the query to enable similarity search
+      const embedding = await this.embeddings.generate(input.query)
+
+      // Determine collection type from memory_id prefix
+      const collectionType = this.inferCollectionType(input.memory_id)
+
+      await collection.add({
+        ids: [feedbackId],
+        embeddings: [embedding],
+        documents: [input.query],
+        metadatas: [{
+          memory_id: input.memory_id,
+          query_id: queryId,
+          query: input.query,
+          rating: input.rating,
+          was_used: input.was_used ?? true,
+          feedback_text: input.feedback_text || '',
+          project: input.project_name || '',
+          created_at: timestamp,
+          collection_type: collectionType
+        }]
+      })
+
+      this.logger.info({
+        feedbackId,
+        memoryId: input.memory_id,
+        rating: input.rating
+      }, 'Feedback stored successfully')
+
+      return feedbackId
+
+    } catch (error) {
+      this.logger.error({ error, input }, 'Failed to store feedback')
+      throw error
+    }
+  }
+
+  /**
+   * Get feedback for a specific memory
+   */
+  async getFeedbackForMemory(memoryId: string): Promise<MemoryFeedback[]> {
+    try {
+      const collection = await this.collections.getFeedback()
+
+      const results = await collection.get({
+        where: { memory_id: { $eq: memoryId } },
+        include: ['documents', 'metadatas']
+      })
+
+      if (!results.ids.length) {
+        return []
+      }
+
+      return results.ids.map((id, index) => ({
+        id,
+        memoryId: results.metadatas?.[index]?.memory_id as string || '',
+        queryId: results.metadatas?.[index]?.query_id as string || '',
+        query: results.documents?.[index] || '',
+        rating: results.metadatas?.[index]?.rating as 1 | 2 | 3 | 4 | 5,
+        wasUsed: results.metadatas?.[index]?.was_used as boolean ?? true,
+        feedbackText: results.metadatas?.[index]?.feedback_text as string | undefined,
+        project: results.metadatas?.[index]?.project as string | undefined,
+        timestamp: results.metadatas?.[index]?.created_at as string || '',
+        collectionType: results.metadatas?.[index]?.collection_type as 'decisions' | 'memories' | 'patterns' | 'corrections'
+      }))
+
+    } catch (error) {
+      this.logger.error({ error, memoryId }, 'Failed to get feedback for memory')
+      return []
+    }
+  }
+
+  /**
+   * Get all feedback for a project
+   */
+  async getFeedbackForProject(project: string, limit: number = 100): Promise<MemoryFeedback[]> {
+    try {
+      const collection = await this.collections.getFeedback()
+
+      const results = await collection.get({
+        where: { project: { $eq: project } },
+        include: ['documents', 'metadatas'],
+        limit
+      })
+
+      if (!results.ids.length) {
+        return []
+      }
+
+      return results.ids.map((id, index) => ({
+        id,
+        memoryId: results.metadatas?.[index]?.memory_id as string || '',
+        queryId: results.metadatas?.[index]?.query_id as string || '',
+        query: results.documents?.[index] || '',
+        rating: results.metadatas?.[index]?.rating as 1 | 2 | 3 | 4 | 5,
+        wasUsed: results.metadatas?.[index]?.was_used as boolean ?? true,
+        feedbackText: results.metadatas?.[index]?.feedback_text as string | undefined,
+        project: results.metadatas?.[index]?.project as string | undefined,
+        timestamp: results.metadatas?.[index]?.created_at as string || '',
+        collectionType: results.metadatas?.[index]?.collection_type as 'decisions' | 'memories' | 'patterns' | 'corrections'
+      }))
+
+    } catch (error) {
+      this.logger.error({ error, project }, 'Failed to get feedback for project')
+      return []
+    }
+  }
+
+  /**
+   * Get recent feedback (for adaptive learning)
+   */
+  async getRecentFeedback(limit: number = 100): Promise<MemoryFeedback[]> {
+    try {
+      const collection = await this.collections.getFeedback()
+
+      // Get all feedback - ChromaDB doesn't support ordering, so we'll sort in memory
+      const results = await collection.get({
+        include: ['documents', 'metadatas'],
+        limit: limit * 2 // Get more to allow for sorting
+      })
+
+      if (!results.ids.length) {
+        return []
+      }
+
+      const feedback: MemoryFeedback[] = results.ids.map((id, index) => ({
+        id,
+        memoryId: results.metadatas?.[index]?.memory_id as string || '',
+        queryId: results.metadatas?.[index]?.query_id as string || '',
+        query: results.documents?.[index] || '',
+        rating: results.metadatas?.[index]?.rating as 1 | 2 | 3 | 4 | 5,
+        wasUsed: results.metadatas?.[index]?.was_used as boolean ?? true,
+        feedbackText: results.metadatas?.[index]?.feedback_text as string | undefined,
+        project: results.metadatas?.[index]?.project as string | undefined,
+        timestamp: results.metadatas?.[index]?.created_at as string || '',
+        collectionType: results.metadatas?.[index]?.collection_type as 'decisions' | 'memories' | 'patterns' | 'corrections'
+      }))
+
+      // Sort by timestamp descending and take limit
+      return feedback
+        .sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime())
+        .slice(0, limit)
+
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to get recent feedback')
+      return []
+    }
+  }
+
+  /**
+   * Get feedback statistics
+   */
+  async getStats(): Promise<{
+    totalFeedback: number
+    avgRating: number
+    positiveRate: number
+    usageRate: number
+    byCollection: Record<string, { count: number; avgRating: number }>
+  }> {
+    try {
+      const collection = await this.collections.getFeedback()
+      const totalCount = await collection.count()
+
+      if (totalCount === 0) {
+        return {
+          totalFeedback: 0,
+          avgRating: 0,
+          positiveRate: 0,
+          usageRate: 0,
+          byCollection: {}
+        }
+      }
+
+      const results = await collection.get({
+        include: ['metadatas'],
+        limit: 1000 // Sample for stats
+      })
+
+      let totalRating = 0
+      let positiveCount = 0
+      let usedCount = 0
+      const byCollection: Record<string, { count: number; totalRating: number }> = {}
+
+      for (const metadata of results.metadatas || []) {
+        const rating = metadata?.rating as number
+        const wasUsed = metadata?.was_used as boolean
+        const collectionType = (metadata?.collection_type as string) || 'unknown'
+
+        totalRating += rating
+        if (rating >= 4) positiveCount++
+        if (wasUsed) usedCount++
+
+        if (!byCollection[collectionType]) {
+          byCollection[collectionType] = { count: 0, totalRating: 0 }
+        }
+        byCollection[collectionType].count++
+        byCollection[collectionType].totalRating += rating
+      }
+
+      const count = results.ids.length
+
+      return {
+        totalFeedback: totalCount,
+        avgRating: count > 0 ? totalRating / count : 0,
+        positiveRate: count > 0 ? positiveCount / count : 0,
+        usageRate: count > 0 ? usedCount / count : 0,
+        byCollection: Object.fromEntries(
+          Object.entries(byCollection).map(([key, value]) => [
+            key,
+            {
+              count: value.count,
+              avgRating: value.count > 0 ? value.totalRating / value.count : 0
+            }
+          ])
+        )
+      }
+
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to get feedback stats')
+      return {
+        totalFeedback: 0,
+        avgRating: 0,
+        positiveRate: 0,
+        usageRate: 0,
+        byCollection: {}
+      }
+    }
+  }
+
+  /**
+   * Infer collection type from memory ID prefix
+   */
+  private inferCollectionType(memoryId: string): 'decisions' | 'memories' | 'patterns' | 'corrections' {
+    if (memoryId.startsWith('dec_')) return 'decisions'
+    if (memoryId.startsWith('mem_')) return 'memories'
+    if (memoryId.startsWith('pat_')) return 'patterns'
+    if (memoryId.startsWith('cor_')) return 'corrections'
+    // Default to decisions if unknown
+    return 'decisions'
+  }
+}

package/src/retrieval/fusion/index.ts

@@ -0,0 +1,194 @@
+/**
+ * Fusion Strategies
+ * Different methods for combining dense and sparse search results
+ */
+
+import type { HybridSearchResult } from '../types'
+import { fuseWithRRF, fuseMultipleWithRRF, calculateRRFScore } from './rrf'
+
+export interface FusionConfig {
+  /** Fusion method to use */
+  method: 'rrf' | 'linear' | 'max'
+  /** RRF k parameter (for RRF method) */
+  rrfK?: number
+  /** Dense weight (for linear method) */
+  denseWeight?: number
+  /** Sparse weight (for linear method) */
+  sparseWeight?: number
+}
+
+const DEFAULT_CONFIG: FusionConfig = {
+  method: 'rrf',
+  rrfK: 60,
+  denseWeight: 0.7,
+  sparseWeight: 0.3
+}
+
+/**
+ * RRF Fusion - Reciprocal Rank Fusion
+ */
+export class RRFFusion {
+  private k: number
+
+  constructor(k: number = 60) {
+    this.k = k
+  }
+
+  fuse(
+    denseResults: HybridSearchResult[],
+    sparseResults: HybridSearchResult[]
+  ): HybridSearchResult[] {
+    return fuseWithRRF(denseResults, sparseResults, { k: this.k })
+  }
+
+  fuseMultiple(rankedLists: HybridSearchResult[][]): HybridSearchResult[] {
+    return fuseMultipleWithRRF(rankedLists, { k: this.k })
+  }
+}
+
+/**
+ * Linear Fusion - Weighted combination of scores
+ */
+export class LinearFusion {
+  private denseWeight: number
+  private sparseWeight: number
+
+  constructor(denseWeight: number = 0.7, sparseWeight: number = 0.3) {
+    this.denseWeight = denseWeight
+    this.sparseWeight = sparseWeight
+  }
+
+  fuse(
+    denseResults: HybridSearchResult[],
+    sparseResults: HybridSearchResult[]
+  ): HybridSearchResult[] {
+    // Create map for merging
+    const merged = new Map<string, HybridSearchResult>()
+
+    // Add dense results
+    for (const result of denseResults) {
+      merged.set(result.id, {
+        ...result,
+        scores: {
+          ...result.scores,
+          fusion: result.scores.dense * this.denseWeight,
+          final: result.scores.dense * this.denseWeight
+        },
+        provenance: 'dense'
+      })
+    }
+
+    // Add/merge sparse results
+    for (const result of sparseResults) {
+      const existing = merged.get(result.id)
+      if (existing) {
+        const combinedScore =
+          existing.scores.fusion + result.scores.sparse * this.sparseWeight
+        merged.set(result.id, {
+          ...existing,
+          scores: {
+            ...existing.scores,
+            sparse: result.scores.sparse,
+            fusion: combinedScore,
+            final: combinedScore
+          },
+          provenance: 'both'
+        })
+      } else {
+        merged.set(result.id, {
+          ...result,
+          scores: {
+            ...result.scores,
+            fusion: result.scores.sparse * this.sparseWeight,
+            final: result.scores.sparse * this.sparseWeight
+          },
+          provenance: 'sparse'
+        })
+      }
+    }
+
+    // Sort by fusion score
+    const results = Array.from(merged.values())
+    results.sort((a, b) => b.scores.fusion - a.scores.fusion)
+
+    return results
+  }
+}
+
+/**
+ * Max Fusion - Take maximum score from either list
+ */
+export class MaxFusion {
+  fuse(
+    denseResults: HybridSearchResult[],
+    sparseResults: HybridSearchResult[]
+  ): HybridSearchResult[] {
+    // Create map for merging
+    const merged = new Map<string, HybridSearchResult>()
+
+    // Add dense results
+    for (const result of denseResults) {
+      merged.set(result.id, {
+        ...result,
+        scores: {
+          ...result.scores,
+          fusion: result.scores.dense,
+          final: result.scores.dense
+        },
+        provenance: 'dense'
+      })
+    }
+
+    // Add/merge sparse results
+    for (const result of sparseResults) {
+      const existing = merged.get(result.id)
+      if (existing) {
+        const maxScore = Math.max(existing.scores.dense || 0, result.scores.sparse)
+        merged.set(result.id, {
+          ...existing,
+          scores: {
+            ...existing.scores,
+            sparse: result.scores.sparse,
+            fusion: maxScore,
+            final: maxScore
+          },
+          provenance: 'both'
+        })
+      } else {
+        merged.set(result.id, {
+          ...result,
+          scores: {
+            ...result.scores,
+            fusion: result.scores.sparse,
+            final: result.scores.sparse
+          },
+          provenance: 'sparse'
+        })
+      }
+    }
+
+    // Sort by fusion score
+    const results = Array.from(merged.values())
+    results.sort((a, b) => b.scores.fusion - a.scores.fusion)
+
+    return results
+  }
+}
+
+/**
+ * Create a fusion strategy based on config
+ */
+export function createFusionStrategy(config: FusionConfig = DEFAULT_CONFIG) {
+  switch (config.method) {
+    case 'rrf':
+      return new RRFFusion(config.rrfK)
+    case 'linear':
+      return new LinearFusion(config.denseWeight, config.sparseWeight)
+    case 'max':
+      return new MaxFusion()
+    default:
+      return new RRFFusion(config.rrfK)
+  }
+}
+
+export { fuseWithRRF, fuseMultipleWithRRF, calculateRRFScore } from './rrf'