claude-brain 0.14.2 → 0.14.4

package/src/retrieval/feedback/metrics.ts

@@ -1,223 +1,223 @@
-/**
- * Retrieval Metrics
- * Evaluation metrics for retrieval quality: MRR, Precision@K, NDCG
- */
-
-import type { MemoryFeedback, RetrievalMetrics } from '../types'
-
-/**
- * Calculate Mean Reciprocal Rank (MRR)
- *
- * MRR measures where the first relevant result appears in the ranking.
- * MRR = (1/|Q|) * sum(1/rank_i) for each query
- *
- * @param feedbackByQuery - Feedback grouped by query
- * @param relevanceThreshold - Minimum rating to consider relevant (default: 4)
- */
-export function calculateMRR(
-  feedbackByQuery: Map<string, MemoryFeedback[]>,
-  relevanceThreshold: number = 4
-): number {
-  if (feedbackByQuery.size === 0) return 0
-
-  let sumReciprocal = 0
-  let queryCount = 0
-
-  for (const [, feedbacks] of feedbackByQuery) {
-    // Sort by implicit rank (assuming feedback order matches retrieval order)
-    const sorted = [...feedbacks]
-
-    // Find first relevant result
-    let foundRelevant = false
-    for (let rank = 0; rank < sorted.length; rank++) {
-      if (sorted[rank].rating >= relevanceThreshold) {
-        sumReciprocal += 1 / (rank + 1)
-        foundRelevant = true
-        break
-      }
-    }
-
-    if (sorted.length > 0) {
-      queryCount++
-    }
-  }
-
-  return queryCount > 0 ? sumReciprocal / queryCount : 0
-}
-
-/**
- * Calculate Precision at K
- *
- * Precision@K = (# relevant in top K) / K
- *
- * @param feedbackByQuery - Feedback grouped by query
- * @param k - Number of top results to consider
- * @param relevanceThreshold - Minimum rating to consider relevant
- */
-export function calculatePrecisionAtK(
-  feedbackByQuery: Map<string, MemoryFeedback[]>,
-  k: number,
-  relevanceThreshold: number = 4
-): number {
-  if (feedbackByQuery.size === 0) return 0
-
-  let sumPrecision = 0
-  let queryCount = 0
-
-  for (const [, feedbacks] of feedbackByQuery) {
-    const topK = feedbacks.slice(0, k)
-    if (topK.length === 0) continue
-
-    const relevantCount = topK.filter(f => f.rating >= relevanceThreshold).length
-    sumPrecision += relevantCount / k
-
-    queryCount++
-  }
-
-  return queryCount > 0 ? sumPrecision / queryCount : 0
-}
-
-/**
- * Calculate Normalized Discounted Cumulative Gain (NDCG)
- *
- * NDCG measures ranking quality, giving higher weight to relevant results at the top.
- * DCG = sum(rating_i / log2(i + 1))
- * NDCG = DCG / IDCG (ideal DCG)
- *
- * @param feedbackByQuery - Feedback grouped by query
- * @param k - Number of top results to consider (optional, default all)
- */
-export function calculateNDCG(
-  feedbackByQuery: Map<string, MemoryFeedback[]>,
-  k?: number
-): number {
-  if (feedbackByQuery.size === 0) return 0
-
-  let sumNDCG = 0
-  let queryCount = 0
-
-  for (const [, feedbacks] of feedbackByQuery) {
-    if (feedbacks.length === 0) continue
-
-    const results = k ? feedbacks.slice(0, k) : feedbacks
-
-    // Calculate DCG
-    let dcg = 0
-    for (let i = 0; i < results.length; i++) {
-      // Use rating as relevance score (1-5)
-      const relevance = results[i].rating
-      dcg += relevance / Math.log2(i + 2) // +2 because log2(1) = 0
-    }
-
-    // Calculate ideal DCG (sort by rating descending)
-    const idealResults = [...results].sort((a, b) => b.rating - a.rating)
-    let idcg = 0
-    for (let i = 0; i < idealResults.length; i++) {
-      const relevance = idealResults[i].rating
-      idcg += relevance / Math.log2(i + 2)
-    }
-
-    // Normalize
-    if (idcg > 0) {
-      sumNDCG += dcg / idcg
-    }
-
-    queryCount++
-  }
-
-  return queryCount > 0 ? sumNDCG / queryCount : 0
-}
-
-/**
- * Calculate all retrieval metrics from feedback
- */
-export function calculateAllMetrics(
-  feedback: MemoryFeedback[],
-  kValues: number[] = [3, 5, 10]
-): RetrievalMetrics {
-  // Group feedback by query
-  const feedbackByQuery = new Map<string, MemoryFeedback[]>()
-
-  for (const f of feedback) {
-    const existing = feedbackByQuery.get(f.queryId) || []
-    existing.push(f)
-    feedbackByQuery.set(f.queryId, existing)
-  }
-
-  // Calculate precision at each K
-  const precisionAtK: Record<number, number> = {}
-  for (const k of kValues) {
-    precisionAtK[k] = calculatePrecisionAtK(feedbackByQuery, k)
-  }
-
-  return {
-    mrr: calculateMRR(feedbackByQuery),
-    precisionAtK,
-    ndcg: calculateNDCG(feedbackByQuery),
-    queryCount: feedbackByQuery.size,
-    avgLatencyMs: 0 // Not tracked in feedback
-  }
-}
-
-/**
- * Calculate average rating from feedback
- */
-export function calculateAverageRating(feedback: MemoryFeedback[]): number {
-  if (feedback.length === 0) return 0
-  const sum = feedback.reduce((acc, f) => acc + f.rating, 0)
-  return sum / feedback.length
-}
-
-/**
- * Calculate positive feedback rate (rating >= 4)
- */
-export function calculatePositiveRate(
-  feedback: MemoryFeedback[],
-  threshold: number = 4
-): number {
-  if (feedback.length === 0) return 0
-  const positive = feedback.filter(f => f.rating >= threshold).length
-  return positive / feedback.length
-}
-
-/**
- * Calculate usage rate (was_used = true)
- */
-export function calculateUsageRate(feedback: MemoryFeedback[]): number {
-  if (feedback.length === 0) return 0
-  const used = feedback.filter(f => f.wasUsed).length
-  return used / feedback.length
-}
-
-export class FeedbackMetrics {
-  calculateMRR(feedback: Map<string, MemoryFeedback[]>): number {
-    return calculateMRR(feedback)
-  }
-
-  calculatePrecisionAtK(
-    feedback: Map<string, MemoryFeedback[]>,
-    k: number
-  ): number {
-    return calculatePrecisionAtK(feedback, k)
-  }
-
-  calculateNDCG(feedback: Map<string, MemoryFeedback[]>): number {
-    return calculateNDCG(feedback)
-  }
-
-  calculateAll(feedback: MemoryFeedback[]): RetrievalMetrics {
-    return calculateAllMetrics(feedback)
-  }
-
-  calculateAverageRating(feedback: MemoryFeedback[]): number {
-    return calculateAverageRating(feedback)
-  }
-
-  calculatePositiveRate(feedback: MemoryFeedback[]): number {
-    return calculatePositiveRate(feedback)
-  }
-
-  calculateUsageRate(feedback: MemoryFeedback[]): number {
-    return calculateUsageRate(feedback)
-  }
-}
+/**
+ * Retrieval Metrics
+ * Evaluation metrics for retrieval quality: MRR, Precision@K, NDCG
+ */
+
+import type { MemoryFeedback, RetrievalMetrics } from '../types'
+
+/**
+ * Calculate Mean Reciprocal Rank (MRR)
+ *
+ * MRR measures where the first relevant result appears in the ranking.
+ * MRR = (1/|Q|) * sum(1/rank_i) for each query
+ *
+ * @param feedbackByQuery - Feedback grouped by query
+ * @param relevanceThreshold - Minimum rating to consider relevant (default: 4)
+ */
+export function calculateMRR(
+  feedbackByQuery: Map<string, MemoryFeedback[]>,
+  relevanceThreshold: number = 4
+): number {
+  if (feedbackByQuery.size === 0) return 0
+
+  let sumReciprocal = 0
+  let queryCount = 0
+
+  for (const [, feedbacks] of feedbackByQuery) {
+    // Sort by implicit rank (assuming feedback order matches retrieval order)
+    const sorted = [...feedbacks]
+
+    // Find first relevant result
+    let foundRelevant = false
+    for (let rank = 0; rank < sorted.length; rank++) {
+      if (sorted[rank].rating >= relevanceThreshold) {
+        sumReciprocal += 1 / (rank + 1)
+        foundRelevant = true
+        break
+      }
+    }
+
+    if (sorted.length > 0) {
+      queryCount++
+    }
+  }
+
+  return queryCount > 0 ? sumReciprocal / queryCount : 0
+}
+
+/**
+ * Calculate Precision at K
+ *
+ * Precision@K = (# relevant in top K) / K
+ *
+ * @param feedbackByQuery - Feedback grouped by query
+ * @param k - Number of top results to consider
+ * @param relevanceThreshold - Minimum rating to consider relevant
+ */
+export function calculatePrecisionAtK(
+  feedbackByQuery: Map<string, MemoryFeedback[]>,
+  k: number,
+  relevanceThreshold: number = 4
+): number {
+  if (feedbackByQuery.size === 0) return 0
+
+  let sumPrecision = 0
+  let queryCount = 0
+
+  for (const [, feedbacks] of feedbackByQuery) {
+    const topK = feedbacks.slice(0, k)
+    if (topK.length === 0) continue
+
+    const relevantCount = topK.filter(f => f.rating >= relevanceThreshold).length
+    sumPrecision += relevantCount / k
+
+    queryCount++
+  }
+
+  return queryCount > 0 ? sumPrecision / queryCount : 0
+}
+
+/**
+ * Calculate Normalized Discounted Cumulative Gain (NDCG)
+ *
+ * NDCG measures ranking quality, giving higher weight to relevant results at the top.
+ * DCG = sum(rating_i / log2(i + 1))
+ * NDCG = DCG / IDCG (ideal DCG)
+ *
+ * @param feedbackByQuery - Feedback grouped by query
+ * @param k - Number of top results to consider (optional, default all)
+ */
+export function calculateNDCG(
+  feedbackByQuery: Map<string, MemoryFeedback[]>,
+  k?: number
+): number {
+  if (feedbackByQuery.size === 0) return 0
+
+  let sumNDCG = 0
+  let queryCount = 0
+
+  for (const [, feedbacks] of feedbackByQuery) {
+    if (feedbacks.length === 0) continue
+
+    const results = k ? feedbacks.slice(0, k) : feedbacks
+
+    // Calculate DCG
+    let dcg = 0
+    for (let i = 0; i < results.length; i++) {
+      // Use rating as relevance score (1-5)
+      const relevance = results[i].rating
+      dcg += relevance / Math.log2(i + 2) // +2 because log2(1) = 0
+    }
+
+    // Calculate ideal DCG (sort by rating descending)
+    const idealResults = [...results].sort((a, b) => b.rating - a.rating)
+    let idcg = 0
+    for (let i = 0; i < idealResults.length; i++) {
+      const relevance = idealResults[i].rating
+      idcg += relevance / Math.log2(i + 2)
+    }
+
+    // Normalize
+    if (idcg > 0) {
+      sumNDCG += dcg / idcg
+    }
+
+    queryCount++
+  }
+
+  return queryCount > 0 ? sumNDCG / queryCount : 0
+}
+
+/**
+ * Calculate all retrieval metrics from feedback
+ */
+export function calculateAllMetrics(
+  feedback: MemoryFeedback[],
+  kValues: number[] = [3, 5, 10]
+): RetrievalMetrics {
+  // Group feedback by query
+  const feedbackByQuery = new Map<string, MemoryFeedback[]>()
+
+  for (const f of feedback) {
+    const existing = feedbackByQuery.get(f.queryId) || []
+    existing.push(f)
+    feedbackByQuery.set(f.queryId, existing)
+  }
+
+  // Calculate precision at each K
+  const precisionAtK: Record<number, number> = {}
+  for (const k of kValues) {
+    precisionAtK[k] = calculatePrecisionAtK(feedbackByQuery, k)
+  }
+
+  return {
+    mrr: calculateMRR(feedbackByQuery),
+    precisionAtK,
+    ndcg: calculateNDCG(feedbackByQuery),
+    queryCount: feedbackByQuery.size,
+    avgLatencyMs: 0 // Not tracked in feedback
+  }
+}
+
+/**
+ * Calculate average rating from feedback
+ */
+export function calculateAverageRating(feedback: MemoryFeedback[]): number {
+  if (feedback.length === 0) return 0
+  const sum = feedback.reduce((acc, f) => acc + f.rating, 0)
+  return sum / feedback.length
+}
+
+/**
+ * Calculate positive feedback rate (rating >= 4)
+ */
+export function calculatePositiveRate(
+  feedback: MemoryFeedback[],
+  threshold: number = 4
+): number {
+  if (feedback.length === 0) return 0
+  const positive = feedback.filter(f => f.rating >= threshold).length
+  return positive / feedback.length
+}
+
+/**
+ * Calculate usage rate (was_used = true)
+ */
+export function calculateUsageRate(feedback: MemoryFeedback[]): number {
+  if (feedback.length === 0) return 0
+  const used = feedback.filter(f => f.wasUsed).length
+  return used / feedback.length
+}
+
+export class FeedbackMetrics {
+  calculateMRR(feedback: Map<string, MemoryFeedback[]>): number {
+    return calculateMRR(feedback)
+  }
+
+  calculatePrecisionAtK(
+    feedback: Map<string, MemoryFeedback[]>,
+    k: number
+  ): number {
+    return calculatePrecisionAtK(feedback, k)
+  }
+
+  calculateNDCG(feedback: Map<string, MemoryFeedback[]>): number {
+    return calculateNDCG(feedback)
+  }
+
+  calculateAll(feedback: MemoryFeedback[]): RetrievalMetrics {
+    return calculateAllMetrics(feedback)
+  }
+
+  calculateAverageRating(feedback: MemoryFeedback[]): number {
+    return calculateAverageRating(feedback)
+  }
+
+  calculatePositiveRate(feedback: MemoryFeedback[]): number {
+    return calculatePositiveRate(feedback)
+  }
+
+  calculateUsageRate(feedback: MemoryFeedback[]): number {
+    return calculateUsageRate(feedback)
+  }
+}