evalir 0.0.1 → 0.0.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    evalir (0.0.1)
+    evalir (0.0.2)
 
 GEM
   remote: http://rubygems.org/

data/README.md

@@ -12,6 +12,8 @@ What can Evalir do?
 * Precision at rank k
 * Average Precision
 * Precision-Recall curve
+* Reciprocal Rank
+* [Mean Reciprocal Rank](http://en.wikipedia.org/wiki/Mean_reciprocal_rank)
 * [Mean Average Precision (MAP)](http://en.wikipedia.org/wiki/Information_retrieval#Mean_average_precision)
 * [F-measure](http://en.wikipedia.org/wiki/Information_retrieval#F-measure)
 * [R-Precision](http://en.wikipedia.org/wiki/Information_retrieval#R-Precision)
@@ -44,6 +46,7 @@ For example, we have the aforementioned information need and query, and a list o
     puts "F-3: #{e.f_measure(3)}"
 	puts "Precision at rank 10: #{e.precision_at_rank(10)}"
 	puts "Average Precision: #{e.average_precision}"
+	puts "NDCG @ 5: #{e.ndcg_at(5)}"
 	
 When you have several information needs and want to compute aggregate statistics, use an EvaliratorCollection like this:
 
@@ -51,7 +54,9 @@ When you have several information needs and want to compute aggregate statistics
 	queries.each do |query|
 	  relevant = get_relevant_docids(query)
 	  results = get_results(query)
-	  e << Evalir.Evalirator.new(relevant, results)
+	  e.add(relevant, results)
 	end
+	
 	puts "MAP: #{e.mean_average_precision}"
-	puts "Precision-Recall Curve: #{e.precision_recall_curve}"
+	puts "Precision-Recall Curve: #{e.precision_recall_curve}"
+	puts "Avg. NDCG @ 3: #{e.average_ndcg_at(3)}"

data/lib/evalir/evalirator.rb

@@ -149,6 +149,17 @@ module Evalir
       avg
     end
     
+    # The reciprocal rank, meaning
+    # 1 divided by the rank of the
+    # most highly ranked relevant
+    # result.
+    def reciprocal_rank
+      @search_hits.each_with_index do |h,i|
+        return 1.0 / (i + 1) if @relevant_docids.include? h
+      end
+      return 0.0
+    end
+    
     # Discounted Cumulative Gain at
     # rank k. For a relevant search
     # result at position x, its con-
@@ -167,5 +178,26 @@ module Evalir
       end
       dcg
     end
+    
+    # Normalized Discounted Cumulative
+    # Gain at rank <em>k</em>. This is
+    # the #dcg_at normalized by the optimal
+    # dcg value at rank k.
+    def ndcg_at(k, logbase=2)
+      dcg = dcg_at(k, logbase)
+      dcg > 0 ? dcg / ideal_dcg_at(k, logbase) : 0
+    end
+    
+    private
+      def ideal_dcg_at(k, logbase=2)
+        idcg = 0.0
+        n = @true_positives
+        (1..k).each do |i|
+          break unless n > 0
+          idcg += i == 1 ? 1.0 : 1.0 / Math.log(i, logbase)
+          n -= 1
+        end
+        idcg
+      end
   end
 end

data/lib/evalir/evalirator_collection.rb

@@ -43,11 +43,11 @@ module Evalir
     # a fancy way of saying 'average average
     # precision'!
     def mean_average_precision
-      avg = 0.0
-      @evalirators.each do |e|
-        avg += (e.average_precision / @evalirators.size)
-      end
-      avg
+      @evalirators.reduce(0.0) {|avg,e| avg + (e.average_precision / @evalirators.size)}
+    end
+    
+    def mean_reciprocal_rank
+      self.reduce(0.0) { |avg,e| avg + (e.reciprocal_rank / self.size)}
     end
     
     # Gets the data for the precision-recall
@@ -55,17 +55,25 @@ module Evalir
     # <em>to</em>], with a step size of <em>step</em>.
     # This is the average over all evalirators.
     def precision_recall_curve(from = 0, to = 100, step = 10)
+      raise "From must be in the interval [0, 100)" unless (from >= 0 and from < 100)
+      raise "To must be in the interval (from, 100]" unless (to > from and to <= 100)
+      raise "Invalid step size - (to-from) must be divisible by step." unless ((to - from) % step) == 0
       return nil if @evalirators.empty?
-      
-      #n = self.size.to_f
-      x = 1
+
+      steps = ((to - from) / step) + 1
       curves = self.lazy_map { |e| e.precision_recall_curve(from, to, step) }
-      return curves.reduce do |acc, data|
-        x += 1
+      curves.reduce([0] * steps) do |acc, data|
         data.each_with_index.map do |d,i|
-          acc[i] = (acc[i] + d) / x
+          acc[i] += d / self.size
         end
       end
     end
+    
+    # Gets the average Normalized Discounted
+    # Cumulative Gain over all queries.
+    def average_ndcg_at(k, logbase = 2)
+      values = self.lazy_map {|e| e.ndcg_at(k, logbase)}
+      values.reduce(0.0) { |acc, v| acc + (v / self.size) }
+    end
   end
 end

data/lib/evalir/version.rb

@@ -1,3 +1,3 @@
 module Evalir
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/test/test_evalirator_collection.rb

@@ -16,9 +16,18 @@ class EvaliratorCollectionTest < Test::Unit::TestCase
     assert_equal(2, @e.count)
   end
   
+  def test_mean_reciprocal_rank
+    expected = (1.0 + 0.5) / 2
+    assert_equal(expected, @e.mean_reciprocal_rank)
+  end
+  
   def test_precision_recall_curve
     expected = [1.0, 0.5, 0.5, 0.5, 0.375, 0.4, 0.417, 0.429, 0.375, 0.389, 0.4]
     actual = @e.precision_recall_curve.collect { |f| f.round(3) }
     assert_equal(expected, actual)
   end
+  
+  def test_average_ndcg_at_3
+    assert_equal(0.5, @e.average_ndcg_at(3).round(2))
+  end
 end

data/test/test_evalirator_ranked.rb

@@ -36,6 +36,10 @@ class EvaliratorRankedTest < Test::Unit::TestCase
     assert_equal(0.4, @e.r_precision)
   end
   
+  def test_reciprocal_rank
+    assert_equal(1.0, @e.reciprocal_rank)
+  end
+  
   def test_average_precision
     e1 = Evalir::Evalirator.new([1,3,4,5,6,10], [1,2,3,4,5,6,7,8,9,10])
     assert_equal(0.78, e1.average_precision.round(2))
@@ -48,5 +52,21 @@ class EvaliratorRankedTest < Test::Unit::TestCase
     expected = 1.0 + (1.0/Math.log(3,2))
     assert_equal(expected, @e.dcg_at(5))
   end
+
+  def test_ndcg_at_3
+    dcg = 1.0 + (1.0/Math.log(3,2))
+    idcg = 2.0 + (1.0/Math.log(3,2))
+    assert_equal(dcg/idcg, @e.ndcg_at(3))
+  end
+  
+  def test_dcg_when_no_relevant
+    e = Evalir::Evalirator.new([1,2,3],[4,5,6])
+    assert_equal(0.0, e.dcg_at(3))
+  end
+    
+  def test_ndcg_when_no_relevant
+    e = Evalir::Evalirator.new([1,2,3], [4,5,6])
+    assert_equal(0.0, e.ndcg_at(3))
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: evalir
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-09-30 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70244820710980 !ruby/object:Gem::Requirement
+  requirement: &70127782700140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70244820710980
+  version_requirements: *70127782700140
 description: Evalir is used to measure search relevance at Companybook, and offers
   a number of standard measurements, from the basic precision and recall to single
   value summaries such as NDCG and MAP.
@@ -58,7 +58,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1697956995838933814
+      hash: 3533848118518243422
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -67,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1697956995838933814
+      hash: 3533848118518243422
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10