retreval 0.1

data/lib/retreval/runner.rb

@@ -0,0 +1,121 @@
+require_relative 'options'
+require_relative 'gold_standard'
+require_relative 'query_result'
+require 'yaml'
+require 'pp'
+
+# The Retreval allows to load and define Gold Standards, add Query Results and 
+# calculate the most common metrics used in information retrieval evaluation.
+module Retreval
+  
+  # A simple class that performs the task of running this library when called
+  # from the commandline
+  class Runner
+    
+    # Invokes a new Runner object by loading the options from the commandline
+    def initialize(args)
+      @options = Options.new(args)
+    end
+    
+    # Takes the passed options for a GoldStandard file and loads it according to the format specified
+    def load_gold_standard
+      unless @options.gold_standard_file.nil?
+        
+          
+        print "Loading gold standard file '#{@options.gold_standard_file}' ... " if $verbose
+        @gold_standard = GoldStandard.new
+        case @options.format
+        when "yaml"
+          @gold_standard.load_from_yaml_file @options.gold_standard_file
+        when "plain"
+          @gold_standard.load_from_plaintext_file @options.gold_standard_file
+        else
+          raise "I don't understand the format '#{@options.format}'"
+        end
+        
+        print "done\n" if $verbose
+        print "Gold standard loaded from #{@options.gold_standard_file} contains:
+          - #{@gold_standard.queries.count} queries, 
+          - #{@gold_standard.documents.count} documents,
+          - #{@gold_standard.judgements.count} judgements, made by
+          - #{@gold_standard.users.count} users\n\n" if $verbose        
+      end
+    end
+    
+    # Takes the passed options for a QueryResultSet file and loads it according to the format specified
+    def load_query_result_set
+      unless @options.query_result_set_file.nil?
+          
+        print "Loading query result set from file '#{@options.query_result_set_file}' ... " if $verbose
+        @query_result_set = QueryResultSet.new :gold_standard => @gold_standard
+        case @options.format
+        when "yaml"
+          @query_result_set.load_from_yaml_file @options.query_result_set_file
+        when "plain"
+          @query_result_set.load_from_yaml_file @options.query_result_set_file
+        else
+          raise "I don't understand the format '#{@options.format}'"
+        end
+        
+        print "done\n" if $verbose
+        print "Query results loaded from #{@options.query_result_set_file} contain:
+          - #{@query_result_set.query_results.count} query results\n\n" if $verbose
+      end
+    end
+    
+    # Performs the default calculations and writes their output to the file specified
+    def begin_calculations
+      @statistics = Hash.new
+      @average_precision = Hash.new
+      
+      @query_result_set.query_results.each_with_index do |result, index|
+        begin
+          print "Cleaning up results and removing documents without judgements ... \n" if $verbose
+          result.cleanup
+          
+          print "Calculating statistics for result #{index+1} of #{@query_result_set.query_results.count} ... "
+          @statistics[result.query.querystring] = result.statistics
+          @average_precision[result.query.querystring] = result.average_precision
+          print "Done.\n"
+          
+          result.print_ranked_table if $verbose          
+          
+          write_to_yaml_file :data => @statistics, :filename => "statistics.yml"
+          write_to_yaml_file :data => @average_precision, :filename => "avg_precision.yml"
+          
+        # rescue Exception => e
+        #  raise "Error while calculating results: #{e}"
+        end
+      end
+      
+      print "Finished calculating all results. Exiting.\n" if $verbose
+      print "The mean average precision was #{@query_result_set.mean_average_precision}\n" if $verbose
+      exit
+      
+    end
+    
+    # Writes an object to a YAML file.
+    # Called by:
+    #     write_to_yaml_file :data => my_data, :filename => "my_data_file.yml"
+    def write_to_yaml_file(args)
+      data = args[:data]
+      filename = args[:filename]
+      
+      if data.nil? or filename.nil? 
+        raise "Must pass filename and data in order to write to file!"
+      end
+      
+      filename = @options.output + "_" + filename
+      File.open(filename, "w") { |f| f.write data.to_yaml }
+    end
+    
+    # Called when the script is executed from the command line
+    def run
+      
+      load_gold_standard
+      load_query_result_set
+      begin_calculations
+      
+    end
+  end
+end

data/output_avg_precision.yml

@@ -0,0 +1,2 @@
+--- 
+Example Query: 0.5333333333333333

data/output_statistics.yml

@@ -0,0 +1,82 @@
+--- 
+Example Query: 
+- :precision: 1.0
+  :recall: 0.25
+  :false_negatives: 3
+  :false_positives: 0
+  :true_negatives: 6
+  :true_positives: 1
+  :document: ict21307
+  :relevant: true
+- :precision: 0.5
+  :recall: 0.25
+  :false_negatives: 3
+  :false_positives: 1
+  :true_negatives: 5
+  :true_positives: 1
+  :document: ict21309
+  :relevant: false
+- :precision: 0.3333333333333333
+  :recall: 0.25
+  :false_negatives: 3
+  :false_positives: 2
+  :true_negatives: 4
+  :true_positives: 1
+  :document: ict21311
+  :relevant: false
+- :precision: 0.25
+  :recall: 0.25
+  :false_negatives: 3
+  :false_positives: 3
+  :true_negatives: 3
+  :true_positives: 1
+  :document: ict21313
+  :relevant: false
+- :precision: 0.4
+  :recall: 0.5
+  :false_negatives: 2
+  :false_positives: 3
+  :true_negatives: 3
+  :true_positives: 2
+  :document: ict21315
+  :relevant: true
+- :precision: 0.3333333333333333
+  :recall: 0.5
+  :false_negatives: 2
+  :false_positives: 4
+  :true_negatives: 2
+  :true_positives: 2
+  :document: ict21317
+  :relevant: false
+- :precision: 0.2857142857142857
+  :recall: 0.5
+  :false_negatives: 2
+  :false_positives: 5
+  :true_negatives: 1
+  :true_positives: 2
+  :document: ict21319
+  :relevant: false
+- :precision: 0.25
+  :recall: 0.5
+  :false_negatives: 2
+  :false_positives: 6
+  :true_negatives: 0
+  :true_positives: 2
+  :document: ict21321
+  :relevant: false
+- :precision: 0.3333333333333333
+  :recall: 0.75
+  :false_negatives: 1
+  :false_positives: 6
+  :true_negatives: 0
+  :true_positives: 3
+  :document: ict21323
+  :relevant: true
+- :precision: 0.4
+  :recall: 1.0
+  :false_negatives: 0
+  :false_positives: 6
+  :true_negatives: 0
+  :true_positives: 4
+  :document: ict21325
+  :relevant: true

data/retreval.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s| 
+  s.name	= "retreval" 
+  s.summary	= "A Ruby API for Evaluating Retrieval Results" 
+  s.description = File.read(File.join(File.dirname(__FILE__), 'README.md')) 
+  # s.requirements = [ 'Nothing special' ]
+  s.version = "0.1"
+  s.author = "Werner Robitza"
+  s.email = "werner.robitza@univie.ac.at"
+  s.homepage = "http://github.com/slhck/retreval"
+  s.platform = Gem::Platform::RUBY
+  s.required_ruby_version = '>=1.9' 
+  s.files	= Dir['**/**'] 
+  s.executables = [ 'retreval' ] 
+  s.test_files = Dir["test/test*.rb"] 
+  s.has_rdoc	= true
+end

data/test/test_gold_standard.rb

@@ -0,0 +1,111 @@
+require 'test/unit'
+require 'retreval/gold_standard'
+
+# Some basic unit tests for the GoldStandard
+class TestGoldStandard < Test::Unit::TestCase 
+  
+  
+  # Adds one test judgement to this test case
+  def add_test_judgement
+    @gold_standard = Retreval::GoldStandard.new
+    @gold_standard.add_judgement :document => "doc1", :query => "query1", :relevant => true, :user => "John Doe"
+  end
+  
+  
+  # Tests whether the Document is correctly included
+  def test_document
+    add_test_judgement
+    assert(@gold_standard.contains_document? :id => "doc1")
+  end
+  
+  
+  # Tests whether the Query is correctly included
+  def test_query
+    add_test_judgement
+    assert(@gold_standard.contains_query? :querystring => "query1")
+  end
+  
+  
+  # Tests whether the User is correctly included
+  def test_user
+    add_test_judgement
+    assert(@gold_standard.contains_user? :id => "John Doe")
+  end
+  
+  
+  # Tests whether the Judgement is correctly included
+  def test_judgement
+    add_test_judgement
+    assert(@gold_standard.contains_judgement? :document => "doc1", :query => "query1")
+  end
+  
+  
+  # Tests whether the Judgement (i.e. the relevancy) is correctly added
+  def test_relevant
+    add_test_judgement
+    assert(@gold_standard.relevant? :document => "doc1", :query => "query1")
+  end
+  
+  # Tests if the kappa measure is calculated correctly.
+  # See http://nlp.stanford.edu/IR-book/html/htmledition/assessing-relevance-1.html
+  # for the examples in this test
+  def test_kappa_ir_book
+    
+    @gold_standard = Retreval::GoldStandard.new
+    
+    for i in (1..300) do 
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => true
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => true
+    end
+    
+    for i in (301..320) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => true
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => false
+    end
+    
+    for i in (321..330) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => false
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => true
+    end
+    
+    for i in (331..400) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => false
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => false
+    end
+    
+    assert_equal(0.7759103641456584, @gold_standard.kappa, "IR Book kappa test failed!")
+  end
+  
+  
+  # Tests if the kappa measure is calculated correctly.
+  # See http://nlp.stanford.edu/IR-book/html/htmledition/assessing-relevance-1.html
+  # for the examples in this test
+  def test_kappa_wikipedia
+    
+    @gold_standard = Retreval::GoldStandard.new
+    
+    for i in (1..20) do 
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => true
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => true
+    end
+    
+    for i in (21..25) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => true
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => false
+    end
+    
+    for i in (26..35) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => false
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => true
+    end
+    
+    for i in (36..50) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Alice", :relevant => false
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query#{i}", :user => "Bob", :relevant => false
+    end
+    puts "#{@gold_standard.kappa}"
+    assert_equal(0.3939393939393937, @gold_standard.kappa, "Wikipedia kappa test failed!")
+    
+  end
+  
+end

data/test/test_query_result.rb

@@ -0,0 +1,166 @@
+require 'test/unit'
+require 'retreval/gold_standard'
+require 'retreval/query_result'
+
+# Some basic unit tests for QueryResult
+# Unranked results include 4 documents of 10, which are all retrieved.
+# The ranked results are evaluated from this table:
+#      Index Relevant  Precision   Recall        Document ID
+#      1     [X]       1.000        0.250        doc1
+#      2     [X]       1.000        0.500        doc2
+#      3     [ ]       0.667        0.500        doc5
+#      4     [X]       0.750        0.750        doc3
+#      5     [ ]       0.600        0.750        doc6
+#      6     [X]       0.667        1.000        doc4
+#      7     [ ]       0.571        1.000        doc7
+#      8     [ ]       0.500        1.000        doc8
+#      9     [ ]       0.444        1.000        doc9
+#      10    [ ]       0.400        1.000        doc10
+class TestQueryResult < Test::Unit::TestCase 
+  
+  
+  # Adds 10 test judgements to this test case
+  def add_test_judgements
+    
+    @gold_standard = Retreval::GoldStandard.new
+    
+    for i in (1..4) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query1", :relevant => true
+    end
+    
+    for i in (5..10) do
+      @gold_standard.add_judgement :document => "doc#{i}", :query => "query1", :relevant => false
+    end
+    
+  end
+  
+  
+  # Adds the unranked query results to be tested against to this test case
+  def add_unranked_query_result
+    
+    @query_result = Retreval::UnrankedQueryResult.new :query => "query1", :gold_standard => @gold_standard
+    
+    for i in (1..4) do
+      @query_result.add_document :id => "doc#{i}"
+    end
+    
+    for i in (5..10) do
+      @query_result.add_document :id => "doc#{i}"
+    end
+    
+  end
+  
+  
+  # Adds the ranked query results to be tested against to this test case
+  def add_ranked_query_result
+    
+    @query_result = Retreval::RankedQueryResult.new :query => "query1", :gold_standard => @gold_standard
+    
+    @query_result.add_document :id => "doc1"
+    @query_result.add_document :id => "doc2"
+    @query_result.add_document :id => "doc5"
+    @query_result.add_document :id => "doc3"
+    @query_result.add_document :id => "doc6"
+    @query_result.add_document :id => "doc4"
+    @query_result.add_document :id => "doc7"
+    @query_result.add_document :id => "doc8"
+    @query_result.add_document :id => "doc9"
+    @query_result.add_document :id => "doc10"
+    
+  end
+  
+  
+  # Tests the unranked precision
+  def test_unranked_precision
+    
+    add_test_judgements
+    add_unranked_query_result
+    assert_equal(0.4, @query_result.statistics[:precision])
+    
+  end
+  
+  
+  # Tests if the unranked recall is calculated correctly
+  def test_unranked_recall
+
+    add_test_judgements    
+    add_unranked_query_result
+    assert_equal(1.0, @query_result.statistics[:recall])
+    
+  end
+  
+  
+  # Tests if the ranked recalls are calculated correctly
+  def test_ranked_precision
+    
+    add_test_judgements
+    add_ranked_query_result
+    expected_precision = [
+        1,
+        1,
+        0.6666666666666666,
+        0.75,
+        0.6,
+        0.6666666666666666,
+        0.5714285714285714,
+        0.5,
+        0.4444444444444444,
+        0.4
+      ]
+    @query_result.statistics.each_with_index do |rank, index|
+      assert_equal(expected_precision[index], rank[:precision])
+    end
+    
+  end
+  
+  # Tests if the ranked recalls are calculated correctly
+  def test_ranked_recall
+    
+    add_test_judgements
+    add_ranked_query_result
+    expected_recall = [
+        0.25,
+        0.5,
+        0.5,
+        0.75,
+        0.75,
+        1,
+        1,
+        1,
+        1,
+        1
+      ]
+    @query_result.statistics.each_with_index do |rank, index|
+      assert_equal(expected_recall[index], rank[:recall])
+    end
+    
+  end
+  
+  
+  # Tests the correct calculation of the eleven point precision as outlined here:
+  # http://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-ranked-retrieval-results-1.html
+  def test_eleven_point_precision
+    
+    add_test_judgements
+    add_ranked_query_result
+    expected_results = [
+        1.0,
+        1.0,
+        1.0,
+        0.6666666666666666,
+        0.6666666666666666,
+        0.6666666666666666,
+        0.6,
+        0.6,
+        0.4,
+        0.4,
+        0.4,
+      ]
+    @query_result.eleven_point_precision.each_with_index do |p, index|
+      assert_equal(expected_results[index], p[1])
+    end
+    
+  end
+  
+  
+end