sapor 0.1b1 → 0.2a1

data/lib/sapor/polychotomy.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 #
 # Statistical Analysis of Polling Results (SAPoR)
-# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
 #
 # This file is part of SAPoR.
 #
@@ -17,37 +17,32 @@
 # You can find a copy of the GNU General Public License in /doc/gpl.txt
 #
 
-require 'prime'
-
 module Sapor
   #
-  # Represents a polychotomy.
+  # Superclass for polychotomies.
   #
   class Polychotomy
-    include NumberFormatter
-
     attr_reader :error_estimate, :no_of_data_points, :no_of_simulations
 
     def initialize(results, area, dichotomies, max_error)
-      @results = results
-      @area = area
-      @choices = results.keys
-      @coalitions = area.coalitions
-      @ranges = extract_ranges_from_dichotomies(dichotomies, max_error)
-      range_sizes = @ranges.values.map(&:size)
-      @enum = PseudoRandomMultiRangeEnumerator.new(range_sizes).each
-      @no_of_simulations = 0
-      @no_of_data_points = 0
-      @distributions = create_new_votes_distributions # TODO: Rename to @votes
-      @seats = create_new_seats_distributions
-      @comparisons = {}
-      @choices.each do |a|
-        @choices.each do |b|
-          @comparisons[a + '>' + b] = 0.to_lf
-        end
+      initialize_static_information(results, area, dichotomies, max_error)
+      initialize_calculation_information
+    end
+
+    def most_probable_fraction(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        calculate_most_probable_fraction(key, @votes)
+      end
+    end
+
+    def most_probable_value(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        @votes[key].most_probable_value
       end
-      @combinations_sum = 0.to_lf
-      @error_estimate = 1.0
     end
 
     def range(choice)
@@ -58,95 +53,55 @@ module Sapor
       @enum.size
     end
 
-    def most_probable_value(key)
-      if @no_of_simulations == 0
-        nil
-      else
-        @distributions[key].most_probable_value
+    def encode_with(coder)
+      (instance_variables - [:@area, :@enum, :@logger]).each do |var|
+        var = var.to_s
+        coder[var.gsub('@', '')] = eval(var)
       end
     end
-
-    def calculate_most_probable_fraction(key, distributions)
-      distributions[key].most_probable_value.to_f / @area.population_size
+    
+    def revive_volatile_fields(logger = nil)
+      @area = Poll.lookup_area(@area_code)
+      @enum = create_enumerator(@ranges)
+      @logger = logger
+      1.upto(@no_of_data_points).each { |i| @enum.next() }
     end
+    
+    private
 
-    def most_probable_fraction(key)
-      if @no_of_simulations == 0
-        nil
-      else
-        calculate_most_probable_fraction(key, @distributions)
-      end
+    def calculate_most_probable_fraction(key, votes)
+      votes[key].most_probable_value.to_f / @area.population_size
     end
 
-    def refine
-      no_of_new_simulations = 0
-      new_votes = create_new_votes_distributions
-      new_seats = create_new_seats_distributions
-      while @no_of_data_points == 0 || no_of_new_simulations == 0 ||
-            no_of_new_simulations < @no_of_simulations
-        no_of_new_simulations += try_next_data_point(new_votes, new_seats)
-      end
-      unless @no_of_simulations == 0
-        @error_estimate = calculate_error_estimate(new_votes)
-      end
-      @distributions = merge_distributions(@distributions, new_votes)
-      @seats = merge_distributions(@seats, new_seats)
-      @no_of_simulations += no_of_new_simulations
+    def calculate_most_probable_rounded_fraction(key, votes)
+      votes[key].most_probable_rounded_fraction(@area.population_size)
     end
 
-    def try_next_data_point(new_votes, new_seats)
-      data_point = next_data_point
-      if data_point[OTHER] >= 0
-        simulate(new_votes, new_seats, data_point)
-        new_simulation = 1
-      else
-        new_simulation = 0
+    def create_comparisons(choices)
+      comparisons = {}
+      choices.each do |a|
+        choices.each do |b|
+          comparisons[a + '>' + b] = 0.to_lf
+        end
       end
-      @no_of_data_points += 1
-      new_simulation
+      comparisons
     end
 
-    def report
-      choice_lengths = @choices.map(&:length)
-      choice_lengths << 6
-      max_choice_width = choice_lengths.max
-      max_seats_width = @area.no_of_seats.to_s.size
-      sorted_choices = sort_choices_by_result
-      choice_lines = sorted_choices.map.with_index do |choice, i|
-        next_choice = sorted_choices[i + 1]
-        create_choice_report_line(choice, next_choice, max_choice_width, max_seats_width)
-      end
-      coalition_lengths = @coalitions.map { |coalition| coalition_label(coalition).length }
-      coalition_lengths << 6
-      max_coalition_width = coalition_lengths.max
-      sorted_coalitions = sort_coalitions_by_result
-      coalition_lines = sorted_coalitions.map do |coalition|
-        create_coalition_report_line(coalition, max_coalition_width, max_seats_width)
-      end
-      'Most probable rounded fractions, fractions and 95% confidence' \
-      " intervals:\n" + 'Choice'.ljust(max_choice_width) +
-        "  Result    MPRF    MPF      CI(95%)      P(>↓)  Seats\n" +
-        choice_lines.join("\n") + "\n" +
-        'Coalition'.ljust(max_coalition_width) + "  Result    MPRF    MPF" +
-        "      CI(95%)     P(>50%)  Seats  P(>50%)\n" +
-        coalition_lines.join("\n")
+    def create_enumerator(ranges)
+      range_sizes = ranges.values.map(&:size)
+      PseudoRandomMultiRangeEnumerator.new(range_sizes).each
     end
 
-    def progress_report
-      space_size_ratio = space_size / @no_of_data_points
-      if space_size_ratio > 10
-        space_size_ratio = with_thousands_separator(space_size_ratio.round)
-      else
-        space_size_ratio = space_size_ratio.round(1)
+    def create_rankings(choices)
+      rankings = {}
+      choices.each do |choice|
+        1.upto(choices.size).each do |i|
+          rankings[choice + '@' + (i - 1).to_s] = 0.to_lf
+        end
       end
-      "#{with_thousands_separator(@no_of_simulations)} simulations out of " \
-      "#{with_thousands_separator(@no_of_data_points)} data" \
-      " points, 1 / #{space_size_ratio} of search space size" \
-      " (#{with_thousands_separator(space_size)})."
+      rankings
     end
 
-    private
-
     def extract_ranges_from_dichotomies(dichotomies, max_error)
       ranges = {}
       level = 1 - (max_error**2)
@@ -159,37 +114,36 @@ module Sapor
       ranges
     end
 
-    def create_new_seats_distributions
-      distributions = {}
-      @choices.each do |choice|
-        distributions[choice] = CombinationsDistribution.new
-        Range.new(0, @area.no_of_seats).each do |value|
-          distributions[choice][value] = 0.to_lf
-        end
-      end
-      @coalitions.each do |coalition|
-        distributions[coalition] = CombinationsDistribution.new
-        Range.new(0, @area.no_of_seats).each do |value|
-          distributions[coalition][value] = 0.to_lf
-        end
-      end
-      distributions
+    def initialize_calculation_information
+      @enum = create_enumerator(@ranges)
+      @no_of_simulations = 0
+      @no_of_data_points = 0
+      @combinations_sum = 0.to_lf
+      @error_estimate = 1.0
+      @votes = create_new_votes_distributions
     end
 
-    def create_new_votes_distributions
-      distributions = {}
-      @choices.each do |choice|
-        unless choice == OTHER
-          distributions[choice] = CombinationsDistribution.new
-          @ranges[choice].each do |value|
-            distributions[choice][value] = 0.to_lf
-          end
-        end
-      end
-      @coalitions.each do |coalition|
-        distributions[coalition] = CombinationsDistribution.new
+    def initialize_static_information(results, area, dichotomies, max_error)
+      @results = results
+      @area = area
+      @area_code = area.area_code
+      @choices = results.keys
+      @comparisons = create_comparisons(@choices)
+      @rankings = create_rankings(@choices)
+      @ranges = extract_ranges_from_dichotomies(dichotomies, max_error)
+    end
+
+    def larger_than(a, b)
+      probability = @comparisons[a + '>' + b] / @combinations_sum
+      probability.mantissa * (10**probability.exponent)
+    end
+
+    def most_probable_rounded_fraction(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        calculate_most_probable_rounded_fraction(key, @votes)
       end
-      distributions
     end
 
     def next_data_point
@@ -202,74 +156,13 @@ module Sapor
       data_point
     end
 
-    def simulate(votes, seats, data_point)
-      combinations = 1.to_lf
-      data_point.each do |choice, value|
-        combinations *= BinomialsCache.binomial(value, @results[choice])
-      end
-      @combinations_sum += combinations
-      data_point.each do |choice, value|
-        votes[choice][value] += combinations unless choice == OTHER
-      end
-      @choices.each do |a|
-        @choices.each do |b|
-          if data_point[a] > data_point[b]
-            @comparisons[a + '>' + b] += combinations
-          end
-        end
-      end
-      @coalitions.each do |coalition|
-        coalition_value = coalition.map { |choice| data_point.key?(choice) ? data_point[choice] : 0}.inject(:+)
-        if votes[coalition][coalition_value].nil?
-          votes[coalition][coalition_value] = combinations
-        else
-          votes[coalition][coalition_value] += combinations
-        end
-      end
-      projection = @area.seats(data_point)
-      @choices.each do |choice|
-        if projection.key?(choice)
-          seats[choice][projection[choice]] += combinations
-        else
-          seats[choice][0] += combinations
-        end
-      end
-      @coalitions.each do |coalition|
-        coalition_value = coalition.map { |choice| projection.key?(choice) ? projection[choice] : 0}.inject(:+)
-        if seats[coalition][coalition_value].nil?
-          seats[coalition][coalition_value] = combinations
-        else
-          seats[coalition][coalition_value] += combinations
-        end
-      end
-    end
-
-    def calculate_error_estimate(new_simulations)
-      error_estimate = 0
-      @choices.each do |choice|
-        unless choice == OTHER
-          mpv_new = calculate_most_probable_fraction(choice, new_simulations)
-          mpv_old = calculate_most_probable_fraction(choice, @distributions)
-          delta = (mpv_new - mpv_old).abs
-          error_estimate = [error_estimate, delta].max
-        end
-      end
-      error_estimate
+    def ranking(choice, i)
+      probability = @rankings[choice + '@' + i.to_s] / @combinations_sum
+      probability.mantissa * (10**probability.exponent)
     end
 
-    def merge_distributions(distributions1, distributions2)
-      merged_distributions = {}
-      @choices.each do |choice|
-        unless choice == OTHER
-          merged_distributions[choice] = distributions1[choice] + \
-                                         distributions2[choice]
-        end
-      end
-      @coalitions.each do |coalition|
-        merged_distributions[coalition] = distributions1[coalition] + \
-                                          distributions2[coalition]
-      end
-      merged_distributions
+    def result(choice)
+      @results[choice].to_f / @results.values.inject(:+)
     end
 
     def sort_choices_by_result
@@ -283,77 +176,25 @@ module Sapor
         end
       end
     end
-
-    def coalition_label(coalition)
-      coalition.sort.join(' + ')
-    end
-
-    def sort_coalitions_by_result
-      @coalitions.sort do |a, b|
-        comparison = result(b) <=> result(a)
-        if comparison == 0
-          coalition_label(a) <=> coalition_label(b)
-        else
-          comparison
+    
+    def write_rankings(filename)
+      ci_filename = filename.gsub('.poll', '-polychotomy-rankings.psv')
+      File.open(ci_filename, 'w') do |file| 
+        file.puts('Choice | ' + 0.upto(@choices.size - 1).map {|i| i + 1}.join(' | ')) 
+        @choices.each do | choice |
+          file.puts(choice + ' | ' + 0.upto(@choices.size - 1).map {|i| ranking(choice, i)}.join(' | '))
         end
       end
     end
-
-    def result(choice)
-      @results[choice].to_f / @results.values.inject(:+)
-    end
-
-    def coalition_result(coalition)
-      coalition.map { |choice| @results[choice].to_f }.inject(:+) / @results.values.inject(:+)
-    end
-
-    def larger_than(a, b)
-      probability = @comparisons[a + '>' + b] / @combinations_sum
-      probability.mantissa * (10**probability.exponent)
-    end
-
-    def create_choice_report_line(choice, next_choice, max_choice_width, max_seats_width)
-      ci_values = @distributions[choice].confidence_interval(0.95)
-      confidence_interval = ci_values.map { |x| x.to_f / @area.population_size }
-      ci_seats = @seats[choice].confidence_interval(0.95)
-      choice.ljust(max_choice_width) + '  ' + \
-        six_char_percentage(result(choice)) + '  ' + \
-        six_char_percentage(most_probable_rounded_fraction(choice)) + '  ' + \
-        six_char_percentage(most_probable_fraction(choice)) + '  ' + \
-        six_char_percentage(confidence_interval.first) + '–' + \
-        six_char_percentage(confidence_interval.last) + '  ' + \
-        (next_choice.nil? ? '      ' : six_char_percentage(larger_than(choice, next_choice))) + '  ' + \
-        (max_seats_width == 1 ? ' ' : '') + \
-        ci_seats.first.to_s.rjust(max_seats_width) + '–' + \
-        ci_seats.last.to_s.rjust(max_seats_width)
-    end
-
-    def create_coalition_report_line(coalition, max_coalition_width, max_seats_width)
-      ci_values = @distributions[coalition].confidence_interval(0.95)
-      majority_votes_probability = @distributions[coalition].threshold_probability(0.5, @area.population_size)
-      confidence_interval = ci_values.map { |x| x.to_f / @area.population_size }
-      ci_seats = @seats[coalition].confidence_interval(0.95)
-      seats_majority = 1 + @area.no_of_seats / 2
-      majority_seats_probability = @seats[coalition].value_threshold_probability(seats_majority)
-      coalition_label(coalition).ljust(max_coalition_width) + '  ' + \
-        six_char_percentage(coalition_result(coalition)) + '  ' + \
-        six_char_percentage(most_probable_rounded_fraction(coalition)) + '  ' + \
-        six_char_percentage(most_probable_fraction(coalition)) + '  ' + \
-        six_char_percentage(confidence_interval.first) + '–' + \
-        six_char_percentage(confidence_interval.last) + '   ' + \
-        six_char_percentage(majority_votes_probability) + '  ' + \
-        (max_seats_width == 1 ? ' ' : '') + \
-        ci_seats.first.to_s.rjust(max_seats_width) + '–' + \
-        ci_seats.last.to_s.rjust(max_seats_width) + '    ' + \
-        six_char_percentage(majority_seats_probability)
-    end
-
-    def most_probable_rounded_fraction(key)
-      if @no_of_simulations == 0
-        nil
-      else
-        @distributions[key].most_probable_rounded_fraction(@area.population_size)
+    
+    def write_state_summary(filename)
+      state_summary_filename = filename.gsub('.poll', '_state_summary.txt')
+      File.open(state_summary_filename, 'w') do |file|
+        file.puts("ErrorEstimate=#{@error_estimate}")
+        file.puts("NumberOfSimulations=#{@no_of_simulations}")
+        file.puts("NumberOfDataPoints=#{@no_of_data_points}")
+        file.puts("SpaceSize=#{space_size}")
       end
-    end
+    end    
   end
 end

data/lib/sapor/pseudorandom_multirange_enumerator.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 #
 # Statistical Analysis of Polling Results (SAPoR)
-# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
 #
 # This file is part of SAPoR.
 #

data/lib/sapor/referendum_polychotomy.rb

@@ -0,0 +1,165 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Represents a polychotomy for a referendum.
+  #
+  class ReferendumPolychotomy < Polychotomy
+    include NumberFormatter
+
+    def initialize(results, area, dichotomies, max_error)
+      super(results, area, dichotomies, max_error)
+    end
+
+    def refine
+      no_of_new_simulations = 0
+      new_votes = create_new_votes_distributions
+      while @no_of_data_points == 0 || no_of_new_simulations == 0 ||
+            no_of_new_simulations < @no_of_simulations
+        no_of_new_simulations += try_next_data_point(new_votes)
+      end
+      unless @no_of_simulations == 0
+        @error_estimate = calculate_error_estimate(new_votes)
+      end
+      @votes = merge_distributions(@votes, new_votes)
+      @no_of_simulations += no_of_new_simulations
+    end
+
+    def try_next_data_point(new_votes)
+      data_point = next_data_point
+      if data_point[OTHER] >= 0
+        simulate(new_votes, data_point)
+        new_simulation = 1
+      else
+        new_simulation = 0
+      end
+      @no_of_data_points += 1
+      new_simulation
+    end
+
+    def report
+      choice_lengths = @choices.map(&:length)
+      choice_lengths << 6
+      max_choice_width = choice_lengths.max
+      sorted_choices = sort_choices_by_result
+      choice_lines = sorted_choices.map.with_index do |choice, i|
+        next_choice = sorted_choices[i + 1]
+        create_choice_report_line(choice, i, next_choice, max_choice_width)
+      end
+      'Most probable rounded fractions, fractions and 95% confidence' \
+      " intervals:\n" + 'Choice'.ljust(max_choice_width) +
+        '  Result    MPRF    MPF      CI(95%)      P(>↓)   P(R↑)    P(R)' \
+        "   P(R↓)\n" + choice_lines.join("\n")
+    end
+
+    def progress_report
+      space_size_ratio = space_size.to_f / @no_of_data_points
+      if space_size_ratio > 10
+        space_size_ratio = with_thousands_separator(space_size_ratio.round)
+      else
+        space_size_ratio = space_size_ratio.round(1)
+      end
+      "#{with_thousands_separator(@no_of_simulations)} simulations out of " \
+      "#{with_thousands_separator(@no_of_data_points)} data" \
+      " points, 1 / #{space_size_ratio} of search space size" \
+      " (#{with_thousands_separator(space_size)})."
+    end
+
+    def write_outputs(filename)
+      write_rankings(filename)
+    end
+    
+    private
+
+    def create_new_votes_distributions
+      votes = {}
+      @choices.each do |choice|
+        next if choice == OTHER
+        votes[choice] = CombinationsDistribution.new
+        @ranges[choice].each do |value|
+          votes[choice][value] = 0.to_lf
+        end
+      end
+      votes
+    end
+
+    def simulate(votes, data_point)
+      combinations = 1.to_lf
+      data_point.each do |choice, value|
+        combinations *= BinomialsCache.binomial(value, @results[choice])
+      end
+      @combinations_sum += combinations
+      data_point.each do |choice, value|
+        votes[choice][value] += combinations unless choice == OTHER
+      end
+      @choices.each do |a|
+        @choices.each do |b|
+          if data_point[a] > data_point[b]
+            @comparisons[a + '>' + b] += combinations
+          end
+        end
+      end
+      @choices.sort { |a, b| data_point[b] <=> data_point[a] }.each_with_index do |choice, i|
+        @rankings[choice + '@' + i.to_s] += combinations
+      end
+    end
+
+    def calculate_error_estimate(new_simulations)
+      error_estimate = 0
+      @choices.each do |choice|
+        next if choice == OTHER
+        mpv_new = calculate_most_probable_fraction(choice, new_simulations)
+        mpv_old = calculate_most_probable_fraction(choice, @votes)
+        mpv_delta = (mpv_new - mpv_old).abs
+        mprf_new = calculate_most_probable_rounded_fraction(choice, new_simulations)
+        mprf_old = calculate_most_probable_rounded_fraction(choice, @votes)
+        mprf_delta = (mprf_new - mprf_old).abs
+        error_estimate = [error_estimate, mpv_delta, mprf_delta].max
+      end
+      error_estimate
+    end
+
+    def merge_distributions(distributions1, distributions2)
+      merged_distributions = {}
+      @choices.each do |choice|
+        unless choice == OTHER
+          merged_distributions[choice] = distributions1[choice] + \
+                                         distributions2[choice]
+        end
+      end
+      merged_distributions
+    end
+
+    def create_choice_report_line(choice, i, next_choice, max_choice_width)
+      ci_values = @votes[choice].confidence_interval(0.95)
+      confidence_interval = ci_values.map { |x| x.to_f / @area.population_size }
+      choice.ljust(max_choice_width) + '  ' + \
+        six_char_percentage(result(choice)) + '  ' + \
+        six_char_percentage(most_probable_rounded_fraction(choice)) + '  ' + \
+        six_char_percentage(most_probable_fraction(choice)) + '  ' + \
+        six_char_percentage(confidence_interval.first) + '–' + \
+        six_char_percentage(confidence_interval.last) + '  ' + \
+        (next_choice.nil? ? '      ' : six_char_percentage(larger_than(choice, next_choice))) + '  ' + \
+        (i == 0 ? '      ' : six_char_percentage(ranking(choice, i - 1))) + '  ' + \
+        six_char_percentage(ranking(choice, i)) + \
+        (next_choice.nil? ? '' : '  ' + six_char_percentage(ranking(choice, i + 1)))
+    end
+  end
+end

data/lib/sapor/regional_data/area.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 #
 # Statistical Analysis of Polling Results (SAPoR)
-# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
 #
 # This file is part of SAPoR.
 #
@@ -23,7 +23,9 @@ module Sapor
   # polls can be registered.
   #
   class Area
-    include Singleton
+    def threshold
+      nil
+    end
 
     def lines_to_election_results(lines)
       results = {}