sapor 0.3.0

data/lib/sapor/polychotomy.rb

@@ -0,0 +1,200 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Superclass for polychotomies.
+  #
+  class Polychotomy
+    attr_reader :error_estimate, :no_of_data_points, :no_of_simulations
+
+    def initialize(results, area, dichotomies, max_error)
+      initialize_static_information(results, area, dichotomies, max_error)
+      initialize_calculation_information
+    end
+
+    def most_probable_fraction(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        calculate_most_probable_fraction(key, @votes)
+      end
+    end
+
+    def most_probable_value(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        @votes[key].most_probable_value
+      end
+    end
+
+    def range(choice)
+      @ranges[choice]
+    end
+
+    def space_size
+      @enum.size
+    end
+
+    def encode_with(coder)
+      (instance_variables - [:@area, :@enum, :@logger]).each do |var|
+        var = var.to_s
+        coder[var.gsub('@', '')] = eval(var)
+      end
+    end
+    
+    def revive_volatile_fields(logger = nil)
+      @area = Poll.lookup_area(@area_code)
+      @enum = create_enumerator(@ranges)
+      @logger = logger
+      1.upto(@no_of_data_points).each { |i| @enum.next() }
+    end
+    
+    private
+
+    def calculate_most_probable_fraction(key, votes)
+      votes[key].most_probable_value.to_f / @area.population_size
+    end
+
+    def calculate_most_probable_rounded_fraction(key, votes)
+      votes[key].most_probable_rounded_fraction(@area.population_size)
+    end
+
+    def create_comparisons(choices)
+      comparisons = {}
+      choices.each do |a|
+        choices.each do |b|
+          comparisons[a + '>' + b] = 0.to_lf
+        end
+      end
+      comparisons
+    end
+
+    def create_enumerator(ranges)
+      range_sizes = ranges.values.map(&:size)
+      PseudoRandomMultiRangeEnumerator.new(range_sizes).each
+    end
+
+    def create_rankings(choices)
+      rankings = {}
+      choices.each do |choice|
+        1.upto(choices.size).each do |i|
+          rankings[choice + '@' + (i - 1).to_s] = 0.to_lf
+        end
+      end
+      rankings
+    end
+
+    def extract_ranges_from_dichotomies(dichotomies, max_error)
+      ranges = {}
+      level = 1 - (max_error**2)
+      @choices.each do |choice|
+        unless choice == OTHER
+          ranges[choice] = dichotomies.confidence_interval_values(choice,
+                                                                  level).sort
+        end
+      end
+      ranges
+    end
+
+    def initialize_calculation_information
+      @enum = create_enumerator(@ranges)
+      @no_of_simulations = 0
+      @no_of_data_points = 0
+      @combinations_sum = 0.to_lf
+      @error_estimate = 1.0
+      @votes = create_new_votes_distributions
+    end
+
+    def initialize_static_information(results, area, dichotomies, max_error)
+      @results = results
+      @area = area
+      @area_code = area.area_code
+      @choices = results.keys
+      @comparisons = create_comparisons(@choices)
+      @rankings = create_rankings(@choices)
+      @ranges = extract_ranges_from_dichotomies(dichotomies, max_error)
+    end
+
+    def larger_than(a, b)
+      probability = @comparisons[a + '>' + b] / @combinations_sum
+      probability.mantissa * (10**probability.exponent)
+    end
+
+    def most_probable_rounded_fraction(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        calculate_most_probable_rounded_fraction(key, @votes)
+      end
+    end
+
+    def next_data_point
+      indexes = @enum.next
+      data_point = {}
+      indexes.each_with_index do |ix, i|
+        data_point[@ranges.keys[i]] = @ranges.values[i][ix]
+      end
+      data_point[OTHER] = @area.population_size - data_point.values.inject(:+)
+      data_point
+    end
+
+    def ranking(choice, i)
+      probability = @rankings[choice + '@' + i.to_s] / @combinations_sum
+      probability.mantissa * (10**probability.exponent)
+    end
+
+    def result(choice)
+      @results[choice].to_f / @results.values.inject(:+)
+    end
+
+    def sort_choices_by_result
+      sorted_choices = @choices.reject { |choice| choice == OTHER }
+      sorted_choices.sort do |a, b|
+        comparison = result(b) <=> result(a)
+        if comparison == 0
+          a <=> b
+        else
+          comparison
+        end
+      end
+    end
+    
+    def write_rankings(filename)
+      ci_filename = filename.gsub('.poll', '-polychotomy-rankings.psv')
+      File.open(ci_filename, 'w') do |file| 
+        file.puts('Choice | ' + 0.upto(@choices.size - 1).map {|i| i + 1}.join(' | ')) 
+        @choices.each do | choice |
+          file.puts(choice + ' | ' + 0.upto(@choices.size - 1).map {|i| ranking(choice, i)}.join(' | '))
+        end
+      end
+    end
+    
+    def write_state_summary(filename)
+      state_summary_filename = filename.gsub('.poll', '_state_summary.txt')
+      File.open(state_summary_filename, 'w') do |file|
+        file.puts("ErrorEstimate=#{@error_estimate}")
+        file.puts("NumberOfSimulations=#{@no_of_simulations}")
+        file.puts("NumberOfDataPoints=#{@no_of_data_points}")
+        file.puts("SpaceSize=#{space_size}")
+      end
+    end    
+  end
+end

data/lib/sapor/pseudorandom_multirange_enumerator.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+require 'prime'
+
+module Sapor
+  #
+  # A pseudo-random multi-range enumerator. It uses prime numbers to iterate
+  # in a pseudo-random way through the hyper-rectangle with dimensions
+  # provided as the constructor parameter. If a suitable set of primes can't
+  # be found, the constructor will raise an exception.
+  #
+  class PseudoRandomMultiRangeEnumerator
+    def initialize(dimensions)
+      @dimensions = dimensions
+      @incrementers = calculate_incrementers
+    end
+
+    def each
+      return enum_for { @dimensions.inject(:*) } unless block_given?
+      counters = Array.new(@dimensions.size, 0)
+      loop do
+        yield(counters.dup)
+        @incrementers.each_with_index do |incrementer, i|
+          counters[i] = (counters[i] + incrementer).modulo(@dimensions[i])
+        end
+        break if all_zeros?(counters)
+      end
+    end
+
+    private
+
+    def calculate_incrementers
+      incrementers = []
+      incrementer = 0
+      calculate_incrementer_indexes.each do |i|
+        incrementer = next_incrementer(incrementer)
+        if @dimensions[i] < incrementer
+          fail(ArgumentError, 'Could not construct suitable incrementers.')
+        end
+        incrementers[i] = incrementer
+      end
+      incrementers
+    end
+
+    def calculate_incrementer_indexes
+      @dimensions.each_with_index.sort.map(&:last)
+    end
+
+    def next_incrementer(incrementer)
+      incrementer += 1
+      while incrementer > 1 &&
+            (!prime?(incrementer) || !relative_prime?(incrementer))
+        incrementer += 1
+      end
+      incrementer
+    end
+
+    def prime?(number)
+      Prime.prime?(number)
+    end
+
+    def relative_prime?(incrementer)
+      @dimensions.map { |dimension| dimension.gcd(incrementer) }.max == 1
+    end
+
+    def all_zeros?(counters)
+      counters.max == 0
+    end
+  end
+end

data/lib/sapor/referendum_polychotomy.rb

@@ -0,0 +1,165 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Represents a polychotomy for a referendum.
+  #
+  class ReferendumPolychotomy < Polychotomy
+    include NumberFormatter
+
+    def initialize(results, area, dichotomies, max_error)
+      super(results, area, dichotomies, max_error)
+    end
+
+    def refine
+      no_of_new_simulations = 0
+      new_votes = create_new_votes_distributions
+      while @no_of_data_points == 0 || no_of_new_simulations == 0 ||
+            no_of_new_simulations < @no_of_simulations
+        no_of_new_simulations += try_next_data_point(new_votes)
+      end
+      unless @no_of_simulations == 0
+        @error_estimate = calculate_error_estimate(new_votes)
+      end
+      @votes = merge_distributions(@votes, new_votes)
+      @no_of_simulations += no_of_new_simulations
+    end
+
+    def try_next_data_point(new_votes)
+      data_point = next_data_point
+      if data_point[OTHER] >= 0
+        simulate(new_votes, data_point)
+        new_simulation = 1
+      else
+        new_simulation = 0
+      end
+      @no_of_data_points += 1
+      new_simulation
+    end
+
+    def report
+      choice_lengths = @choices.map(&:length)
+      choice_lengths << 6
+      max_choice_width = choice_lengths.max
+      sorted_choices = sort_choices_by_result
+      choice_lines = sorted_choices.map.with_index do |choice, i|
+        next_choice = sorted_choices[i + 1]
+        create_choice_report_line(choice, i, next_choice, max_choice_width)
+      end
+      'Most probable rounded fractions, fractions and 95% confidence' \
+      " intervals:\n" + 'Choice'.ljust(max_choice_width) +
+        '  Result    MPRF    MPF      CI(95%)      P(>↓)   P(R↑)    P(R)' \
+        "   P(R↓)\n" + choice_lines.join("\n")
+    end
+
+    def progress_report
+      space_size_ratio = space_size.to_f / @no_of_data_points
+      if space_size_ratio > 10
+        space_size_ratio = with_thousands_separator(space_size_ratio.round)
+      else
+        space_size_ratio = space_size_ratio.round(1)
+      end
+      "#{with_thousands_separator(@no_of_simulations)} simulations out of " \
+      "#{with_thousands_separator(@no_of_data_points)} data" \
+      " points, 1 / #{space_size_ratio} of search space size" \
+      " (#{with_thousands_separator(space_size)})."
+    end
+
+    def write_outputs(filename)
+      write_rankings(filename)
+    end
+    
+    private
+
+    def create_new_votes_distributions
+      votes = {}
+      @choices.each do |choice|
+        next if choice == OTHER
+        votes[choice] = CombinationsDistribution.new
+        @ranges[choice].each do |value|
+          votes[choice][value] = 0.to_lf
+        end
+      end
+      votes
+    end
+
+    def simulate(votes, data_point)
+      combinations = 1.to_lf
+      data_point.each do |choice, value|
+        combinations *= BinomialsCache.binomial(value, @results[choice])
+      end
+      @combinations_sum += combinations
+      data_point.each do |choice, value|
+        votes[choice][value] += combinations unless choice == OTHER
+      end
+      @choices.each do |a|
+        @choices.each do |b|
+          if data_point[a] > data_point[b]
+            @comparisons[a + '>' + b] += combinations
+          end
+        end
+      end
+      @choices.sort { |a, b| data_point[b] <=> data_point[a] }.each_with_index do |choice, i|
+        @rankings[choice + '@' + i.to_s] += combinations
+      end
+    end
+
+    def calculate_error_estimate(new_simulations)
+      error_estimate = 0
+      @choices.each do |choice|
+        next if choice == OTHER
+        mpv_new = calculate_most_probable_fraction(choice, new_simulations)
+        mpv_old = calculate_most_probable_fraction(choice, @votes)
+        mpv_delta = (mpv_new - mpv_old).abs
+        mprf_new = calculate_most_probable_rounded_fraction(choice, new_simulations)
+        mprf_old = calculate_most_probable_rounded_fraction(choice, @votes)
+        mprf_delta = (mprf_new - mprf_old).abs
+        error_estimate = [error_estimate, mpv_delta, mprf_delta].max
+      end
+      error_estimate
+    end
+
+    def merge_distributions(distributions1, distributions2)
+      merged_distributions = {}
+      @choices.each do |choice|
+        unless choice == OTHER
+          merged_distributions[choice] = distributions1[choice] + \
+                                         distributions2[choice]
+        end
+      end
+      merged_distributions
+    end
+
+    def create_choice_report_line(choice, i, next_choice, max_choice_width)
+      ci_values = @votes[choice].confidence_interval(0.95)
+      confidence_interval = ci_values.map { |x| x.to_f / @area.population_size }
+      choice.ljust(max_choice_width) + '  ' + \
+        six_char_percentage(result(choice)) + '  ' + \
+        six_char_percentage(most_probable_rounded_fraction(choice)) + '  ' + \
+        six_char_percentage(most_probable_fraction(choice)) + '  ' + \
+        six_char_percentage(confidence_interval.first) + '–' + \
+        six_char_percentage(confidence_interval.last) + '  ' + \
+        (next_choice.nil? ? '      ' : six_char_percentage(larger_than(choice, next_choice))) + '  ' + \
+        (i == 0 ? '      ' : six_char_percentage(ranking(choice, i - 1))) + '  ' + \
+        six_char_percentage(ranking(choice, i)) + \
+        (next_choice.nil? ? '' : '  ' + six_char_percentage(ranking(choice, i + 1)))
+    end
+  end
+end