sapor 0.3.3

data/lib/sapor/binomials_cache.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+require 'large_binomials'
+
+module Sapor
+  #
+  # Caches binomials.
+  #
+  class BinomialsCache
+    include Singleton
+
+    def initialize
+      @cache = {}
+    end
+
+    def self.binomial(k, n)
+      instance.get_binomial(k, n)
+    end
+
+    def get_binomial(k, n)
+      @cache[n] = {} unless @cache.key?(n)
+      unless @cache[n].key?(k)
+        @cache[n][k] = k.large_float_binomial_by_product_of_divisions(n)
+      end
+      @cache[n][k]
+    end
+  end
+end

data/lib/sapor/combinations_distribution.rb

@@ -0,0 +1,222 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Represents the distribution of combinations over a set of values. This is
+  # basically a simple hash with some helper methods for the Sapor domain.
+  #
+  class CombinationsDistribution
+    def initialize
+      @distribution = {}
+    end
+
+    def []=(value, combinations)
+      @distribution[value] = combinations
+    end
+
+    def [](value)
+      @distribution[value]
+    end
+
+    def +(other)
+      sum = CombinationsDistribution.new
+      @distribution.keys.each { |key| sum[key] = self[key] }
+      other.values.each do |key|
+        if self[key].nil?
+          sum[key] = other[key]
+        else
+          sum[key] += other[key]
+        end
+      end
+      sum
+    end
+
+    def empty?
+      @distribution.empty?
+    end
+
+    def interval_probabilities(intervals, population_size)
+      value_interval_probabilities(intervals.map {|interval| [interval.first * population_size, interval.last * population_size]})
+    end
+    
+    def value_interval_probabilities(value_intervals)
+      total = @distribution.values.inject(:+)
+      value_intervals.map do | value_interval |
+        distribution_in_interval = @distribution.select do |k, _|
+          k >= value_interval.first && k < value_interval.last
+        end
+        if distribution_in_interval.empty?
+          0
+        else
+          in_interval = distribution_in_interval.values.inject(:+)
+          probability = in_interval / total
+          probability.mantissa * (10**probability.exponent)
+        end
+      end
+    end
+
+    def value_threshold_probability(threshold_value)
+      total = @distribution.values.inject(:+)
+      distribution_over_threshold = @distribution.select do |k, _|
+        k >= threshold_value
+      end
+      if distribution_over_threshold.empty?
+        0
+      else
+        over_threshold = distribution_over_threshold.values.inject(:+)
+        probability = over_threshold / total
+        probability.mantissa * (10**probability.exponent)
+      end
+    end
+
+    def probabilities(values)
+      total = @distribution.values.inject(:+)
+      values.map do | value |
+        if @distribution.key?(value)
+          probability = @distribution[value] / total
+          probability.mantissa * (10**probability.exponent)
+        else
+          0
+        end
+      end
+    end
+
+    def probability(value)
+      if @distribution.key?(value)
+        total = @distribution.values.inject(:+)
+        probability = @distribution[value] / total
+        probability.mantissa * (10**probability.exponent)
+      else
+        0
+      end
+    end
+
+    def threshold_probability(threshold, population_size)
+      value_threshold_probability(population_size * threshold)
+    end
+
+    def size
+      @distribution.size
+    end
+
+    def values
+      @distribution.keys
+    end
+
+    def most_probable_value
+      @distribution.max { |a, b| a.last <=> b.last }[0]
+    end
+
+    # Given all fractions rounded to one decimal, returns the one that has the
+    # highest probability.
+    def most_probable_rounded_fraction(population_size)
+      rf_combinations = \
+      calculate_rounded_fractions_combinations(population_size)
+      max_probability = rf_combinations.values.max
+      opt_rfs = rf_combinations.reject { |_, v| v < max_probability }.keys
+      opt_rfs.sort[opt_rfs.size / 2]
+    end
+
+    def confidence_interval(level, population_size = nil)
+      combinations_sum = @distribution.values.inject(:+)
+      one_side_level = (1 - level) / 2
+      one_side_threshold = combinations_sum * one_side_level
+      bottom = find_confidence_interval_bottom(one_side_threshold,
+                                               population_size)
+      top = find_confidence_interval_top(one_side_threshold, population_size)
+      [bottom, top]
+    end
+
+    def confidence_interval_values(level)
+      interval = confidence_interval(level)
+      @distribution.keys.reject do |value|
+        value < interval.first || value > interval.last
+      end
+    end
+
+    private
+
+    def confidence_interval_index(sorted_combinations, one_side_threshold)
+      i = 0
+      sum_to_i = sorted_combinations[i][1]
+      while sum_to_i < one_side_threshold
+        i += 1
+        sum_to_i += sorted_combinations[i][1]
+      end
+      i
+    end
+
+    def find_confidence_interval_bottom(one_side_threshold, population_size)
+      sorted_combinations = @distribution.sort
+      i = confidence_interval_index(sorted_combinations, one_side_threshold)
+      if i == 0
+        population_size.nil? ? sorted_combinations[0][0] : 0
+      else
+        (sorted_combinations[i - 1][0] + sorted_combinations[i][0] + 1) / 2
+      end
+    end
+
+    def find_confidence_interval_top(one_side_threshold, population_size)
+      sorted_combinations = @distribution.sort.reverse
+      i = confidence_interval_index(sorted_combinations, one_side_threshold)
+      if i == 0
+        population_size.nil? ? sorted_combinations[0][0] : population_size
+      else
+        (sorted_combinations[i - 1][0] + sorted_combinations[i][0]) / 2
+      end
+    end
+
+    def calculate_rounded_fractions_combinations(population_size)
+      sorted_combinations = @distribution.sort
+      combinations_by_interval = []
+      sorted_combinations.each_with_index do |c, ix|
+        if ix == 0
+          bottom = 0
+        else
+          bottom = combinations_by_interval[ix - 1].first.last + 1
+        end
+        if ix == sorted_combinations.size - 1
+          top = population_size
+        else
+          top = (sorted_combinations[ix].first + sorted_combinations[ix + 1].first) / 2
+        end
+        combinations_by_interval << [[bottom, top], c.last]
+      end
+      result = Hash.new(0.to_lf)
+      combinations_by_interval.each do |i_c|
+        bottom = (i_c.first.first.to_f / population_size).round(3)
+        top = (i_c.first.last.to_f / population_size).round(3)
+        ix = bottom
+        loop do
+          interval_bottom = ((ix - 0.0005) * population_size).ceil
+          interval_top = ((ix + 0.0005) * population_size).ceil - 1
+          if (((interval_bottom + interval_top) / 2).to_f / population_size).round(3) == ix
+            low = [i_c.first.first, interval_bottom].max
+            high = [i_c.first.last, interval_top].min
+            result[ix] += i_c.last * (high + 1 - low)
+          end
+          break if ix == top
+          ix = (ix + 0.001).round(3)
+        end
+      end
+      result
+    end
+  end
+end

data/lib/sapor/denominators.rb

@@ -0,0 +1,67 @@
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Class building the denominators for D'Hondt.
+  #
+  class DhondtDenominators
+    def self.get(size)
+      Range.new(1, size)
+    end
+  end
+
+  #
+  # Class building the denominators for modified D'Hondt raising the
+  # denominators to the power 0.9.
+  #
+  class Dhondt09Denominators
+    def self.get(size)
+      Range.new(1, size).map { |a| a**0.9 }
+    end
+  end
+
+  #
+  # Class building the denominators for Sainte-Lague.
+  #
+  class SainteLagueDenominators
+    def self.get(size)
+      Range.new(1, size).map { |a| a * 2 - 1 }
+    end
+  end
+
+  #
+  # Class building the denominators for the modified Sainte-Lague starting with
+  # 1.2.
+  #
+  class SainteLague12Denominators
+    def self.get(size)
+      Range.new(1, size).map { |a| a.equal?(1) ? 1.2 : a * 2 - 1 }
+    end
+  end
+
+  #
+  # Class building the denominators for the modified Sainte-Lague starting with
+  # 1.4.
+  #
+  class SainteLague14Denominators
+    def self.get(size)
+      Range.new(1, size).map { |a| a.equal?(1) ? 1.4 : a * 2 - 1 }
+    end
+  end
+end

data/lib/sapor/dichotomies.rb

@@ -0,0 +1,138 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Represents a set of dichotomies.
+  #
+  class Dichotomies
+    include NumberFormatter
+
+    def initialize(results, population_size, threshold = nil)
+      sample_size = results.values.inject(:+)
+      @dichotomy_hash = {}
+      results.each_pair do |choice, number|
+        @dichotomy_hash[choice] = Dichotomy.new(number, sample_size,
+                                                population_size)
+      end
+      @threshold = threshold
+    end
+
+    def refine
+      @dichotomy_hash.values.each(&:refine)
+    end
+
+    def error_estimate
+      @dichotomy_hash.values.map(&:error_estimate).max
+    end
+
+    def confidence_interval_values(choice, level)
+      @dichotomy_hash[choice].confidence_interval_values(level)
+    end
+
+    def report
+      choice_lengths = @dichotomy_hash.keys.map(&:length)
+      choice_lengths << 6
+      max_choice_width = choice_lengths.max
+      sorted_choices = sort_choices_by_label_and_mpv
+      lines = sorted_choices.map do |choice|
+        create_report_line(choice, @dichotomy_hash[choice], max_choice_width)
+      end
+      "Most probable fractions and 95% confidence intervals:\n" +
+        'Choice'.ljust(max_choice_width) + '    MPF      CI(95%)' +
+        (@threshold.nil? ? '' : '     P(≥' + (100 * @threshold).to_i.to_s +
+         '%)') +
+        "\n" + lines.join("\n")
+    end
+
+    def progress_report
+      size = @dichotomy_hash.values.first.values.size
+      "Number of data points: #{with_thousands_separator(size)}."
+    end
+    
+    def size
+      @dichotomy_hash.values.first.values.size
+    end
+    
+    def write_outputs(filename)
+      write_confidence_intervals(filename)
+      write_probabilities(filename)
+    end
+
+    private
+
+    def compare_choices_by_label_and_mpv(a, b)
+      if a == OTHER
+        1
+      elsif b == OTHER
+        -1
+      else
+        mpv_a = @dichotomy_hash[a].most_probable_value
+        mpv_b = @dichotomy_hash[b].most_probable_value
+        mpv_a == mpv_b ? a <=> b : mpv_b <=> mpv_a
+      end
+    end
+
+    def sort_choices_by_label_and_mpv
+      @dichotomy_hash.keys.sort do |a, b|
+        compare_choices_by_label_and_mpv(a, b)
+      end
+    end
+
+    def create_report_line(choice, dichotomy, max_choice_width)
+      choice.ljust(max_choice_width) + '  ' + \
+        six_char_percentage(dichotomy.most_probable_fraction) + '  ' + \
+        six_char_percentage(dichotomy.confidence_interval.first) + '–' + \
+        six_char_percentage(dichotomy.confidence_interval.last) +
+        (@threshold.nil? ? '' : '  ' +
+         six_char_percentage(dichotomy.threshold_probability(@threshold)))
+    end
+    
+    def write_confidence_intervals(filename)
+      ci_filename = filename.gsub('.poll', '-dichotomies-confidence-intervals.psv')
+      File.open(ci_filename, 'w') do |file| 
+        file.puts('Choice | CI(80%) Bottom | CI(80%) Top | CI(90%) Bottom | CI(90%) Top | CI(95%) Bottom | CI(95%) Top | CI(99%) Bottom | CI(99%) Top') 
+        @dichotomy_hash.each_pair do | choice, dichotomy |
+          ci80 = dichotomy.confidence_interval(0.8)
+          ci90 = dichotomy.confidence_interval(0.9)
+          ci95 = dichotomy.confidence_interval(0.95)
+          ci99 = dichotomy.confidence_interval(0.99)
+          file.puts(choice + ' | ' + ci80.first.to_s + ' | ' + \
+                    ci80.last.to_s + ' | ' + ci90.first.to_s + ' | ' + \
+                    ci90.last.to_s + ' | ' + ci95.first.to_s + ' | ' + \
+                    ci95.last.to_s + ' | ' + ci99.first.to_s + ' | ' + \
+                    ci99.last.to_s)
+        end
+      end
+    end
+    
+    def write_probabilities(filename)
+      ci_filename = filename.gsub('.poll', '-dichotomies-probabilities.psv')
+      File.open(ci_filename, 'w') do |file|
+        intervals = Range.new(0, 1999).map{ | i | [i.to_f / 2000, (i + 1).to_f / 2000]}
+        file.puts('Choice | ' + intervals.map{|a| "#{sprintf('%.4f', a.first)}–#{sprintf('%.4f', a.last)}"}.join(' | '))
+        @dichotomy_hash.each_pair do | choice, dichotomy |
+          unless choice == OTHER
+            file.puts(choice + ' | ' + dichotomy.interval_probabilities(intervals).join(' | '))
+          end
+        end
+      end
+    end    
+  end
+end