sapor 0.3.6

data/lib/sapor/options.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2020 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+
+  #
+  # Represents a set of options.
+  #
+  class Options
+    DEFAULT_MAX_ERROR = 0.001
+    DEFAULT_MAX_POLYCHOTOMY_ITERATIONS = 2 ** 25
+    DEFAULT_MIN_DICHOTOMIES_ITERATIONS = 0
+    DEFAULT_MIN_POLYCHOTOMY_ITERATIONS = 2 ** 20
+
+    MAX_ERROR_KEY = 'emx'
+    MAX_POLYCHOTOMY_ITERATIONS_KEY = 'pmx'
+    MIN_DICHOTOMIES_ITERATIONS_KEY = 'dmn'
+    MIN_POLYCHOTOMY_ITERATIONS_KEY = 'pmn'
+    
+    FLOAT_OPTIONS = [MAX_ERROR_KEY]
+    INTEGER_OPTIONS = [MAX_POLYCHOTOMY_ITERATIONS_KEY, MIN_DICHOTOMIES_ITERATIONS_KEY, MIN_POLYCHOTOMY_ITERATIONS_KEY]
+
+    def initialize(arguments)
+      @hash = { MAX_ERROR_KEY => DEFAULT_MAX_ERROR,
+        MAX_POLYCHOTOMY_ITERATIONS_KEY => DEFAULT_MAX_POLYCHOTOMY_ITERATIONS,
+        MIN_DICHOTOMIES_ITERATIONS_KEY => DEFAULT_MIN_DICHOTOMIES_ITERATIONS,
+        MIN_POLYCHOTOMY_ITERATIONS_KEY => DEFAULT_MIN_POLYCHOTOMY_ITERATIONS }
+      arguments.each do | argument |
+        key_value = argument[1..-1].split('=')
+        key = key_value.first
+        value = key_value.last
+        if INTEGER_OPTIONS.include?(key)
+          value = value.to_i
+        elsif FLOAT_OPTIONS.include?(key)
+          value = value.to_f
+        end
+        @hash[key] = value
+      end
+    end
+    
+    def max_error
+      @hash[MAX_ERROR_KEY]
+    end
+    
+    def max_polychotomy_iterations
+      @hash[MAX_POLYCHOTOMY_ITERATIONS_KEY]
+    end
+
+    def min_dichotomies_iterations
+      @hash[MIN_DICHOTOMIES_ITERATIONS_KEY]
+    end
+
+    def min_polychotomy_iterations
+      @hash[MIN_POLYCHOTOMY_ITERATIONS_KEY]
+    end
+  end
+end

data/lib/sapor/poll.rb

@@ -0,0 +1,282 @@
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2020 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+require 'yaml'
+
+module Sapor
+  OTHER = 'Other'.freeze
+
+  #
+  # Represents a poll.
+  #
+  class Poll
+    include NumberFormatter
+    attr_reader :area, :logger, :type
+
+    AREA_KEY = 'Area'.freeze
+
+    BELGIAN_AREAS = [BelgiumBrussels.instance, BelgiumFlanders.instance,
+                     BelgiumWallonia.instance, Flanders.instance,
+                     Wallonia.instance].freeze
+    DANISH_AREAS = [Denmark.instance, DenmarkWithE.instance,
+                    DenmarkWithEAndP.instance, DenmarkWithP.instance].freeze
+    EUROPEAN_UNION_AREAS = [EuropeanUnion27Austria.instance,
+                            EuropeanUnion27Croatia.instance,
+                            EuropeanUnion27Denmark.instance,
+                            EuropeanUnion27Estonia.instance,
+                            EuropeanUnion27Finland.instance,
+                            EuropeanUnion27France.instance,
+                            EuropeanUnion27Ireland.instance,
+                            EuropeanUnion27IrelandWithIa.instance,
+                            EuropeanUnion27Italy.instance,
+                            EuropeanUnion27Netherlands.instance,                            
+                            EuropeanUnion27Poland.instance,
+                            EuropeanUnion27Romania.instance,
+                            EuropeanUnion27Slovakia.instance,
+                            EuropeanUnion27Spain.instance,
+                            EuropeanUnion27Sweden.instance,
+                            EuropeanUnionAustria.instance,
+                            EuropeanUnionBulgaria.instance,
+                            EuropeanUnionCroatia.instance,
+                            EuropeanUnionCyprus.instance,
+                            EuropeanUnionCzechRepublic.instance,
+                            EuropeanUnionDenmark.instance,
+                            EuropeanUnionEstonia.instance,
+                            EuropeanUnionFinland.instance,
+                            EuropeanUnionFlanders.instance,
+                            EuropeanUnionFrance.instance,
+                            EuropeanUnionFrenchCommunityOfBelgium.instance,
+                            EuropeanUnionGermany.instance,
+                            EuropeanUnionGreatBritain.instance,
+                            EuropeanUnionGreece.instance,
+                            EuropeanUnionHungary.instance,
+                            EuropeanUnionIreland.instance,
+                            EuropeanUnionIrelandWithIa.instance,
+                            EuropeanUnionItaly.instance,
+                            EuropeanUnionLatvia.instance,
+                            EuropeanUnionLithuania.instance,
+                            EuropeanUnionLuxembourg.instance,
+                            EuropeanUnionMalta.instance,
+                            EuropeanUnionNetherlands.instance,
+                            EuropeanUnionNorthernIreland.instance,
+                            EuropeanUnionPoland.instance,
+                            EuropeanUnionPortugal.instance,
+                            EuropeanUnionRomania.instance,
+                            EuropeanUnionSlovakia.instance,
+                            EuropeanUnionSlovenia.instance,
+                            EuropeanUnionSpain.instance,
+                            EuropeanUnionSweden.instance].freeze
+    NORWEGIAN_AREAS = [Norway.instance, NorwegianMunicipality::BERGEN,
+                       NorwegianMunicipality::OSLO,
+                       NorwegianMunicipality::TRONDHEIM].freeze
+    POLISH_AREAS = [Poland.instance,
+                    PolandWithKoAndLWithoutNPoRAndZl.instance,
+                    PolandWithKoKonfKpLAndZpWithoutKK15NPisPoPslRAndZl.instance,
+                    PolandWithKoSldAndWiWithoutNPoAndZl.instance,
+                    PolandWithSldAndWiWithoutZL.instance,
+                    PolandWithSldWithoutZL.instance].freeze
+    PORTUGUESE_AREAS = [Portugal.instance,
+                        PortugalWithAAndChWithoutPaf.instance,
+                        PortugalWithAAndIlWithoutPaf.instance,
+                        PortugalWithAChAndIlWithoutPaf.instance,
+                        PortugalWithAWithoutPaf.instance,
+                        PortugalWithChAndIlWithoutPaf.instance,
+                        PortugalWithoutPaf.instance].freeze
+    AREAS_MAP = {}
+    (BELGIAN_AREAS + DANISH_AREAS + EUROPEAN_UNION_AREAS + NORWEGIAN_AREAS + \
+     POLISH_AREAS + PORTUGUESE_AREAS + \
+     [Austria.instance, Catalonia.instance, CataloniaWithVox.instance,
+      Estonia.instance, Finland.instance, FinlandWithSin.instance,
+      France.instance, Greece.instance, Hungary.instance, Iceland.instance,
+      IcelandWithJ.instance, Ireland.instance, Latvia.instance,
+      Luxembourg.instance, Netherlands.instance, Slovakia.instance,
+      Slovenia.instance, Spain.instance, Sweden.instance,
+      Sweden20140914.instance, UnitedKingdom.instance,
+      UnitedKingdomWithBrexit.instance, UnitedKingdomWithBrexitAndChuk.instance,
+      UnitedKingdomWithTig.instance, Utopia.instance]).map do |area|
+      AREAS_MAP[area.area_code] = area
+    end
+
+    TYPE_KEY = 'Type'.freeze
+    REFERENDUM_TYPE_VALUE = 'Referendum'.freeze
+    ELECTION_TYPE_VALUE = 'Election'.freeze
+
+    DEFAULT_CONFIDENCE_LEVEL = 0.95
+
+    def initialize(filename, metadata, results)
+      @filename = filename
+      @logger = LogFacade.create_logger
+      @area_code = metadata.delete(AREA_KEY)
+      @area = Poll.lookup_area(@area_code)
+      @type = metadata.delete(TYPE_KEY)
+      @results = interpret(results)
+    end
+
+    def analyze(options)
+      analyze_as_dichotomies(options)
+      analyze_as_polychotomy(options)
+      @logger.info('Done.')
+    end
+
+    def continue_analysis(options)
+      @logger = LogFacade.create_logger
+      @logger.info('Continuing the analysis as a polychotomy...')
+      @area = Poll.lookup_area(@area_code)
+      @analysis.revive_volatile_fields(@logger)
+      analyze_until_convergence(options)
+      @logger.info('Done.')
+    end
+
+    def confidence_interval(choice, level = DEFAULT_CONFIDENCE_LEVEL)
+      @analysis.confidence_interval(choice, level) unless @analysis.nil?
+    end
+
+    def most_probable_fraction(choice)
+      @analysis.most_probable_fraction(choice) unless @analysis.nil?
+    end
+
+    def most_probable_value(choice)
+      @analysis.most_probable_value(choice) unless @analysis.nil?
+    end
+
+    def result(choice)
+      @results[choice]
+    end
+
+    def encode_with(coder)
+      (instance_variables - [:@area, :@logger]).each do |var|
+        var = var.to_s
+        coder[var.gsub('@', '')] = eval(var)
+      end
+    end
+
+    def self.lookup_area(area_code)
+      AREAS_MAP[area_code]
+    end
+
+    private
+
+    def self.line_to_hash(line, current, results)
+      if line.chomp.eql?('==')
+        current = results
+      else
+        elements = line.chomp.split('=')
+        current[elements.first] = elements.last
+      end
+      current
+    end
+
+    def self.as_hashes(lines)
+      metadata = {}
+      results = {}
+      current = metadata
+      lines.each do |line|
+        current = line_to_hash(line, current, results)
+      end
+      [metadata, results]
+    end
+
+    def self.from_lines(filename, lines)
+      hashes = as_hashes(lines)
+      metadata = hashes.first
+      results = hashes.last
+      new(filename, metadata, results)
+    end
+
+    def self.from_file(filename)
+      from_lines(filename, File.open(filename))
+    end
+
+    def interpret(results)
+      interpreted = {}
+      results.each_pair do |key, value|
+        interpreted[key] = value.to_i
+      end
+      interpreted
+    end
+
+    def population_size
+      @area.population_size
+    end
+
+    def referendum?
+      @type == REFERENDUM_TYPE_VALUE
+    end
+
+    def threshold
+      if referendum?
+        0.5
+      else
+        @area.threshold
+      end
+    end
+
+    def analyze_until_convergence(options)
+      while should_continue_analysis?(@analysis, options)
+        @analysis.refine
+        @logger.info(@analysis.report)
+        @logger.info('Error estimate: ε ≤ ' +
+                     three_digits_percentage(@analysis.error_estimate) +
+                     '.')
+        @logger.info(@analysis.progress_report)
+        @analysis.write_outputs(@filename)
+        save_state
+      end
+    end
+
+    def should_continue_analysis?(analysis, options)
+      analysis.kind_of?(Dichotomies) \
+        && (options.min_dichotomies_iterations > analysis.size \
+            || analysis.error_estimate > options.max_error) \
+        || analysis.kind_of?(Polychotomy) \
+           && options.max_polychotomy_iterations > analysis.no_of_simulations \
+           && (options.min_polychotomy_iterations > analysis.no_of_simulations \
+               || analysis.error_estimate > options.max_error)
+    end
+
+    def analyze_as_dichotomies(options)
+      @logger.info('Analyzing as a set of dichotomies...')
+      @analysis = Dichotomies.new(@results, population_size, threshold)
+      analyze_until_convergence(options)
+    end
+
+    def analyze_as_polychotomy(options)
+      @logger.info('Analyzing as a polychotomy...')
+      if referendum?
+        @analysis = ReferendumPolychotomy.new(@results, @area, @analysis,
+                                              options.max_error)
+      else
+        @analysis = RepresentativesPolychotomy.new(@results, @area, @analysis,
+                                                   options.max_error, @logger)
+      end
+      analyze_until_convergence(options)
+    end
+
+    def save_state
+      new_yaml_file = @filename.gsub('.poll', '-state-new.yaml')
+      open(new_yaml_file, 'w') do |file|
+        file.write(to_yaml)
+      end
+      yaml_file = @filename.gsub('.poll', '-state.yaml')
+      old_yaml_file = @filename.gsub('.poll', '-state-old.yaml')
+      File.delete(old_yaml_file) if File.exist?(old_yaml_file)
+      File.rename(yaml_file, old_yaml_file) if File.exist?(yaml_file)
+      File.rename(new_yaml_file, yaml_file)
+    end
+  end
+end

data/lib/sapor/polychotomy.rb

@@ -0,0 +1,200 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2020 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Superclass for polychotomies.
+  #
+  class Polychotomy
+    attr_reader :error_estimate, :no_of_data_points, :no_of_simulations
+
+    def initialize(results, area, dichotomies, max_error)
+      initialize_static_information(results, area, dichotomies, max_error)
+      initialize_calculation_information
+    end
+
+    def most_probable_fraction(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        calculate_most_probable_fraction(key, @votes)
+      end
+    end
+
+    def most_probable_value(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        @votes[key].most_probable_value
+      end
+    end
+
+    def range(choice)
+      @ranges[choice]
+    end
+
+    def space_size
+      @enum.size
+    end
+
+    def encode_with(coder)
+      (instance_variables - [:@area, :@enum, :@logger]).each do |var|
+        var = var.to_s
+        coder[var.gsub('@', '')] = eval(var)
+      end
+    end
+    
+    def revive_volatile_fields(logger = nil)
+      @area = Poll.lookup_area(@area_code)
+      @enum = create_enumerator(@ranges)
+      @logger = logger
+      1.upto(@no_of_data_points).each { |i| @enum.next() }
+    end
+    
+    private
+
+    def calculate_most_probable_fraction(key, votes)
+      votes[key].most_probable_value.to_f / @area.population_size
+    end
+
+    def calculate_most_probable_rounded_fraction(key, votes)
+      votes[key].most_probable_rounded_fraction(@area.population_size)
+    end
+
+    def create_comparisons(choices)
+      comparisons = {}
+      choices.each do |a|
+        choices.each do |b|
+          comparisons[a + '>' + b] = 0.to_lf
+        end
+      end
+      comparisons
+    end
+
+    def create_enumerator(ranges)
+      range_sizes = ranges.values.map(&:size)
+      PseudoRandomMultiRangeEnumerator.new(range_sizes).each
+    end
+
+    def create_rankings(choices)
+      rankings = {}
+      choices.each do |choice|
+        1.upto(choices.size).each do |i|
+          rankings[choice + '@' + (i - 1).to_s] = 0.to_lf
+        end
+      end
+      rankings
+    end
+
+    def extract_ranges_from_dichotomies(dichotomies, max_error)
+      ranges = {}
+      level = 1 - (max_error**2)
+      @choices.each do |choice|
+        unless choice == OTHER
+          ranges[choice] = dichotomies.confidence_interval_values(choice,
+                                                                  level).sort
+        end
+      end
+      ranges
+    end
+
+    def initialize_calculation_information
+      @enum = create_enumerator(@ranges)
+      @no_of_simulations = 0
+      @no_of_data_points = 0
+      @combinations_sum = 0.to_lf
+      @error_estimate = 1.0
+      @votes = create_new_votes_distributions
+    end
+
+    def initialize_static_information(results, area, dichotomies, max_error)
+      @results = results
+      @area = area
+      @area_code = area.area_code
+      @choices = results.keys
+      @comparisons = create_comparisons(@choices)
+      @rankings = create_rankings(@choices)
+      @ranges = extract_ranges_from_dichotomies(dichotomies, max_error)
+    end
+
+    def larger_than(a, b)
+      probability = @comparisons[a + '>' + b] / @combinations_sum
+      probability.mantissa * (10**probability.exponent)
+    end
+
+    def most_probable_rounded_fraction(key)
+      if @no_of_simulations == 0
+        nil
+      else
+        calculate_most_probable_rounded_fraction(key, @votes)
+      end
+    end
+
+    def next_data_point
+      indexes = @enum.next
+      data_point = {}
+      indexes.each_with_index do |ix, i|
+        data_point[@ranges.keys[i]] = @ranges.values[i][ix]
+      end
+      data_point[OTHER] = @area.population_size - data_point.values.inject(:+)
+      data_point
+    end
+
+    def ranking(choice, i)
+      probability = @rankings[choice + '@' + i.to_s] / @combinations_sum
+      probability.mantissa * (10**probability.exponent)
+    end
+
+    def result(choice)
+      @results[choice].to_f / @results.values.inject(:+)
+    end
+
+    def sort_choices_by_result
+      sorted_choices = @choices.reject { |choice| choice == OTHER }
+      sorted_choices.sort do |a, b|
+        comparison = result(b) <=> result(a)
+        if comparison == 0
+          a <=> b
+        else
+          comparison
+        end
+      end
+    end
+    
+    def write_rankings(filename)
+      ci_filename = filename.gsub('.poll', '-polychotomy-rankings.psv')
+      File.open(ci_filename, 'w') do |file| 
+        file.puts('Choice | ' + 0.upto(@choices.size - 1).map {|i| i + 1}.join(' | ')) 
+        @choices.each do | choice |
+          file.puts(choice + ' | ' + 0.upto(@choices.size - 1).map {|i| ranking(choice, i)}.join(' | '))
+        end
+      end
+    end
+    
+    def write_state_summary(filename)
+      state_summary_filename = filename.gsub('.poll', '_state_summary.txt')
+      File.open(state_summary_filename, 'w') do |file|
+        file.puts("ErrorEstimate=#{@error_estimate}")
+        file.puts("NumberOfSimulations=#{@no_of_simulations}")
+        file.puts("NumberOfDataPoints=#{@no_of_data_points}")
+        file.puts("SpaceSize=#{space_size}")
+      end
+    end    
+  end
+end