sapor 0.1b1

data/README.md

@@ -0,0 +1,70 @@
+Statistical Analysis of Polling Results (SAPoR)
+===============================================
+
+Installation
+------------
+
+You can install this program by cloning this repository, and then executing the
+following commands in the directory where it got cloned:
+
+    sudo gem build sapor.gemspec 
+    sudo gem install sapor-0.1b1.gem 
+
+    bin/create_installation_package.sh 
+    tar -xzf sapor-0.1b1.tar.gz
+    cd sapor-0.1b1/
+    sudo ./install.sh
+
+Usage
+-----
+
+Simply call `sapor` or `sapor help` to get instructions on how to use the
+program.
+
+Examples
+--------
+
+See [this](Examples.md) page.
+
+Technical Documentation
+-----------------------
+
+See [this](Technical%20Documentation.md) page.
+
+Changelog
+---------
+
+**Version 0.1**
+
+* First round analysis of poll results as a set of dichotomies, reporting on the
+  most probable fraction, the 95% confidence interval for the vote share, and
+  probability to reach a threshold
+* Second round analysis of poll results as a polychotomy, reporting on the most
+  probable fraction, the most probable rounded fraction, the 95% confidence
+  interval for the vote share, the probability to be larger than the next party,
+  the 95% confidence interval for the number of seats in parliament, and for
+  coalitions the most probable fraction, the most probable rounded fraction, the
+  95% confidence interval for the vote share, the probability to have a majority
+  of the popular vote (vote share larger than 50%), the 95% confidence interval
+  for the number of seats and the probability to have a majority in parliament.
+* Areas: Catalan parliamentary election, 2015
+
+License
+-------
+
+Statistical Analysis of Polling Results (SAPoR)
+Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

data/Rakefile

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+require 'rspec/core/rake_task'
+
+desc 'Default: run specs.'
+task default: :spec
+
+desc 'Run specs'
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = './spec/**/*_spec.rb'
+end
+
+desc 'Run Mutant'
+task :mutant do
+  require 'mutant'
+  status = Mutant::CLI.run(['-I', 'lib', '-r', 'sapor', '--use', 'rspec', 'Sapor*'])
+  abort 'Mutant task is not successful' if status.nonzero?
+end

data/Technical Documentation.md

@@ -0,0 +1,14 @@
+Technical Documentation
+=======================
+
+Class Diagram
+-------------
+
+![Class diagram](Class%20Diagram.png "Class diagram")
+
+Areas
+-----
+
+The following diagram shows the hierarchy of all areas.
+
+![Area Class diagram](Area%20Class%20Diagram.png "Area Class diagram")

data/bin/create_installation_package.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright © 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the terms of the GNU General
+# Public License as published by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+# 
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+#
+# Creates an installation package.
+#
+
+# Create an empty temporary directory
+
+SCRIPTDIR="$( cd "$( dirname "$0" )" && pwd )"
+VERSION="0.1b1"
+TEMPDIR="sapor-${VERSION}"
+
+if [ -d "$TEMPDIR" ]; then
+    rm -R "$TEMPDIR"
+fi
+
+mkdir "$TEMPDIR"
+
+# Copy all resources to the temporary directory
+
+BINDIR=${SCRIPTDIR}/../bin
+cp ${BINDIR}/install.sh "$TEMPDIR"
+cp ${BINDIR}/sapor.sh "$TEMPDIR"
+cp ${BINDIR}/sapor.rb "$TEMPDIR"
+
+# Creates the archive file
+
+TARFILE="sapor-${VERSION}.tar.gz"
+tar -pczf $TARFILE "$TEMPDIR"
+
+# Remove the temporary directory
+
+rm -R $TEMPDIR

data/bin/install.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright © 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the terms of the GNU General
+# Public License as published by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+# 
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+#
+# Installs SAPoR into /opt, and creates a link from /usr/bin to the sapor script.
+#
+# Note: Requires root permissions to create the directory. Use sudo to execute this script.
+#
+
+SAPORDIR="/opt/sapor"
+
+if [ -d "$SAPORDIR" ]; then
+    rm -R "$SAPORDIR"
+fi
+
+mkdir "$SAPORDIR"
+
+cp sapor* "$SAPORDIR"
+chmod a+x "$SAPORDIR/sapor.sh"
+ln -f "$SAPORDIR/sapor.sh" /usr/bin/sapor
+
+LOG4R=$(gem list log4r | awk '/log4r/ {print $1}')
+if [ ${#LOG4R[@]} -eq "0" ]; then
+	gem install -r log4r
+fi
+
+SAPORGEM=$(gem list sapor | awk '/sapor/ {print $1}')
+if [ ${#SAPORGEM[@]} -eq "0" ]; then
+	gem install -r sapor
+fi

data/bin/sapor.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+require 'sapor'
+
+Sapor.analyze(ARGV[1])

data/bin/sapor.sh

@@ -0,0 +1,105 @@
+#!/bin/sh
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright © 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the terms of the GNU General
+# Public License as published by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+# 
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+#
+# Central point to run SAPoR. The script accepts the following arguments:
+# - help: To display the help information.
+# - version: To display the version information.
+# - copyright: To display the copyright information.
+# - warranty: To display the warranty information.
+#
+
+ACTION="$1"
+
+export SAPORDIR="/opt/sapor"
+export LOCALSAPORDIR="${HOME}/.sapor"
+export RUBY="ruby"
+
+VERSION="0.1b1"
+COPYRIGHTYEAR="2014"
+
+case "$ACTION" in
+  analyze)
+    if [ ! -d "$LOCALSAPORDIR" ]; then
+      mkdir "$LOCALSAPORDIR"
+    fi
+    if [ ! -d "${LOCALSAPORDIR}/cache" ]; then
+      mkdir "${LOCALSAPORDIR}/cache"
+    fi
+    $RUBY -I "${SAPORDIR}/lib" "${SAPORDIR}/sapor.rb" "${LOCALSAPORDIR}" $2
+    ;;
+  help)
+    echo "Statistical Analysis of Polling Results (SAPoR) v${VERSION}"
+    echo "Copyright © ${COPYRIGHTYEAR} Filip van Laenen <f.a.vanlaenen@ieee.org>"
+    echo
+    echo "Usage:"
+    echo "  sapor action [parameters]"
+    echo
+    echo "where actions and parameters include:"
+    echo "  analyze <poll-file>  run an analysis on the poll file"
+    echo "  help                 show this message"
+    echo "  version              show the version information"
+    echo "  copyright            show the copyright information"
+    echo "  warranty             show the warranty information"
+    ;;
+  version)
+    echo "Statistical Analysis of Polling Results (SAPoR) v${VERSION}"
+    echo "Copyright © ${COPYRIGHTYEAR} Filip van Laenen <f.a.vanlaenen@ieee.org>"
+    echo "This program comes with ABSOLUTELY NO WARRANTY; for details run 'wruf warranty'."
+    echo "This is free software, and you are welcome to redistribute it"
+    echo "under certain conditions; run 'wruf copyright' for details."
+    ;;
+  copyright)
+    echo "Statistical Analysis of Polling Results (SAPoR) v${VERSION}"
+    echo "Copyright © ${COPYRIGHTYEAR} Filip van Laenen <f.a.vanlaenen@ieee.org>"
+    echo
+    echo "This program is free software: you can redistribute it and/or modify"
+    echo "it under the terms of the GNU General Public License as published by"
+    echo "the Free Software Foundation, either version 3 of the License, or"
+    echo "(at your option) any later version."
+    echo
+    echo "This program is distributed in the hope that it will be useful,"
+    echo "but WITHOUT ANY WARRANTY; without even the implied warranty of"
+    echo "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the"
+    echo "GNU General Public License for more details."
+    echo
+    echo "You should have received a copy of the GNU General Public License"
+    echo "along with this program.  If not, see <http://www.gnu.org/licenses/>."
+    ;;
+  warranty)
+    echo "Statistical Analysis of Polling Results (SAPoR) v${VERSION}"
+    echo "Copyright © ${COPYRIGHTYEAR} Filip van Laenen <f.a.vanlaenen@ieee.org>"
+    echo
+    echo "There is no warranty for the program, to the extent permitted by applicable law."
+    echo "Except when otherwise stated in writing the copyright holders and/or other"
+    echo "parties provide the program “as is” without warranty of any kind, either"
+    echo "expressed or implied, including, but not limited to, the implied warranties of"
+    echo "merchantability and fitness for a particular purpose. The entire risk as to the"
+    echo "quality and performance of the program is with you. Should the program prove"
+    echo "defective, you assume the cost of all necessary servicing, repair or correction."
+    ;;
+  *)
+    echo "Statistical Analysis of Polling Results (SAPoR) v${VERSION}"
+    echo "Copyright © ${COPYRIGHTYEAR} Filip van Laenen <f.a.vanlaenen@ieee.org>"
+    echo
+    echo "Usage: sapor {analyze|help|version|warranty|copyright}" >&2
+    echo "Type 'sapor help' to get more information."
+    exit 1
+    ;;
+esac
+

data/lib/sapor.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+# Library namespace
+module Sapor
+  def self.analyze(filename)
+    Poll.from_file(filename).analyze
+  end
+end
+
+require 'sapor/number_formatter'
+require 'sapor/dichotomies'
+require 'sapor/combinations_distribution'
+require 'sapor/dichotomy'
+require 'sapor/log4r_logger'
+require 'sapor/log_facade'
+require 'sapor/pseudorandom_multirange_enumerator'
+require 'sapor/binomials_cache'
+require 'sapor/polychotomy'
+require 'sapor/first_past_the_post'
+require 'sapor/proportional'
+require 'sapor/leveled_proportional'
+require 'sapor/regional_data/area'
+require 'sapor/regional_data/catalonia'
+require 'sapor/regional_data/norway'
+require 'sapor/regional_data/united_kingdom'
+require 'sapor/regional_data/utopia'
+require 'sapor/poll'

data/lib/sapor/binomials_cache.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+require 'large_binomials'
+
+module Sapor
+  #
+  # Caches binomials.
+  #
+  class BinomialsCache
+    include Singleton
+
+    def initialize
+      @cache = {}
+    end
+
+    def self.binomial(k, n)
+      instance.get_binomial(k, n)
+    end
+
+    def get_binomial(k, n)
+      @cache[n] = {} unless @cache.key?(n)
+      unless @cache[n].key?(k)
+        @cache[n][k] = k.large_float_binomial_by_product_of_divisions(n)
+      end
+      @cache[n][k]
+    end
+  end
+end

data/lib/sapor/combinations_distribution.rb

@@ -0,0 +1,180 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2014 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+module Sapor
+  #
+  # Represents the distribution of combinations over a set of values. This is
+  # basically a simple hash with some helper methods for the Sapor domain.
+  #
+  class CombinationsDistribution
+    def initialize
+      @distribution = {}
+    end
+
+    def []=(value, combinations)
+      @distribution[value] = combinations
+    end
+
+    def [](value)
+      @distribution[value]
+    end
+
+    def +(other)
+      sum = CombinationsDistribution.new
+      @distribution.keys.each { |key| sum[key] = self[key] }
+      other.values.each do |key|
+        if self[key].nil?
+          sum[key] = other[key]
+        else
+          sum[key] += other[key]
+        end
+      end
+      sum
+    end
+
+    def empty?
+      @distribution.empty?
+    end
+
+    def value_threshold_probability(threshold_value)
+      total = @distribution.values.inject(:+)
+      distribution_over_threshold = @distribution.select do |k, _|
+        k >= threshold_value
+      end
+      if distribution_over_threshold.empty?
+        0
+      else
+        over_threshold = distribution_over_threshold.values.inject(:+)
+        probability = over_threshold / total
+        probability.mantissa * (10**probability.exponent)
+      end
+    end
+
+    def threshold_probability(threshold, population_size)
+      value_threshold_probability(population_size * threshold)
+    end
+
+    def size
+      @distribution.size
+    end
+
+    def values
+      @distribution.keys
+    end
+
+    def most_probable_value
+      @distribution.max { |a, b| a.last <=> b.last }[0]
+    end
+
+    # Given all fractions rounded to one decimal, returns the one that has the
+    # highest probability.
+    def most_probable_rounded_fraction(population_size)
+      rf_combinations = \
+      calculate_rounded_fractions_combinations(population_size)
+      max_probability = rf_combinations.values.max
+      opt_rfs = rf_combinations.reject { |_, v| v < max_probability }.keys
+      opt_rfs.sort[opt_rfs.size / 2]
+    end
+
+    def confidence_interval(level, population_size = nil)
+      combinations_sum = @distribution.values.inject(:+)
+      one_side_level = (1 - level) / 2
+      one_side_threshold = combinations_sum * one_side_level
+      bottom = find_confidence_interval_bottom(one_side_threshold,
+                                               population_size)
+      top = find_confidence_interval_top(one_side_threshold, population_size)
+      [bottom, top]
+    end
+
+    def confidence_interval_values(level)
+      interval = confidence_interval(level)
+      @distribution.keys.reject do |value|
+        value < interval.first || value > interval.last
+      end
+    end
+
+    private
+
+    def confidence_interval_index(sorted_combinations, one_side_threshold)
+      i = 0
+      sum_to_i = sorted_combinations[i][1]
+      while sum_to_i < one_side_threshold
+        i += 1
+        sum_to_i += sorted_combinations[i][1]
+      end
+      i
+    end
+
+    def find_confidence_interval_bottom(one_side_threshold, population_size)
+      sorted_combinations = @distribution.sort
+      i = confidence_interval_index(sorted_combinations, one_side_threshold)
+      if i == 0
+        population_size.nil? ? sorted_combinations[0][0] : 0
+      else
+        (sorted_combinations[i - 1][0] + sorted_combinations[i][0] + 1) / 2
+      end
+    end
+
+    def find_confidence_interval_top(one_side_threshold, population_size)
+      sorted_combinations = @distribution.sort.reverse
+      i = confidence_interval_index(sorted_combinations, one_side_threshold)
+      if i == 0
+        population_size.nil? ? sorted_combinations[0][0] : population_size
+      else
+        (sorted_combinations[i - 1][0] + sorted_combinations[i][0]) / 2
+      end
+    end
+
+    def calculate_rounded_fractions_combinations(population_size)
+      sorted_combinations = @distribution.sort
+      combinations_by_interval = []
+      sorted_combinations.each_with_index do |c, ix|
+        if ix == 0
+          bottom = 0
+        else
+          bottom = combinations_by_interval[ix - 1].first.last + 1
+        end
+        if ix == sorted_combinations.size - 1
+          top = population_size
+        else
+          top = (sorted_combinations[ix].first + sorted_combinations[ix + 1].first) / 2
+        end
+        combinations_by_interval << [[bottom, top], c.last]
+      end
+      result = Hash.new(0.to_lf)
+      combinations_by_interval.each do |i_c|
+        bottom = (i_c.first.first.to_f / population_size).round(3)
+        top = (i_c.first.last.to_f / population_size).round(3)
+        ix = bottom
+        loop do
+          interval_bottom = ((ix - 0.0005) * population_size).ceil
+          interval_top = ((ix + 0.0005) * population_size).ceil - 1
+          if (((interval_bottom + interval_top) / 2).to_f / population_size).round(3) == ix
+            low = [i_c.first.first, interval_bottom].max
+            high = [i_c.first.last, interval_top].min
+            result[ix] += i_c.last * (high + 1 - low)
+          end
+          break if ix == top
+          ix = (ix + 0.001).round(3)
+        end
+      end
+      result
+    end
+  end
+end