sapor 0.3.3

data/data/hu/hungary_2014_screen_scraper.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+INDEX = 'index-2014.txt'.freeze
+TARGET = 'hungary-2014.txt'.freeze
+
+BASE_URL = 'http://valasztas.hu/dyn/pv14/szavossz/en/'.freeze
+
+DISTRICT_MATCHER = %r{<h2>Result of the single-member constituencies\s+(.*)\.\s*\(.*</h2>}m
+RESULT_ROW_MATCHER = %r{<tr>\s*((<td[^>]*>[^<]*</td>\s*)+)}m
+SEQUENCE_CELL_MATCHER = %r{<td[^>]*>&nbsp;(\d+)&nbsp;</td>}m
+CELL_MATCHER = %r{<td[^>]*>([^<]*)</td>}m
+
+File.open(TARGET, 'w') do |output|
+  File.open(INDEX).each do |line|
+    url = BASE_URL + line.chomp
+    puts url
+    content = `curl -s #{url}`.encode('utf-8', 'iso-8859-2')
+    district = content.scan(DISTRICT_MATCHER).first.first.gsub(/&ndash;/, '–')
+    puts district
+    content.scan(RESULT_ROW_MATCHER) do |match|
+      row = match.first
+      unless row.scan(SEQUENCE_CELL_MATCHER).first.nil?
+        cells = row.scan(CELL_MATCHER)
+        party = cells[2].first
+        result = cells[3].first.gsub(/&nbsp;/, '')
+        output.puts "#{district} | #{party} | #{result}"
+      end
+    end
+    output.puts
+  end
+end

data/data/hu/hungary_2014_to_psv.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+SOURCE = 'hungary-2014.txt'.freeze
+TARGET = '../../lib/sapor/regional_data/hungary-2014.psv'.freeze
+
+File.open(TARGET, 'w') do |output|
+  output.puts '# Statistical Analysis of Polling Results (SAPoR)'
+  output.puts '# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>'
+  output.puts '#'
+  output.puts '# This file is part of SAPoR.'
+  output.puts '#'
+  output.puts '# SAPoR is free software: you can redistribute it and/or' \
+              ' modify it under the'
+  output.puts '# terms of the GNU General Public License as published by the' \
+              ' Free Software'
+  output.puts '# Foundation, either version 3 of the License, or (at your' \
+              ' option) any later'
+  output.puts '# version.'
+  output.puts '#'
+  output.puts '# SAPoR is distributed in the hope that it will be useful, but' \
+              ' WITHOUT ANY'
+  output.puts '# WARRANTY; without even the implied warranty of' \
+              ' MERCHANTABILITY or FITNESS FOR'
+  output.puts '# A PARTICULAR PURPOSE.  See the GNU General Public License' \
+              ' for more details.'
+  output.puts '#'
+  output.puts '# You can find a copy of the GNU General Public License in' \
+              ' /doc/gpl.txt'
+  output.puts '#'
+  File.open(SOURCE).each do |line|
+    next if line.chomp.start_with?('#')
+    if line.chomp.empty?
+      output.puts
+    else
+      elements = line.chomp.split('|').map(&:strip)
+      party = elements[1]
+      if party == 'FIDESZ-KDNP'
+        elements[1] = 'Fidesz–KDNP'
+        output.puts(elements.join(' | '))
+      elsif party == 'JOBBIK'
+        elements[1] = 'Jobbik'
+        output.puts(elements.join(' | '))
+      elsif party == 'MSZP-EGYÜTT-DK-PM-MLP'
+        elements[2] = (elements[2].to_i / 5).to_s
+        elements[1] = 'DK'
+        output.puts(elements.join(' | '))
+        elements[1] = 'Együtt'
+        output.puts(elements.join(' | '))
+        elements[1] = 'MLP'
+        output.puts(elements.join(' | '))
+        elements[1] = 'MSZP'
+        output.puts(elements.join(' | '))
+        elements[1] = 'PM'
+        output.puts(elements.join(' | '))
+      elsif party == 'SZOCIÁLDEMOKRATÁK'
+        elements[1] = 'Szociáldemokraták'
+        output.puts(elements.join(' | '))
+      else
+        output.puts line
+      end
+    end
+  end
+end

data/data/hu/index-2014.txt

@@ -0,0 +1,106 @@
+M01/E01/evkjkv_e.html
+M01/E02/evkjkv_e.html
+M01/E03/evkjkv_e.html
+M01/E04/evkjkv_e.html
+M01/E05/evkjkv_e.html
+M01/E06/evkjkv_e.html
+M01/E07/evkjkv_e.html
+M01/E08/evkjkv_e.html
+M01/E09/evkjkv_e.html
+M01/E10/evkjkv_e.html
+M01/E11/evkjkv_e.html
+M01/E12/evkjkv_e.html
+M01/E13/evkjkv_e.html
+M01/E14/evkjkv_e.html
+M01/E15/evkjkv_e.html
+M01/E16/evkjkv_e.html
+M01/E17/evkjkv_e.html
+M01/E18/evkjkv_e.html
+M02/E01/evkjkv_e.html
+M02/E02/evkjkv_e.html
+M02/E03/evkjkv_e.html
+M02/E04/evkjkv_e.html
+M03/E01/evkjkv_e.html
+M03/E02/evkjkv_e.html
+M03/E03/evkjkv_e.html
+M03/E04/evkjkv_e.html
+M03/E05/evkjkv_e.html
+M03/E06/evkjkv_e.html
+M04/E01/evkjkv_e.html
+M04/E02/evkjkv_e.html
+M04/E03/evkjkv_e.html
+M04/E04/evkjkv_e.html
+M05/E01/evkjkv_e.html
+M05/E02/evkjkv_e.html
+M05/E03/evkjkv_e.html
+M05/E04/evkjkv_e.html
+M05/E05/evkjkv_e.html
+M05/E06/evkjkv_e.html
+M05/E07/evkjkv_e.html
+M06/E01/evkjkv_e.html
+M06/E02/evkjkv_e.html
+M06/E03/evkjkv_e.html
+M06/E04/evkjkv_e.html
+M07/E01/evkjkv_e.html
+M07/E02/evkjkv_e.html
+M07/E03/evkjkv_e.html
+M07/E04/evkjkv_e.html
+M07/E05/evkjkv_e.html
+M08/E01/evkjkv_e.html
+M08/E02/evkjkv_e.html
+M08/E03/evkjkv_e.html
+M08/E04/evkjkv_e.html
+M08/E05/evkjkv_e.html
+M09/E01/evkjkv_e.html
+M09/E02/evkjkv_e.html
+M09/E03/evkjkv_e.html
+M09/E04/evkjkv_e.html
+M09/E05/evkjkv_e.html
+M09/E06/evkjkv_e.html
+M10/E01/evkjkv_e.html
+M10/E02/evkjkv_e.html
+M10/E03/evkjkv_e.html
+M11/E01/evkjkv_e.html
+M11/E02/evkjkv_e.html
+M11/E03/evkjkv_e.html
+M11/E04/evkjkv_e.html
+M12/E01/evkjkv_e.html
+M12/E02/evkjkv_e.html
+M12/E03/evkjkv_e.html
+M13/E01/evkjkv_e.html
+M13/E02/evkjkv_e.html
+M14/E01/evkjkv_e.html
+M14/E02/evkjkv_e.html
+M14/E03/evkjkv_e.html
+M14/E04/evkjkv_e.html
+M14/E05/evkjkv_e.html
+M14/E06/evkjkv_e.html
+M14/E07/evkjkv_e.html
+M14/E08/evkjkv_e.html
+M14/E09/evkjkv_e.html
+M14/E10/evkjkv_e.html
+M14/E11/evkjkv_e.html
+M14/E12/evkjkv_e.html
+M15/E01/evkjkv_e.html
+M15/E02/evkjkv_e.html
+M15/E03/evkjkv_e.html
+M15/E04/evkjkv_e.html
+M16/E01/evkjkv_e.html
+M16/E02/evkjkv_e.html
+M16/E03/evkjkv_e.html
+M16/E04/evkjkv_e.html
+M16/E05/evkjkv_e.html
+M16/E06/evkjkv_e.html
+M17/E01/evkjkv_e.html
+M17/E02/evkjkv_e.html
+M17/E03/evkjkv_e.html
+M18/E01/evkjkv_e.html
+M18/E02/evkjkv_e.html
+M18/E03/evkjkv_e.html
+M19/E01/evkjkv_e.html
+M19/E02/evkjkv_e.html
+M19/E03/evkjkv_e.html
+M19/E04/evkjkv_e.html
+M20/E01/evkjkv_e.html
+M20/E02/evkjkv_e.html
+M20/E03/evkjkv_e.html

data/data/pl/2015-gl-lis-okr.csv

@@ -0,0 +1,42 @@
+Siedziba OKW,1 - Komitet Wyborczy Prawo i Sprawiedliwość,2 - Komitet Wyborczy Platforma Obywatelska RP,3 - Komitet Wyborczy Partia Razem,4 - Komitet Wyborczy KORWiN,5 - Komitet Wyborczy Polskie Stronnictwo Ludowe,6 - Koalicyjny Komitet Wyborczy Zjednoczona Lewica SLD+TR+PPS+UP+Zieloni,7 - Komitet Wyborczy Wyborców „Kukiz'15”,8 - Komitet Wyborczy Nowoczesna Ryszarda Petru,9 - Komitet Wyborczy Wyborców JOW Bezpartyjni,10 - Komitet Wyborczy Wyborców Zbigniewa Stonogi,11 - Komitet Wyborczy Wyborców  Ruch Społeczny Rzeczypospolitej Polskiej,12 - Komitet Wyborczy Wyborców Zjednoczeni dla Śląska,13 - Komitet Wyborczy Samoobrona,14 - Komitet Wyborczy Wyborców Grzegorza Brauna „Szczęść Boże!”,15 - Komitet Wyborczy Kongres Nowej Prawicy,16 - Komitet Wyborczy Wyborców Mniejszość Niemiecka,16 - Komitet Wyborczy Wyborców Obywatele do Parlamentu
+Legnica,127.370,90.060,13.003,15.427,13.886,37.298,34.229,25.506,,,,,,,,,
+Wałbrzych,72.929,76.424,7.984,10.075,7.448,20.517,20.634,15.544,2.540,,,,,,,,
+Wrocław,163.323,159.582,22.059,27.341,13.604,31.932,45.726,55.756,,2.779,,,,1.269,,,
+Bydgoszcz,113.024,110.948,14.062,17.002,22.701,37.583,29.080,27.334,,2.543,,,,,,,
+Toruń,121.703,93.432,13.227,14.134,24.476,39.003,30.159,23.563,,2.041,,,772,,,,
+Lublin,232.014,80.892,13.395,24.376,37.733,29.172,45.448,22.158,,,599,,580,1.353,,,
+Chełm,163.122,41.823,8.116,14.835,38.689,24.161,35.567,12.745,,,,,628,,,,
+Zielona Góra,97.877,97.676,13.825,17.274,17.743,34.695,30.284,34.586,,2.258,,,,,,,
+Łódź,107.350,112.333,16.627,17.163,9.710,37.615,25.992,32.274,,,,,,,,,
+Piotrków Trybunalski,134.134,44.173,8.747,11.160,21.364,21.352,28.808,15.983,,,,,,,,,
+Sieradz,147.623,78.314,13.077,15.171,29.044,30.256,32.973,19.681,,1.905,,,712,989,,,
+Kraków I (południe),133.213,55.454,8.840,12.089,8.226,14.473,23.551,15.731,,,,,,,,,
+Kraków II (północ),209.607,133.558,21.053,34.446,14.644,34.643,39.379,52.822,,,,,,1.372,1.244,,
+Nowy Sącz,188.010,43.309,6.627,13.379,12.929,7.924,24.318,11.468,,1.606,,,,,891,,
+Tarnów,151.623,42.887,7.082,13.766,23.552,9.933,28.005,13.233,,,,,,906,654,,
+Płock,131.431,49.353,10.062,12.214,31.994,24.440,25.257,15.451,,,,,,,,,
+Radom,130.758,48.332,7.203,10.770,25.933,13.258,23.149,13.233,,2.050,,,,,656,,
+Siedlce,191.733,50.858,9.964,16.332,38.396,16.327,31.378,17.386,1.045,,582,,,1.239,,,
+Warszawa I (miasto),327.342,301.672,60.663,67.700,7.882,93.666,84.937,146.629,,,2.760,,,,,,1.964
+Warszawa II (okręg),190.355,123.227,18.895,23.586,18.666,27.774,35.418,49.098,,3.597,,,,,,,
+Opole,93.926,88.731,10.202,13.346,12.464,22.841,42.533,24.152,1.772,,,,,751,,27.530,
+Krosno,174.900,44.971,7.595,13.984,23.805,14.893,29.919,12.981,2.289,1.528,,,,,,,
+Rzeszów,284.642,66.516,11.623,25.163,23.667,22.409,47.071,21.106,,2.919,,,,1.363,778,,
+Białystok,197.575,72.894,11.258,20.297,35.116,32.015,39.509,23.361,,2.088,,,626,,629,,
+Gdańsk,126.466,148.305,16.992,21.366,12.904,28.168,30.536,39.184,,3.210,,,,,,,
+Gdynia,145.698,156.147,18.918,20.677,15.064,30.973,37.410,38.344,,3.477,,,,,,,
+Bielsko-Biała,136.330,79.506,12.557,15.788,11.533,22.122,31.570,27.882,,,,,,,,,
+Częstochowa,84.773,49.580,8.625,10.111,11.799,26.305,27.521,15.942,,1.378,,,,600,,,
+Gliwice,89.584,85.130,12.181,14.949,7.342,21.163,35.793,26.122,,,,,,1.403,,,
+Rybnik,114.799,70.188,9.669,13.136,5.197,17.201,32.794,18.341,,,,7.928,,710,,,
+Katowice,135.367,116.658,16.786,22.803,4.064,27.837,41.344,35.591,,,,10.740,,,,,
+Sosnowiec,84.410,72.755,12.652,13.682,6.687,39.774,29.161,25.522,,,,,,,,,
+Kielce,200.652,80.866,13.106,19.395,44.574,36.880,44.115,23.360,2.344,1.792,,,448,1.158,,,
+Elbląg,63.236,60.359,7.608,9.480,13.411,15.407,16.514,11.668,1.426,1.253,,,,,,,
+Olsztyn,80.970,72.055,9.948,13.584,22.458,23.312,23.888,18.141,,1.810,,,,,,,
+Kalisz,115.668,89.668,11.497,15.422,39.874,32.047,28.996,25.557,1.973,1.982,,,500,,,,
+Konin,103.781,56.111,11.032,11.061,19.027,32.649,24.486,19.237,,,,,,,,,
+Piła,73.665,83.845,10.591,11.004,20.712,24.723,24.359,18.859,,2.515,,,,,,,
+Poznań,97.975,146.141,18.854,19.548,7.809,33.080,24.825,59.387,2.267,,,,,,,,
+Koszalin,62.032,65.262,8.466,9.672,8.972,24.697,20.428,17.488,,,,,,,,,
+Szczecin,110.697,121.479,15.678,20.291,14.776,32.584,32.030,32.964,,,,,,,,,

data/data/pl/poland_2015_to_psv.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+SOURCE = '2015-gl-lis-okr.csv'.freeze
+TARGET = '../../lib/sapor/regional_data/poland-20151025.psv'.freeze
+PARTIES = ['Prawo i Sprawiedliwość',
+           'Platforma Obywatelska',
+           'Lewica Razem', 
+           'KORWiN',
+           'Polskie Stronnictwo Ludowe',
+           'Zjednoczona Lewica',
+           'Kukiz’15',
+           '.Nowoczesna',
+           'JOW Bezpartyjni',
+           'Stonoga Partia Polska',
+           'Ruch Społeczny Rzeczypospolitej Polskiej',
+           'Zjednoczeni dla Śląska',
+           'Samoobrona',
+           'Grzegorz Braun „Szczęść Boże!”',
+           'Kongres Nowej Prawicy',
+           'Mniejszość Niemiecka',
+           'Obywatele do Parlamentu']
+
+File.open(TARGET, 'w') do |output|
+  output.puts '# Statistical Analysis of Polling Results (SAPoR)'
+  output.puts '# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>'
+  output.puts '#'
+  output.puts '# This file is part of SAPoR.'
+  output.puts '#'
+  output.puts '# SAPoR is free software: you can redistribute it and/or' \
+              ' modify it under the'
+  output.puts '# terms of the GNU General Public License as published by the' \
+              ' Free Software'
+  output.puts '# Foundation, either version 3 of the License, or (at your' \
+              ' option) any later'
+  output.puts '# version.'
+  output.puts '#'
+  output.puts '# SAPoR is distributed in the hope that it will be useful, but' \
+              ' WITHOUT ANY'
+  output.puts '# WARRANTY; without even the implied warranty of' \
+              ' MERCHANTABILITY or FITNESS FOR'
+  output.puts '# A PARTICULAR PURPOSE.  See the GNU General Public License' \
+              ' for more details.'
+  output.puts '#'
+  output.puts '# You can find a copy of the GNU General Public License in' \
+              ' /doc/gpl.txt'
+  output.puts '#'
+  File.open(SOURCE).each do |line|
+    next if line.chomp.start_with?('Siedziba OKW')
+    if line.chomp.empty?
+      output.puts
+    else
+      elements = line.chomp.split(',').map(&:strip)
+      district = elements[0]
+      PARTIES.each_with_index do |p, i|
+        unless elements[i+1].nil? || elements[i+1].empty?
+          votes = elements[i+1].delete('.')
+          output.puts "#{district} | #{p} | #{votes}"
+        end
+      end
+    end
+  end
+end

data/data/pl/poland_2015_to_psv_with_ko_and_rsw.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+#
+# Statistical Analysis of Polling Results (SAPoR)
+# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>
+#
+# This file is part of SAPoR.
+#
+# SAPoR is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# SAPoR is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You can find a copy of the GNU General Public License in /doc/gpl.txt
+#
+
+SOURCE = '2015-gl-lis-okr.csv'.freeze
+TARGET = '../../lib/sapor/regional_data/poland-20151025-with-ko-and-rsw-without-n-po-r-and-zl.psv'.freeze
+
+N_PARTY = '.Nowoczesna'.freeze
+PO_PARTY = 'Platforma Obywatelska'.freeze
+R_PARTY = 'Lewica Razem'.freeze
+ZL_PARTY = 'Zjednoczona Lewica'.freeze
+PARTIES = ['Prawo i Sprawiedliwość',
+           PO_PARTY,
+           R_PARTY, 
+           'KORWiN',
+           'Polskie Stronnictwo Ludowe',
+           ZL_PARTY,
+           'Kukiz’15',
+           N_PARTY,
+           'JOW Bezpartyjni',
+           'Stonoga Partia Polska',
+           'Ruch Społeczny Rzeczypospolitej Polskiej',
+           'Zjednoczeni dla Śląska',
+           'Samoobrona',
+           'Grzegorz Braun „Szczęść Boże!”',
+           'Kongres Nowej Prawicy',
+           'Mniejszość Niemiecka',
+           'Obywatele do Parlamentu']
+
+File.open(TARGET, 'w') do |output|
+  output.puts '# Statistical Analysis of Polling Results (SAPoR)'
+  output.puts '# Copyright (C) 2016 Filip van Laenen <f.a.vanlaenen@ieee.org>'
+  output.puts '#'
+  output.puts '# This file is part of SAPoR.'
+  output.puts '#'
+  output.puts '# SAPoR is free software: you can redistribute it and/or' \
+              ' modify it under the'
+  output.puts '# terms of the GNU General Public License as published by the' \
+              ' Free Software'
+  output.puts '# Foundation, either version 3 of the License, or (at your' \
+              ' option) any later'
+  output.puts '# version.'
+  output.puts '#'
+  output.puts '# SAPoR is distributed in the hope that it will be useful, but' \
+              ' WITHOUT ANY'
+  output.puts '# WARRANTY; without even the implied warranty of' \
+              ' MERCHANTABILITY or FITNESS FOR'
+  output.puts '# A PARTICULAR PURPOSE.  See the GNU General Public License' \
+              ' for more details.'
+  output.puts '#'
+  output.puts '# You can find a copy of the GNU General Public License in' \
+              ' /doc/gpl.txt'
+  output.puts '#'
+  File.open(SOURCE).each do |line|
+    next if line.chomp.start_with?('Siedziba OKW')
+    if line.chomp.empty?
+      output.puts
+    else
+      elements = line.chomp.split(',').map(&:strip)
+      district = elements[0]
+      ko_no = 0
+      rsw_no = 0
+      PARTIES.each_with_index do |p, i|
+        unless elements[i+1].nil? || elements[i+1].empty?
+          votes = elements[i+1].delete('.')
+          if p == R_PARTY || p == ZL_PARTY
+            rsw_no += votes.to_i
+          elsif p == PO_PARTY || p == N_PARTY
+            ko_no += votes.to_i
+          else
+            output.puts "#{district} | #{p} | #{votes}"
+          end
+        end
+      end
+      output.puts "#{district} | Koalicja Obywatelska | #{ko_no}"
+      output.puts "#{district} | Lewica Razem–Sojusz Lewicy Demokratycznej–Wiosna | #{rsw_no}"
+    end
+  end
+end