iraq_unrest 0.0.1

data/lib/iraq_unrest/format.rb

@@ -0,0 +1,103 @@
+module IraqUnrest
+
+  # Code from https://github.com/sandal/fatty
+  class Format
+    attr_accessor :params
+
+    # FIXME: Need to understand where the methods to be
+    #        implemented are declared
+    def validate
+    end
+  end
+
+  class Formatter
+
+    class << self
+
+      def formats
+        @formats ||= {}
+      end
+
+      def required_params(*args)
+        @required_params = args
+      end
+
+      def format(name, options={}, &block)
+        formats[name] = Class.new(IraqUnrest::Format, &block)
+      end
+
+      def validate(format, params={})
+        check_required_params(params)
+      end
+
+     # puts ListFormatter.render(:csv, :data => data)
+     def render(format, params={})
+        validate(format, params)
+
+        # Each format object implements all the required steps
+        # so they they can be used interchangeably
+        format_obj = formats[format].new
+
+        # Customize
+        format_obj.params = params
+        format_obj.validate
+        format_obj.render
+      end
+
+      def render_file(file, params={})
+        format = File.extname(file).delete(".").to_sym
+
+        File.open(file, "w") do |f|
+          f << render(format, params)
+        end
+      end
+
+      private
+
+      def check_required_params(params)
+        unless (@required_params || []).all? { |k| params.key?(k) }
+          raise Exception, "One or more required params is missing"
+        end
+      end
+
+    end
+  end
+
+  class DataSetFormatter < Formatter
+
+    required_params :object
+
+    format :csv do
+
+      def render
+        tpl = <<-EOS
+          # header
+          csv << self::ATTRIBUTES
+
+          # data rows
+          self.all.each do |row|
+            csv << row.attributes.values
+          end
+        EOS
+        template = Tilt::CSVTemplate.new { tpl }
+        template.render(params[:object])
+      end
+    end
+
+    format :html do
+
+      def render
+        object = params[:object]
+        erb_dir = File.dirname(__FILE__) + "/erb"
+
+        template = Tilt.new(erb_dir + "/" + object.file_name + ".html.erb")
+        header   = Tilt.new(erb_dir + '/header.html.erb')
+        template.render(self, :header => header.render,
+                              :data   => object.as_rickshaw)
+      end
+
+    end
+
+  end
+
+end

data/lib/iraq_unrest/google_doc.rb

@@ -0,0 +1,31 @@
+module IraqUnrest
+
+  class GoogleDoc
+
+    SPREADSHEETS = {:iraq_government_casualty_figure => 4,
+                    :iraqi_casualties_comparison => 9}
+
+    def spreadsheet(name)
+      raise ArgumentError, "Invalid spreadsheet name" unless SPREADSHEETS.include?(name)
+
+      begin
+        uri = URI.parse("https://docs.google.com/spreadsheet/pub?hl=en_US&key=%s&hl=en_US&gid=%s&output=csv" %
+                        [IraqUnrest::Config.doc_id, SPREADSHEETS[name]])
+
+        curl = ::Curl::Easy.perform(uri.to_s) do |c|
+            c.timeout = IraqUnrest::Config.timeout
+        end
+      rescue Curl::Err::TimeoutError => e
+        raise $!, "Could not connect to Google Docs in a timely manner #{$!}", $!.backtrace
+      end
+
+      if curl.response_code != 200
+        raise Exception, "Did not receive a successfull response from Google Docs"
+      end
+
+      curl.body_str
+    end
+
+  end
+
+end

data/lib/iraq_unrest/iraq_government_casualty_figure.rb

@@ -0,0 +1,53 @@
+module IraqUnrest
+
+  class IraqGovernmentCasualtyFigure < DataSet
+
+    ATTRIBUTES = [ :date, :civilian_killed, :police_killed, :army_killed,
+                   :civilian_wounded, :police_wounded, :army_wounded,
+                   :insurg_killed, :insurg_arrested ]
+
+    attr_accessor *ATTRIBUTES
+
+    def attributes
+      ATTRIBUTES.inject(ActiveSupport::HashWithIndifferentAccess.new) do |result, key|
+        result[key] = read_attribute_for_validation(key)
+        result
+      end
+    end
+
+    def self.raw_csv
+      GoogleDoc.new.spreadsheet(file_name.to_sym)
+    end
+
+    def self.parse(data)
+      result = []
+      parsed = data.split("\n")
+      title = parsed.slice(0).split(",").first.gsub(/\(|\(|\:|\s+/, "").chop.underscore
+      comments = parsed.slice!(1..4)
+
+      if title != file_name
+        raise Exception, "Could not find a valid title in data source"
+      end
+
+      parsed.each do |row|
+        fields = row.split(",", -1)
+
+        obj = new(:date => fields[0],
+                  :civilian_killed => fields[1],
+                  :police_killed => fields[2],
+                  :army_killed => fields[3],
+                  :civilian_wounded => fields[5],
+                  :police_wounded => fields[6],
+                  :army_wounded => fields[7],
+                  :insurg_killed => fields[9],
+                  :insurg_arrested => fields[10])
+
+        result << obj if obj.valid?
+      end
+
+      result
+    end
+
+  end
+
+end

data/lib/iraq_unrest/iraq_unrest_record.rb

@@ -0,0 +1,74 @@
+module IraqUnrest
+
+  class DataSet
+
+    def initialize(attributes={})
+      self.attributes=attributes
+    end
+
+    def attributes=(attrs)
+      attrs.each_pair do |k, v|
+        send("#{k}=", v)
+      end
+    end
+
+    class << self
+
+      def parse
+      end
+
+      def all
+        parse(self.raw_csv)
+      end
+
+      def raw_csv
+        doc = IraqUnrest::GoogleDoc.new.spreadsheet(self.file_name.to_sym)
+      end
+
+      def file_name
+        self.name.split("::").last.underscore
+      end
+
+      # Need to make this clear that this is picking up the
+      # result of ActiveModelSerializer
+      def to_json
+        ActiveModel::ArraySerializer.new(self.all).to_json
+      end
+
+      def as_json
+        ActiveModel::ArraySerializer.new(self.all).as_json
+      end
+
+      # FIXME - we could probably remove a layer of abstraction here
+      def to_rickshaw
+        IraqUnrest::Serializers.as_rickshaw(self.all)
+      end
+
+      def to_csv
+        IraqUnrest::Serializers.as_csv(self.all)
+      end
+
+      def to_html
+        IraqUnrest::Serializers.as_html(self.all)
+      end
+
+      def generate_csv!
+        file = File.new(self.file_name + ".csv", "w")
+        file.puts IraqUnrest::Config.disclaimer
+        csv_data = self.to_csv.to_s.split("\n")
+        csv_data.each { |row| file.puts row }
+
+        file.close
+        file
+      end
+
+      def generate_html!
+        file = File.new(self.file_name + ".html", "w")
+        file.puts self.to_html
+        file.close
+        file
+      end
+
+    end
+  end
+end

data/lib/iraq_unrest/iraqi_casualties_comparison.rb

@@ -0,0 +1,46 @@
+module IraqUnrest
+
+  class IraqiCasualtiesComparison < DataSet
+
+    ATTRIBUTES = [:date, :afp, :iraq_gov, :iraq_body_count]
+
+    attr_accessor *ATTRIBUTES
+
+    def attributes
+      ATTRIBUTES.inject(ActiveSupport::HashWithIndifferentAccess.new) do |result, key|
+        result[key] = read_attribute_for_validation(key)
+        result
+      end
+    end
+
+    def self.raw_csv
+      GoogleDoc.new.spreadsheet(file_name.to_sym)
+    end
+
+    def self.parse(data)
+      result = []
+      parsed = data.split("\n")
+
+      title = parsed.slice!(0).split(",").first.gsub(/\(|\(|\:|A |\s+/, "").underscore
+      comments = parsed.slice!(0..4)
+
+      if title != file_name
+        raise Exception, "Could not find a valid title in data source"
+      end
+
+      parsed.each do |row|
+        fields = row.split(",", -1)
+        obj = new(:date => fields[0],
+                  :iraq_gov  => fields[1],
+                  :iraq_body_count => fields[2],
+                  :afp => fields[3])
+
+        result << obj if obj.valid?
+      end
+
+      result
+    end
+
+  end
+
+end

data/lib/iraq_unrest/serializers.rb

@@ -0,0 +1,46 @@
+module IraqUnrest
+
+  module Serializers
+
+    # Gives to_json and as_json
+    # Used by both members and collections
+    class IraqiCasualtiesComparisonSerializer < ::ActiveModel::Serializer
+      attributes :date, :afp, :iraq_gov, :iraq_body_count
+
+      self.root = false
+
+      def date
+        Date.strptime(object.date, "%b-%Y").end_of_month.to_time.to_i
+      end
+    end
+
+    class IraqGovernmentCasualtyFigureSerializer < ::ActiveModel::Serializer
+      attributes :date, :civilian_killed, :police_killed, :army_killed,
+                 :civilian_wounded, :police_wounded, :army_wounded,
+                 :insurg_killed, :insurg_arrested
+
+      self.root = false
+
+      def date
+        Date.strptime(object.date, "%b-%Y").end_of_month.to_time.to_i
+      end
+    end
+
+    def self.as_rickshaw(data)
+      result = {}
+      attrs = data.first.attributes.except(:date).symbolize_keys.keys
+
+      data.each do |row|
+        attrs.each do |attr|
+          result[attr] ||= []
+          result[attr] << { :x => Date.strptime(row.date, "%b-%Y").end_of_month.to_time.to_i,
+                            :y => row.send(attr).to_i ||= 0 }
+        end
+      end
+
+      result.each {|k,v| result[k].sort_by! { |hsh| hsh[:x]} }
+      result
+    end
+
+  end
+end

data/lib/iraq_unrest/version.rb

@@ -0,0 +1,3 @@
+module IraqUnrest
+  VERSION = "0.0.1"
+end

data/lib/iraq_unrest.rb

@@ -0,0 +1,50 @@
+require 'tilt'
+require 'json'
+require 'curb'
+require 'csv'
+
+require 'active_model_serializers'
+require 'active_support/concern'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/object' # gives us to_json
+require 'active_support/inflector'
+
+require "iraq_unrest/version"
+require "iraq_unrest/config"
+require "iraq_unrest/concerns/validatable"
+require "iraq_unrest/concerns/serializable"
+require "iraq_unrest/serializers"
+require "iraq_unrest/format"
+require "iraq_unrest/google_doc"
+require "iraq_unrest/data_set"
+require "iraq_unrest/iraq_government_casualty_figure"
+require "iraq_unrest/iraqi_casualties_comparison"
+
+IraqUnrest::Config.configure do |config|
+  config.doc_id = "0Aia6y6NymliRdEZESktBSWVqNWM1dkZOSGNIVmtFZEE"
+  config.timeout = 30
+  config.disclaimer = <<-EOS
+##################################################
+# NOTE: This file was generated by a free software
+# program [ https://github.com/bds/iraq_unrest ]
+# which retrieved publicly shared data
+# provided by Agence France-Presse.
+#
+# Figures are provided by Iraqi officials
+# on a monthly basis, and have been compiled by
+# Agence France-Presse at:
+#
+# http://u.afp.com/JSL
+#
+# They are based on data released by the Iraqi
+# ministries of health, interior and defence.
+# If you have any questions, please contact:
+#
+# AFP's Baghdad Bureau
+# https://twitter.com/prashantrao
+##################################################
+EOS
+end
+
+module IraqUnrest
+end

data/test/fixtures/vcr_cassettes/iraq_government_casualty_figure.yml

@@ -0,0 +1,161 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://docs.google.com/spreadsheet/pub?gid=4&hl=en_US&key=0Aia6y6NymliRdEZESktBSWVqNWM1dkZOSGNIVmtFZEE&output=csv
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers: {}
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Content-Type:
+      - text/csv
+      X-Robots-Tag:
+      - noindex, nofollow, nosnippet
+      Content-Disposition:
+      - attachment; filename="Iraq Unrest.csv"
+      Set-Cookie:
+      - NID=67=uwoB2ugbyq-jGSmZq1XiowMDE5_FXC2Pu4_HQPzWowKD3_xYmyXAOshxb8BiGerBt0oWLDZw1S0WTvuQpCiDYu-0qvBvjorrPCNfbGUOqK-vANG2-qAcQbo2hMq8S7Kd;Domain=.google.com;Path=/;Expires=Wed,
+        14-May-2014 16:29:07 GMT;HttpOnly
+      P3p:
+      - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657
+        for more info."
+      Date:
+      - Tue, 12 Nov 2013 16:29:07 GMT
+      Expires:
+      - Tue, 12 Nov 2013 16:29:07 GMT
+      Cache-Control:
+      - private, max-age=0
+      X-Content-Type-Options:
+      - nosniff
+      X-Xss-Protection:
+      - 1; mode=block
+      Server:
+      - GSE
+      Alternate-Protocol:
+      - 443:quic
+      Transfer-Encoding:
+      - chunked
+    body:
+      encoding: ASCII-8BIT
+      string: |-
+        Iraq Government Casualty Figures,,,,,,,,,,
+        ,,,,,,,,,,
+        "Note: These figures are provided by Iraqi officials on a monthly basis, and have been compiled by Agence France-Presse. They are based on data released by the Iraqi ministries of health, interior and defence. If you have any questions, please leave a comment or contact AFP's Baghdad Bureau.",,,,,,,,,,
+        ,,,,,,,,,,
+        ,Killed,,,,Wounded,,,,Insurgents,
+        ,Civilians,Police,Army,Total,Civilians,Police,Army,Total,Killed,Arrested
+        Dec-2013,,,,0,,,,0,,
+        Nov-2013,,,,0,,,,0,,
+        Oct-2013,855,65,44,964,1445,88,67,1600,33,167
+        Sep-2013,768,62,55,885,969,95,76,1140,86,523
+        Aug-2013,267,56,33,356,455,78,54,587,87,496
+        Jul-2013,778,88,55,921,1356,122,89,1567,68,
+        Jun-2013,156,55,29,240,244,90,45,379,23,87
+        May-2013,446,96,88,630,752,193,152,1097,51,222
+        Apr-2013,121,39,45,205,167,67,66,300,40,87
+        Mar-2013,95,45,23,163,135,77,44,256,30,67
+        Feb-2013,88,26,22,136,123,65,40,228,,
+        Jan-2013,120,35,22,177,164,54,40,258,33,72
+        2013,3694,567,416,4677,5810,929,673,7412,451,1721
+        Dec-2012,125,55,28,208,188,88,58,334,19,70
+        Nov-2012,101,35,30,166,129,68,55,252,12,51
+        Oct-2012,88,31,25,144,110,92,62,264,22,77
+        Sep-2012,182,88,95,365,453,110,120,683,,
+        Aug-2012,90,35,39,164,120,80,60,260,26,90
+        Jul-2012,241,40,44,325,480,122,95,697,,
+        Jun-2012,85,26,20,131,111,99,59,269,11,100
+        May-2012,90,20,22,132,115,80,53,248,20,105
+        Apr-2012,88,18,20,126,132,89,50,271,25,185
+        Mar-2012,78,22,12,112,220,85,52,357,30,152
+        Feb-2012,91,39,20,150,180,95,77,352,33,104
+        Jan-2012,99,31,21,151,151,85,85,321,41,214
+        2012,1358,440,376,2174,2389,1093,826,4308,239,1148
+        Dec-2011,90,36,29,155,99,92,88,279,48,150
+        Nov-2011,112,42,33,187,120,110,95,325,55,120
+        Oct-2011,161,55,42,258,195,142,101,438,85,
+        Sep-2011,110,42,33,185,132,150,82,364,45,215
+        Aug-2011,155,45,39,239,183,120,96,399,56,
+        Jul-2011,159,56,44,259,199,135,119,453,22,115
+        Jun-2011,155,77,39,271,192,150,112,454,25,102
+        May-2011,102,45,30,177,115,66,85,266,32,155
+        Apr-2011,120,56,35,211,190,97,90,377,49,199
+        Mar-2011,136,55,56,247,215,80,75,370,72,209
+        Feb-2011,119,45,33,197,200,65,60,325,36,250
+        Jan-2011,159,55,45,259,178,95,90,363,65,295
+        2011,1578,609,458,2645,2018,1302,1093,4413,590,1810
+        Dec-2010,89,41,21,151,114,77,80,271,34,
+        Nov-2010,105,43,23,171,155,78,60,293,40,195
+        Oct-2010,120,45,20,185,202,86,75,363,50,250
+        Sep-2010,185,55,33,273,284,90,111,485,78,487
+        Aug-2010,295,77,54,426,508,180,150,838,98,850
+        Jul-2010,396,89,50,535,680,198,165,1043,100,955
+        Jun-2010,204,50,30,284,400,109,111,620,44,600
+        May-2010,275,45,17,337,520,141,57,718,55,660
+        Apr-2010,274,39,15,328,731,133,53,917,48,590
+        Mar-2010,216,50,101,367,419,120,160,699,57,666
+        Feb-2010,211,96,45,352,414,155,115,684,52,661
+        Jan-2010,135,41,20,196,620,120,42,782,54,681
+        2010,2505,671,429,3605,5047,1487,1179,7713,710,6595
+        Dec-2009,306,48,13,367,1137,119,32,1288,48,635
+        Nov-2009,88,22,12,122,332,56,44,432,38,510
+        Oct-2009,343,42,25,410,1275,135,90,1500,38,585
+        Sep-2009,125,38,40,203,533,102,76,711,43,395
+        Aug-2009,393,48,15,456,1592,129,20,1741,52,540
+        Jul-2009,223,40,12,275,975,93,35,1103,41,400
+        Jun-2009,372,45,20,437,960,101,34,1095,47,635
+        May-2009,124,25,6,155,285,55,4,344,40,904
+        Apr-2009,290,41,24,355,640,97,10,747,53,831
+        Mar-2009,185,53,14,252,445,157,45,647,45,650
+        Feb-2009,211,30,17,258,456,61,31,548,61,508
+        Jan-2009,140,24,27,191,300,35,71,406,29,482
+        2009,2800,456,225,3481,8930,1140,492,10562,535,7075
+        Dec-2008,240,58,18,316,609,206,47,862,44,667
+        Nov-2008,297,29,14,340,600,100,28,728,60,875
+        Oct-2008,278,21,18,317,461,61,40,562,44,855
+        Sep-2008,359,55,26,440,704,144,41,889,66,920
+        Aug-2008,383,30,18,431,680,115,64,859,116,1385
+        Jul-2008,387,45,33,465,601,83,63,747,107,
+        Jun-2008,448,40,21,509,685,111,47,843,124,949
+        May-2008,504,32,27,563,857,89,57,1003,170,2421
+        Apr-2008,966,69,38,1073,1745,159,104,2008,355,1270
+        Mar-2008,925,103,54,1082,925,103,54,1082,645,2592
+        Feb-2008,636,65,20,721,705,121,21,847,237,1341
+        Jan-2008,463,56,22,541,656,119,35,810,229,1225
+        2008,5886,603,309,6798,9228,1411,601,11240,2197,14500
+        Dec-2007,480,64,24,568,730,156,51,937,251,1146
+        Nov-2007,537,45,24,606,521,119,64,704,240,2090
+        Oct-2007,758,116,13,887,144,180,39,363,334,1428
+        Sep-2007,840,61,16,917,849,120,50,1019,361,1520
+        Aug-2007,1771,20,65,1856,1558,100,25,1683,472,2027
+        Jul-2007,1652,144,79,1875,1691,169,118,1978,425,2191
+        Jun-2007,1241,191,31,1463,1561,350,70,1981,417,2265
+        May-2007,1951,127,46,2124,2011,215,63,2289,297,2355
+        Apr-2007,1498,128,63,1689,2330,202,107,2639,219,2939
+        Mar-2007,1869,165,44,2078,2719,277,51,3047,481,5664
+        Feb-2007,1646,131,29,1806,2701,147,47,2895,586,1921
+        Jan-2007,1992,55,40,2087,1941,135,85,2161,586,1921
+        2007,16235,1247,474,17956,18756,2170,770,21696,4669,27467
+        Dec-2006,,,,0,,,,0,423,
+        Nov-2006,1847,103,25,1975,1335,151,,1486,,
+        Oct-2006,1289,119,19,1427,1504,184,,1688,,
+        Sep-2006,,,,0,,,,0,,
+        Aug-2006,,,,0,,,,0,,
+        Jul-2006,1850,,,1850,,,,0,,
+        Jun-2006,887,79,43,1009,1588,126,57,1771,93,
+        May-2006,932,95,28,1055,1271,97,57,1425,324,
+        Apr-2006,,,,0,,,,0,,
+        Mar-2006,,,,0,,,,0,,
+        Feb-2006,,,,0,,,,0,,
+        Jan-2006,,,,0,,,,0,,
+        2006,6805,396,115,7316,5698,558,114,6370,840,0
+        Overall,37167,4422,2386,43975,52066,9161,5075,66302,9780,58595
+        ,,,,,,,,,,
+        NB: Months with only police figures are combined police/army tolls,,,,,,,,,,
+    http_version: 
+  recorded_at: Tue, 12 Nov 2013 16:29:07 GMT
+recorded_with: VCR 2.6.0