my_representatives 0.0.2

data/lib/my_representatives/commonwealth/csv_lower.rb

@@ -0,0 +1,130 @@
+module MyRepresentatives
+  module Commonwealth
+    class CSVLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people, :logger
+
+      def initialize
+        @logger       = Logger.new(STDOUT)
+        @csv_url      = "http://www.aph.gov.au/~/media/03%20Senators%20and%20Members/Address%20Labels%20and%20CSV%20files/SurnameRepsCSV.csv?la=en"
+        @csv_filename = "tmp/csv_commonwealth_lower.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name  = find_electorate(row)
+            electorate_state = state_abbreviation(find_electorate_state(row))
+            electorate       = Electorate.new(electorate_name)
+            electorate.commonwealth_lower!(electorate_state)
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.first_name                 = find_first_name(row)
+            person.last_name                  = find_last_name(row)
+            person.preferred_name             = find_preferred_name(row)
+            person.title                      = find_title(row)
+            person.honorifics                 = find_honorifics(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.salutation                 = find_salutation(row)
+            person.gender                     = find_gender(row)
+            person.formal_name                = format_formal_name(person)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.email                      = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+          end
+        end
+
+        def find_first_name(arr)
+          arr["First Name"]
+        end
+
+        def find_last_name(arr)
+          arr["Surname"]
+        end
+
+        def find_preferred_name(arr)
+          arr["Preferred Name"]
+        end
+
+        def find_title(arr)
+          arr["Title"]
+        end
+
+        def find_honorifics(arr)
+          arr["Honorific"]
+        end
+
+        def find_electorate(arr)
+          arr["Electorate"]
+        end
+
+        def find_electorate_state(arr)
+          arr["State"]
+        end
+
+        def find_gender(arr)
+          arr["Gender"]
+        end
+
+        def find_party(arr)
+          arr["Political Party"]
+        end
+
+        def find_phone(arr)
+          arr["ElectorateTelephone"] ? arr["ElectorateTelephone"] : arr["Telephone"]
+        end
+
+        def find_salutation(arr)
+          arr["Salutation"]
+        end
+
+        def format_formal_name(person)
+          if person.preferred_name
+            "#{person.title} #{person.preferred_name} #{person.last_name} #{person.honorifics}"
+          else
+            "#{person.title} #{person.first_name} #{person.last_name} #{person.honorifics}"
+          end
+        end
+
+        def format_physical_address(arr)
+          "#{arr["ElectorateAddressLine1"]}\n#{arr["ElectorateAddressLine2"]}\n#{arr["ElectorateSuburb"]} #{arr["ElectorateState"]} #{arr["ElectoratePostCode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["LabelAddress"]}\n#{arr["LabelSuburb"]} #{arr["LabelState"]} #{arr["LabelPostCode"]}"
+        end
+
+    end #CSVLower
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/csv_upper.rb

@@ -0,0 +1,129 @@
+module MyRepresentatives
+  module Commonwealth
+    class CSVUpper
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people, :logger
+
+      def initialize
+        @logger       = Logger.new(STDOUT)
+        @csv_url      = "http://www.aph.gov.au/~/media/03%20Senators%20and%20Members/Address%20Labels%20and%20CSV%20files/allsenel.csv?la=en"
+        @csv_filename = "tmp/csv_commonwealth_upper.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name  = state_abbreviation(find_electorate_state(row))
+            electorate       = Electorate.new(electorate_name)
+            electorate.commonwealth_upper!(electorate_name)
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.first_name                 = find_first_name(row)
+            person.last_name                  = find_last_name(row)
+            person.preferred_name             = find_preferred_name(row)
+            person.title                      = find_title(row)
+            person.honorifics                 = find_honorifics(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.salutation                 = find_salutation(row)
+            person.gender                     = find_gender(row)
+            person.formal_name                = format_formal_name(person)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.email                      = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+          end
+        end
+
+        def find_first_name(arr)
+          arr["First Name"]
+        end
+
+        def find_last_name(arr)
+          arr["Surname"]
+        end
+
+        def find_preferred_name(arr)
+          arr["Preferred Name"]
+        end
+
+        def find_title(arr)
+          arr["Title"]
+        end
+
+        def find_honorifics(arr)
+          arr["Honorifics"]
+        end
+
+        def find_electorate(arr)
+          arr["Electorate"]
+        end
+
+        def find_electorate_state(arr)
+          arr["State"]
+        end
+
+        def find_gender(arr)
+          arr["Gender"]
+        end
+
+        def find_party(arr)
+          arr["Political Party"]
+        end
+
+        def find_phone(arr)
+          arr["ElectorateTelephone"] ? arr["ElectorateTelephone"] : arr["Telephone"]
+        end
+
+        def find_salutation(arr)
+          arr["Salutation"]
+        end
+
+        def format_formal_name(person)
+          if person.preferred_name
+            "#{person.title} #{person.preferred_name} #{person.last_name} #{person.honorifics}"
+          else
+            "#{person.title} #{person.first_name} #{person.last_name} #{person.honorifics}"
+          end
+        end
+
+        def format_physical_address(arr)
+          "#{arr["ElectorateAddressLine1"]}\n#{arr["ElectorateAddressLine2"]}\n#{arr["ElectorateSuburb"]} #{arr["ElectorateState"]} #{arr["ElectoratePostCode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["LabelAddress"]}\n#{arr["LabelSuburb"]} #{arr["LabelState"]} #{arr["LabelPostCode"]}"
+        end
+
+    end #CSVUpper
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/merge_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module Commonwealth
+    class MergeLower
+      attr_accessor :csv_people, :web_people, :email_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::Commonwealth::CSVLower.new.people
+        @web_people   = MyRepresentatives::Commonwealth::WebLower.new.people
+        @email_people = MyRepresentatives::Commonwealth::CSVEmail.new.people
+        @people = []
+        check_and_update_person
+        add_emails_to_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.phone == person.phone} || @web_people.any? { |wp| wp.email == person.email}
+              index = @web_people.find_index {|wp| wp.phone == person.phone}
+
+              if !index
+                index = @web_people.find_index { |wp| wp.email == person.email }
+              end
+
+              if index
+                person.email            = @web_people[index].email
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+        def add_emails_to_person
+          @people.each do |person|
+            if @email_people.any? { |ep| ep.phone == person.phone} && person.email.nil?
+              index = @email_people.find_index {|ep| ep.phone == person.phone}
+
+              if index
+                person.email        = @email_people[index].email
+              end
+
+            end
+          end
+        end
+
+
+    end # MergeLower
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/merge_upper.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module Commonwealth
+    class MergeUpper
+      attr_accessor :csv_people, :web_people, :email_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::Commonwealth::CSVUpper.new.people
+        @web_people   = MyRepresentatives::Commonwealth::WebUpper.new.people
+        @email_people = MyRepresentatives::Commonwealth::CSVEmail.new.people
+        @people = []
+        check_and_update_person
+        add_emails_to_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.phone == person.phone} || @web_people.any? { |wp| wp.email == person.email}
+              index = @web_people.find_index {|wp| wp.phone == person.phone}
+
+              if !index
+                index = @web_people.find_index { |wp| wp.email == person.email }
+              end
+
+              if index
+                person.email            = @web_people[index].email
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+        def add_emails_to_person
+          @people.each do |person|
+            if @email_people.any? { |ep| ep.phone == person.phone} && person.email.nil?
+              index = @email_people.find_index {|ep| ep.phone == person.phone}
+
+              if index
+                person.email        = @email_people[index].email
+              end
+
+            end
+          end
+        end
+
+
+    end # MergeLower
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/web_index.rb

@@ -0,0 +1,57 @@
+module MyRepresentatives
+  module Commonwealth
+    class WebIndex
+      attr_accessor :index_url, :document, :senator_urls, :member_urls
+
+      def initialize
+        @logger       = Logger.new(STDOUT)
+        @index_url    = nil
+        @document     = nil
+        @member_urls  = []
+        @senator_urls = []
+
+        (1..13).each do |page|
+          @index_url = "http://www.aph.gov.au/Senators_and_Members/Parliamentarian_Search_Results?page=#{page}&expand=1&q=&mem=1&par=-1&gen=0&ps=12&st=1"
+          @document  = find_representatives
+          representative_urls_from_document( { lower_house: true } )
+        end
+
+        (1..7).each do |page|
+          @index_url = "http://www.aph.gov.au/Senators_and_Members/Parliamentarian_Search_Results?page=#{page}&expand=1&q=&sen=1&par=-1&gen=0&ps=12&st=1"
+          @document  = find_representatives
+          representative_urls_from_document( { lower_house: false } )
+        end
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+        def representative_urls_from_document(options = { lower_house: true } )
+          begin
+            results = @document.css("div.search-filter-results")
+            representatives = results.css("div.row") if results
+          rescue
+          end
+
+          representatives.each do |row|
+            begin
+              @member_urls  << "http://www.aph.gov.au" + row.css("h4.title").css("a").first.attr("href") if options[:lower_house]
+              @senator_urls << "http://www.aph.gov.au" + row.css("h4.title").css("a").first.attr("href") if !options[:lower_house]
+            rescue
+            end
+          end
+        end
+
+    end # WebIndex
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/web_lower.rb

@@ -0,0 +1,55 @@
+module MyRepresentatives
+  module Commonwealth
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.member_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          state_name      = state_abbreviation(document.state_name)
+          electorate      = MyRepresentatives::Electorate.new(electorate_name)
+          electorate.commonwealth_lower!(state_name)
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.first_name                 = guess_first(document.formal_name)
+          person.last_name                  = guess_last(document.formal_name)
+          person.title                      = guess_title(document.formal_name)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.formal_name                = document.formal_name
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.gender                     = nil
+          person.preferred_name             = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/web_show.rb

@@ -0,0 +1,99 @@
+module MyRepresentatives
+  module Commonwealth
+    class WebShow
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          @document.css("h1").text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+          @document.css(".medium-7").at('dt:contains("Party")').next_element.text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def phone
+        begin
+          @document.css("#panel21").at('dt:contains("Telephone")').next_element.text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          @document.css("#panel31").at('dt:contains("Email")').next_element.text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          @document.css(".medium-7").css("h3").text.split(",")[0].strip.gsub("Member for ","").gsub("Senator for ","").gsub(/\s{2,}/,"")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def state_name
+        begin
+          @document.css(".medium-7").css("h3").text.split(",")[1].strip.gsub("Member for ","").gsub("Senator for ","").gsub(/\s{2,}/,"")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def image_url
+        begin
+          @document.css(".medium-3").css("img").attr('src').value.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+          # if @document && @document.css("div.section") && @document.css("div.section").css("img")
+          #   true
+          # else
+          #   @logger.debug("Failed to find a valid document to scrape")
+          #   false
+          # end
+        end
+
+    end # WebShow
+  end # Commonwealth
+end # MyRepresentatives

data/lib/my_representatives/commonwealth/web_upper.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module Commonwealth
+    class WebUpper
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.senator_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          state_name      = state_abbreviation(document.electorate_name)
+          electorate      = MyRepresentatives::Electorate.new(electorate_name)
+          electorate.commonwealth_upper!(state_name)
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.first_name                 = guess_first(document.formal_name)
+          person.last_name                  = guess_last(document.formal_name)
+          person.title                      = guess_title(document.formal_name)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.formal_name                = document.formal_name
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.gender                     = nil
+          person.preferred_name             = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebUpper
+  end # Commonwealth
+end # MyRepresentatives