my_representatives 0.0.2

data/lib/my_representatives/vic/csv_lower.rb

@@ -0,0 +1,99 @@
+module MyRepresentatives
+  module VIC
+    class CSVLower
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        @csv_url      = "http://www.parliament.vic.gov.au/members/house/mla?format=csv"
+        @csv_filename = "tmp/csv_vic_lower.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name = find_electorate(row)
+            electorate      = Electorate.new(electorate_name)
+            electorate.state_vic_lower!
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.formal_name                = find_formal_name(row)
+            person.gender                     = find_gender(row)
+            person.email                      = find_email(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.first_name                 = guess_first(person.formal_name)
+            person.last_name                  = guess_last(person.formal_name)
+            person.title                      = guess_title(person.formal_name)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.honorifics                 = "MP"
+            person.preferred_name             = nil
+            person.salutation                 = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+          end
+        end
+
+        def find_formal_name(arr)
+          arr["Name"]
+        end
+
+        def find_electorate(arr)
+          arr["Electorate"]
+        end
+
+        def find_email(arr)
+          arr["Email"]
+        end
+
+        def find_gender(arr)
+          arr["Sex"]
+        end
+
+        def find_party(arr)
+          arr["Party"]
+        end
+
+        def find_phone(arr)
+          arr["Phone"]
+        end
+
+        def format_physical_address(arr)
+          "#{arr["Electorate Office Address line 1"]}\n#{arr["Electorate Office Address line 2"]}\n#{arr["Electorate Office Address line 3"]}\n#{arr["Electorate Office Address line 4"]} #{arr["Electorate Office Address line 5"]} #{arr["Electoral Office Postcode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["PO Address line 1"]}\n#{arr["PO Address line 2"]}\n#{arr["PO Address line 3"]}\n#{arr["PO Address line 4"]} #{arr["PO Address line 5"]}"
+        end
+
+    end # CSVLower
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/csv_upper.rb

@@ -0,0 +1,97 @@
+module MyRepresentatives
+  module VIC
+    class CSVUpper
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        @csv_url      = "http://www.parliament.vic.gov.au/members/house/mlc?format=csv"
+        @csv_filename = "tmp/csv_vic_upper.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+            # Setup the electorate
+            electorate_name = find_electorate(row)
+            electorate      = Electorate.new(electorate_name)
+            electorate.state_vic_upper!
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.formal_name                = find_formal_name(row)
+            person.gender                     = find_gender(row)
+            person.email                      = find_email(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.first_name                 = guess_first(person.formal_name)
+            person.last_name                  = guess_last(person.formal_name)
+            person.title                      = guess_title(person.formal_name)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.honorifics                 = "MLC"
+            person.preferred_name             = nil
+            person.salutation                 = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person unless person.formal_name.downcase.include?("vacant")
+          end
+        end
+
+        def find_formal_name(arr)
+          arr["Name"]
+        end
+
+        def find_electorate(arr)
+          arr["Electorate"]
+        end
+
+        def find_email(arr)
+          arr["Email"]
+        end
+
+        def find_gender(arr)
+          arr["Sex"]
+        end
+
+        def find_party(arr)
+          arr["Party"]
+        end
+
+        def find_phone(arr)
+          arr["Phone"]
+        end
+
+        def format_physical_address(arr)
+          "#{arr["Electorate Office Address line 1"]}\n#{arr["Electorate Office Address line 2"]}\n#{arr["Electorate Office Address line 3"]}\n#{arr["Electorate Office Address line 4"]} #{arr["Electorate Office Address line 5"]} #{arr["Electoral Office Postcode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["PO Address line 1"]}\n#{arr["PO Address line 2"]}\n#{arr["PO Address line 3"]}\n#{arr["PO Address line 4"]} #{arr["PO Address line 5"]}"
+        end
+    end # CSVUpper
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/merge_lower.rb

@@ -0,0 +1,37 @@
+module MyRepresentatives
+  module VIC
+    class MergeLower
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::VIC::CSVLower.new.people
+        @web_people   = MyRepresentatives::VIC::WebLower.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.phone == person.phone} || @web_people.any? { |wp| wp.email == person.email}
+              index = @web_people.find_index {|wp| wp.phone == person.phone}
+
+              if !index
+                index = @web_people.find_index { |wp| wp.email == person.email }
+              end
+
+              if index
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+    end # MergeLower
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/merge_upper.rb

@@ -0,0 +1,37 @@
+module MyRepresentatives
+  module VIC
+    class MergeUpper
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::VIC::CSVUpper.new.people
+        @web_people   = MyRepresentatives::VIC::WebUpper.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.phone == person.phone} || @web_people.any? { |wp| wp.email == person.email}
+              index = @web_people.find_index {|wp| wp.phone == person.phone}
+
+              if !index
+                index = @web_people.find_index { |wp| wp.email == person.email }
+              end
+
+              if index
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+    end # MergeUpper
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/web_index.rb

@@ -0,0 +1,58 @@
+module MyRepresentatives
+  module VIC
+    class WebIndex
+      attr_accessor :index_url, :document, :upper_urls, :lower_urls
+
+      def initialize
+        @logger       = Logger.new(STDOUT)
+        @index_url    = nil
+        @document     = nil
+        @lower_urls  = []
+        @upper_urls = []
+
+        (1..3).each do |page|
+          @index_url = "http://www.parliament.vic.gov.au/members/results?house=Legislative+Assembly&page=#{page}"
+          @document  = find_representatives
+          representative_urls_from_document( { lower_house: true } )
+        end
+
+        (1..2).each do |page|
+          @index_url = "http://www.parliament.vic.gov.au/members/results?house=Legislative+Council&page=#{page}"
+          @document  = find_representatives
+          representative_urls_from_document( { lower_house: false } )
+        end
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+        def representative_urls_from_document(options = { lower_house: true } )
+          begin
+            results = @document.css("div.results-list")
+            representatives = results.css("div.result-item") if results
+          rescue
+          end
+
+          representatives.each do |row|
+            begin
+              uri = "http://www.parliament.vic.gov.au" + row.css("h2").css("a").first.attr("href")
+              @lower_urls  << uri if options[:lower_house] unless uri.include?("vacant")
+              @upper_urls << uri if !options[:lower_house] unless uri.include?("vacant")
+            rescue
+            end
+          end
+        end
+
+    end # WebIndex
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/web_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module VIC
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.lower_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = MyRepresentatives::Electorate.new(electorate_name)
+          electorate.state_vic_lower!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.formal_name                = document.formal_name
+          person.first_name                 = guess_first(document.formal_name)
+          person.last_name                  = guess_last(document.formal_name)
+          person.title                      = guess_title(document.formal_name)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.preferred_name             = document.preferred_name
+          person.gender                     = guess_gender(document.formal_name)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/web_show.rb

@@ -0,0 +1,118 @@
+module MyRepresentatives
+  module VIC
+    class WebShow
+      include MyRepresentatives::Guessable
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          @document.css("div#page-title").css("h2").text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def title
+        begin
+          guess_title(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def first_name
+        begin
+          guess_first(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def last_name
+        begin
+          guess_last(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def preferred_name
+        begin
+          guess_preferred(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          @document.css("dl.member-overview").at('dt:contains("Electorate")').next_element.text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+          @document.css("dl.member-overview").at('dt:contains("Party")').next_element.text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          @document.css("dl.member-overview").at('dt:contains("Email")').next_element.text.strip
+        rescue
+          nil
+        end
+      end
+
+      def image_url
+        begin
+          @document.css("img.details-portrait").attr("src").value
+        rescue
+          nil
+        end
+      end
+
+      def phone
+        begin
+          @document.css("dl.member-overview").at('dt:contains("Phone")').next_element.text.strip
+        rescue
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+        end
+
+    end # WebShow
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/vic/web_upper.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module VIC
+    class WebUpper
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.upper_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = MyRepresentatives::Electorate.new(electorate_name)
+          electorate.state_vic_upper!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.formal_name                = document.formal_name
+          person.first_name                 = guess_first(document.formal_name)
+          person.last_name                  = guess_last(document.formal_name)
+          person.title                      = guess_title(document.formal_name)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.preferred_name             = document.preferred_name
+          person.gender                     = guess_gender(document.formal_name)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebUpper
+  end # VIC
+end # MyRepresentatives

data/lib/my_representatives/wa/csv_lower.rb

@@ -0,0 +1,144 @@
+module MyRepresentatives
+  module WA
+    class CSVLower
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :xls_url, :xls_filename, :csv_filename, :people
+
+      def initialize
+        @xls_url      = "http://www.parliament.wa.gov.au/WebCMS/WebCMS.nsf/resources/file-mla-merge-data/$file/MLA%20Merge%20Data%2026042017.xls"
+        @xls_filename = "tmp/csv_wa_lower.xls"
+        @csv_filename = "tmp/csv_wa_lower.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+
+          open(@xls_filename, "wb") do |file|
+            open(@xls_url) do |uri|
+               file.write(uri.read)
+            end
+          end
+
+          excel = Excel2CSV.read(@xls_filename)
+          CSV.open(@csv_filename, "wb") do |csv|
+            excel.each_with_index do |row, index|
+
+              if index == 0  # This exists to fix duplicate values in the header row
+                renamed_postal_address = false
+                renamed_electorate_address = false
+                renamed_parliament_address = false
+                new_row = []
+
+                row.each do |col|
+                  if col.downcase == "postal address" && renamed_postal_address == false
+                    col = "postal_address_1"
+                    renamed_postal_address = true
+                  elsif col.downcase == "electorate address" && renamed_electorate_address == false
+                    col = "electorate_address_1"
+                    renamed_electorate_address = true
+                  elsif col.downcase == "parliament" && renamed_parliament_address == false
+                    col = "parliament_1"
+                    renamed_parliament_address = true
+                  end
+                  new_row << col
+                end
+                csv << new_row
+              else
+                csv << row
+              end
+            end
+          end
+          FileUtils.rm_r(@xls_filename)
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            if row["First Name"] && row["Surname"] && row["Electorate"] # Fix for blank rows
+
+              # Setup the electorate
+              electorate_name = find_electorate(row)
+              electorate      = Electorate.new(electorate_name)
+              electorate.state_wa_lower!
+
+              # Setup the Representative (Person)
+              person = Person.new(electorate)
+
+              person.title                      = find_title(row)
+              person.first_name                 = find_first_name(row)
+              person.last_name                  = find_last_name(row)
+              person.email                      = find_email(row)
+              person.phone                      = find_phone(row)
+              person.party_name                 = find_party(row)
+              person.gender                     = guess_gender(row["Title"])
+              person.formal_name                = format_formal_name(row,person)
+              person.physical_address           = format_physical_address(row)
+              person.postal_address             = format_postal_address(row)
+              person.honorifics                 = "MLA"
+              person.preferred_name             = nil
+              person.salutation                 = nil
+              person.image_url                  = nil
+              person.homepage_url               = nil
+
+              @people << person
+
+            end
+          end
+        end
+
+        def find_first_name(arr)
+          arr["First Name"]
+        end
+
+        def find_last_name(arr)
+          arr["Surname"]
+        end
+
+        def find_title(arr)
+          arr["Hon."].nil? ? arr["Title"] : "Hon"
+        end
+
+        def find_electorate(arr)
+
+          if arr && arr["Electorate"]
+            arr["Electorate"].gsub("Member for ","")
+          else
+            nil
+          end
+        end
+
+        def find_email(arr)
+          arr["Member Email"]
+        end
+
+        def find_party(arr)
+          nil
+        end
+
+        def find_phone(arr)
+          nil
+        end
+
+        def format_formal_name(arr, person)
+          arr["Hon."].nil? ? "#{person.title} #{person.first_name} #{person.last_name}" : "Hon #{person.first_name} #{person.last_name}"
+        end
+
+        def format_physical_address(arr)
+          "#{arr["electorate_address_1"]}\n#{arr["Electorate Address"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["postal_address_1"]}\n#{arr["Postal Address"]}"
+        end
+
+    end # CSVLower
+  end # WA
+end # MyRepresentatives