my_representatives 0.0.2

data/lib/my_representatives/wa/csv_upper.rb

@@ -0,0 +1,131 @@
+module MyRepresentatives
+  module WA
+    class CSVUpper
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :xls_url, :xls_filename, :csv_filename, :people
+
+      def initialize
+        self.xls_url = "http://www.parliament.wa.gov.au/WebCMS/WebCMS.nsf/resources/file-data-for-legislative-council-members/$file/DATA%20FOR%20LEGISLATIVE%20COUNCIL%20MEMBERS%2027042017.xls"
+        self.xls_filename = "tmp/csv_wa_upper.xls"
+        self.csv_filename = "tmp/csv_wa_upper.csv"
+        self.people = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+
+          open(@xls_filename, "wb") do |file|
+            open(@xls_url) do |uri|
+               file.write(uri.read)
+            end
+          end
+
+          excel = Excel2CSV.read(@xls_filename)
+          CSV.open(@csv_filename, "wb") do |csv|
+            excel.each_with_index do |row, index|
+
+              if index == 0  # This exists to fix newlines in header row
+                new_row = []
+
+                row.each do |col|
+                  col.gsub!("\n","")
+                  new_row << col
+                end
+                csv << new_row
+              else
+                csv << row
+              end
+            end
+          end
+          FileUtils.rm_r(@xls_filename)
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            if row["FIRST_NAME"] && row["SURNAME"] && row["REGION"] # Fix for blank rows
+
+              # Setup the electorate
+              electorate_name = find_electorate(row)
+              electorate      = Electorate.new(electorate_name)
+              electorate.state_wa_upper!
+
+              # Setup the Representative (Person)
+              person = Person.new(electorate)
+
+              person.title                      = find_title(row)
+              person.first_name                 = find_first_name(row)
+              person.last_name                  = find_last_name(row)
+              person.preferred_name             = find_preferred_name(row)
+              person.email                      = find_email(row)
+              person.phone                      = find_phone(row)
+              person.party_name                 = find_party(row)
+              person.physical_address           = find_physical_address(row)
+              person.postal_address             = find_postal_address(row)
+              person.gender                     = guess_gender(row["Title"])
+              person.formal_name                = format_formal_name(row,person)
+              person.honorifics                 = "MLC"
+              person.salutation                 = nil
+              person.image_url                  = nil
+              person.homepage_url               = nil
+
+              @people << person
+
+            end
+          end
+        end
+
+        def find_first_name(arr)
+          arr["FIRST_NAME"]
+        end
+
+        def find_last_name(arr)
+          arr["SURNAME"]
+        end
+
+        def find_preferred_name(arr)
+          arr["PREFERRED_NAME"]
+        end
+
+        def find_title(arr)
+          arr["HON"].nil? ? arr["TITLE"] : "Hon"
+        end
+
+        def find_electorate(arr)
+          arr["REGION"]
+        end
+
+        def find_email(arr)
+          nil
+        end
+
+        def find_party(arr)
+          arr["PARTY"]
+        end
+
+        def find_phone(arr)
+          nil
+        end
+
+        def format_formal_name(arr, person)
+          arr["HON"].nil? ? "#{person.title} #{person.first_name} #{person.last_name}" : "Hon #{person.first_name} #{person.last_name}"
+        end
+
+        def find_physical_address(arr)
+          arr["ELEC_ADDRESS"]
+        end
+
+        def find_postal_address(arr)
+          arr["ELEC_ADDRESS_MAILING"]
+        end
+
+    end # CSVUpper
+  end # WA
+end # MyRepresentatives

data/lib/my_representatives/wa/merge_lower.rb

@@ -0,0 +1,41 @@
+module MyRepresentatives
+  module WA
+    class MergeLower
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::WA::CSVLower.new.people
+        @web_people   = MyRepresentatives::WA::WebLower.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            index = @web_people.find_index {|wp| wp.email == person.email}
+
+            if !index
+              index = @web_people.find_index {|wp| wp.first_name == person.first_name && wp.last_name == person.last_name }
+            end
+
+            if !index
+              index = @web_people.find_index {|wp| wp.preferred_name == person.first_name && wp.last_name == person.last_name }
+            end
+
+            if index
+              person.phone            = @web_people[index].phone
+              person.party_name       = @web_people[index].party_name
+              person.preferred_name   = @web_people[index].preferred_name
+              person.homepage_url     = @web_people[index].homepage_url
+              person.image_url        = @web_people[index].image_url
+              person.successful_merge = true
+            end
+            @people << person if person
+          end
+        end
+
+    end # MergeLower
+  end # WA
+end # MyRepresentatives

data/lib/my_representatives/wa/merge_upper.rb

@@ -0,0 +1,37 @@
+module MyRepresentatives
+  module WA
+    class MergeUpper
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::WA::CSVUpper.new.people
+        @web_people   = MyRepresentatives::WA::WebUpper.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            index = @web_people.find_index {|wp| wp.first_name == person.first_name && wp.last_name == person.last_name }
+
+            if !index
+              index = @web_people.find_index {|wp| wp.preferred_name == person.first_name && wp.last_name == person.last_name }
+            end
+
+            if index
+              person.phone            = @web_people[index].phone
+              person.party_name       = @web_people[index].party_name
+              person.preferred_name   = @web_people[index].preferred_name
+              person.homepage_url     = @web_people[index].homepage_url
+              person.image_url        = @web_people[index].image_url
+              person.successful_merge = true
+            end
+            @people << person if person
+          end
+        end
+
+    end # MergeUpper
+  end # WA
+end # MyRepresentatives

data/lib/my_representatives/wa/web_index.rb

@@ -0,0 +1,45 @@
+module MyRepresentatives
+  module WA
+    class WebIndex
+      attr_accessor :index_url, :document, :upper_urls, :lower_urls
+
+      def initialize
+        @logger       = Logger.new(STDOUT)
+        @index_url    = nil
+        @document     = nil
+        @lower_urls  = []
+        @upper_urls = []
+
+        @index_url = "http://www.parliament.wa.gov.au/parliament/memblist.nsf/WAllMembers"
+        @document  = find_representatives
+        representative_urls_from_document
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+        def representative_urls_from_document
+          begin
+            links = @document.css('//a[@href*="WAllMembersFlat"]')
+            links.map do |link|
+              url = 'http://www.parliament.wa.gov.au'+link['href'].gsub("\\","/")
+              @lower_urls << url if link.parent.text.include?("MLA") && !@lower_urls.include?(url)
+              @upper_urls << url if link.parent.text.include?("MLC") && !@upper_urls.include?(url)
+            end
+          rescue
+          end
+        end
+
+    end # WebIndex
+  end # WA
+end # MyRepresentatives

data/lib/my_representatives/wa/web_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module WA
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.lower_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = MyRepresentatives::Electorate.new(electorate_name)
+          electorate.state_wa_lower!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.formal_name                = document.formal_name
+          person.first_name                 = document.first_name
+          person.last_name                  = document.last_name
+          person.title                      = document.title
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.preferred_name             = document.preferred_name
+          person.gender                     = guess_gender(document.formal_name)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # WA
+end # MyRepresentatives

data/lib/my_representatives/wa/web_show.rb

@@ -0,0 +1,195 @@
+module MyRepresentatives
+  module WA
+    class WebShow
+      include MyRepresentatives::Guessable
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          @document.css("div#content").css("font")[0].text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def title
+        begin
+          guess_title(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def first_name
+        begin
+          guess_first(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def last_name
+        begin
+          guess_last(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def preferred_name
+        begin
+          guess_preferred(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          regex = /Electorate:(.+?)Inaugural/
+          alt_regex = /Electorate:(.+?)Speeches/
+
+          text = @document.css("font[size='2']").text.strip
+          match = text.match(regex)
+          alt_match = text.match(alt_regex)
+
+          if match
+            match[1]
+          elsif alt_match
+            alt_match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+          regex = /Party:(.+?)Electorate/
+          alt_regex = /Party:(.+?)Inaugural/
+
+          text = @document.css("font[size='2']").text.strip
+          match = text.match(regex)
+          alt_match = text.match(alt_regex)
+
+          if match
+            match[1]
+          elsif alt_match
+            alt_match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          regex = /Email:\s(.+?wa.gov.au)/
+
+          text = @document.css("font[size='2']").text.strip
+          match = text.match(regex)
+
+          if match
+            match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def image_url
+        begin
+          uri = nil
+          @document.css("div#content").css("img").each do |img|
+            link = img.attr("src")
+            uri = link if link.include?('(MemberPics)')
+            break if link.include?('(MemberPics)')
+          end
+
+          if uri
+            "http://www.parliament.wa.gov.au/Parliament/Memblist.nsf/#{uri.gsub("../","")}"
+          else
+            nil
+          end
+
+        rescue
+          nil
+        end
+      end
+
+      def phone
+        begin
+          regex1 = /Ph:\s([0-9]{4} [0-9]{4})Email/
+          regex2 = /Ph:\s(08 [0-9]{4} [0-9]{4})Email/
+          regex3 = /Ph:\s([0-9]{4} [0-9]{3} [0-9]{3})Email/
+          regex4 = /Ph:\s(.+?)$/
+          regex5 = /Ph:\s(.+?)Email/m
+
+          text = @document.css("font[size='2']").text.strip
+          match1 = text.match(regex1)
+          match2 = text.match(regex2)
+          match3 = text.match(regex3)
+          match4 = text.match(regex4)
+          match5 = text.match(regex5)
+
+          if match1
+            match = match1[1].gsub(/\D/, '')
+            match.start_with?("08") ? match : "08"+match
+          elsif match2
+            match = match2[1].gsub(/\D/, '')
+            match.start_with?("08") ? match : "08"+match
+          elsif match3
+            match = match1[3].gsub(/\D/, '')
+            match3
+          elsif match4
+            match = match4[1].gsub(/\D/, '')
+            match.start_with?("08") ? match : "08"+match
+          elsif match5
+            match = match5[1].gsub(/\D/, '')
+            match.start_with?("08") ? match : "08"+match
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+        end
+
+    end # WebShow
+  end # WA
+end # MyRepresentatives

data/lib/my_representatives/wa/web_upper.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module WA
+    class WebUpper
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.upper_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = MyRepresentatives::Electorate.new(electorate_name)
+          electorate.state_wa_upper!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.formal_name                = document.formal_name
+          person.first_name                 = document.first_name
+          person.last_name                  = document.last_name
+          person.title                      = document.title
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.preferred_name             = document.preferred_name
+          person.gender                     = guess_gender(document.formal_name)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebUpper
+  end # WA
+end # MyRepresentatives