my_representatives 0.0.2

data/lib/my_representatives/qld/csv_lower.rb

@@ -0,0 +1,117 @@
+require 'excel2csv'
+
+module MyRepresentatives
+  module QLD
+    class CSVLower
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :xls_url, :csv_filename, :xls_filename, :people
+
+      def initialize
+        self.xls_url = "https://www.parliament.qld.gov.au/documents/Members/mailingLists/MEMMERGEEXCEL.xls"
+        self.xls_filename = "tmp/csv_qld_lower.xls"
+        self.csv_filename = "tmp/csv_qld_lower.csv"
+        self.people = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      def csv_from_url
+        create_tmp
+        open(@xls_filename, "wb") do |file|
+          open(@xls_url) do |uri|
+             file.write(uri.read)
+          end
+        end
+
+        excel = Excel2CSV.read(@xls_filename)
+        CSV.open(@csv_filename, "wb") do |csv|
+          excel.each do |row|
+            csv << row
+          end
+        end
+        FileUtils.rm_r(@xls_filename)
+      end
+
+      def people_from_csv
+        CSV.foreach(@csv_filename, headers: true) do |row|
+
+          # Setup the electorate
+          electorate_name = find_electorate(row)
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_qld_lower!
+
+          # Setup the Representative (Person)
+          person = Person.new(electorate)
+
+          person.title                      = find_title(row)
+          person.first_name                 = find_first_name(row)
+          person.last_name                  = find_last_name(row)
+          person.email                      = find_email(row)
+          person.phone                      = find_phone(row)
+          person.party_name                 = find_party(row)
+          person.formal_name                = format_formal_name(person)
+          person.physical_address           = format_physical_address(row)
+          person.postal_address             = format_postal_address(row)
+          person.gender                     = guess_gender(person.title)
+          person.honorifics                 = "MP"
+          person.preferred_name             = nil
+          person.salutation                 = nil
+          person.image_url                  = nil
+          person.homepage_url               = nil
+
+          @people << person
+        end
+      end
+
+      private
+
+        def find_first_name(arr)
+          arr["first"]
+        end
+
+        def find_last_name(arr)
+          arr["last"]
+        end
+
+        def find_electorate(arr)
+          if arr && arr["electorate"]
+            arr["electorate"].gsub("Member for ","")
+          else
+            nil
+          end
+        end
+
+        def find_party(arr)
+          arr["party"]
+        end
+
+        def find_email(arr)
+          nil
+        end
+
+        def find_phone(arr)
+          nil
+        end
+
+        def find_title(arr)
+          arr["title"]
+        end
+
+        def format_formal_name(person)
+          "#{person.title} #{person.first_name} #{person.last_name}"
+        end
+
+        def format_physical_address(arr)
+          "#{arr["address 1"]}\n#{arr["address 2"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["address 1"]}\n#{arr["address 2"]}"
+        end
+
+    end # CSVLower
+  end # QLD
+end # MyRepresentatives

data/lib/my_representatives/qld/merge_lower.rb

@@ -0,0 +1,37 @@
+module MyRepresentatives
+  module QLD
+    class MergeLower
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::QLD::CSVLower.new.people
+        @web_people   = MyRepresentatives::QLD::WebLower.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.electorate.name == person.electorate.name }
+              index = @web_people.find_index { |wp| wp.electorate.name == person.electorate.name}
+
+              if index
+                person.formal_name      = @web_people[index].formal_name
+                person.first_name       = @web_people[index].first_name
+                person.preferred_name   = @web_people[index].preferred_name
+                person.email            = @web_people[index].email unless person.email
+                person.phone            = @web_people[index].phone unless person.phone
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+            end
+            @people << person
+          end
+        end
+
+    end # MergeLower
+  end # QLD
+end # MyRepresentatives

data/lib/my_representatives/qld/web_index.rb

@@ -0,0 +1,47 @@
+module MyRepresentatives
+  module QLD
+    class WebIndex
+      attr_accessor :index_url, :document, :lower_urls
+
+      def initialize
+        @logger     = Logger.new(STDOUT)
+        @lower_urls = []
+
+        # Lower House
+        @index_url = "https://www.parliament.qld.gov.au/members/current/list"
+        @document  = find_representatives
+        representative_urls_from_document
+
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+
+        def representative_urls_from_document
+          begin
+            representatives = @document.css("select")[1].children
+          rescue
+          end
+
+          representatives.each do |row|
+            begin
+              @lower_urls  << "https://www.parliament.qld.gov.au"+row.attr('value') if row.attr('value').include? "MemberDetails?ID="
+            rescue
+            end
+          end
+        end
+
+    end # WebIndex
+  end # QLD
+end # MyRepresentatives

data/lib/my_representatives/qld/web_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module QLD
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.lower_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_nt_lower!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.formal_name                = document.formal_name
+          person.first_name                 = document.first_name
+          person.last_name                  = document.last_name
+          person.preferred_name             = document.preferred_name
+          person.title                      = document.title
+          person.gender                     = guess_gender(document.title)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # QLD
+end # MyRepresentatives

data/lib/my_representatives/qld/web_show.rb

@@ -0,0 +1,146 @@
+module MyRepresentatives
+  module QLD
+    class WebShow
+      include MyRepresentatives::Guessable
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          @document.css(".member-heading").css("h1").text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def title
+        begin
+          guess_title(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def first_name
+        begin
+          guess_first(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def preferred_name
+        begin
+          guess_preferred(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def last_name
+        begin
+          guess_last(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def phone
+        begin
+          regex = /Phone: (.+?)[Fax|Email]/
+          text = @document.at('strong:contains("Phone:")').parent.text.strip
+          match = text.match(regex)
+          if match
+            match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          regex = /Email: (.+?@parliament.qld.gov.au)/
+          text = @document.at('strong:contains("Email:")').parent.text.strip
+          match = text.match(regex)
+          if match
+            match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          regex = /^\s+([A-Z].+?) -/
+          text = @document.at('strong:contains("Electorate")').parent.text.strip
+          match = text.match(regex)
+          if match
+            match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+          regex = /^\s+([A-Z].+?)$/
+          text = @document.at('strong:contains("Party")').parent.text.strip
+          match = text.match(regex)
+          if match
+            match[1].gsub(/ (\(.+?\))/,"")
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def image_url
+        begin
+          "https://www.parliament.qld.gov.au"+@document.css("div.member-bio").css("img").attr('src').value.gsub("../../..","")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+        end
+
+    end # WebShow
+  end # QLD
+end # MyRepresentatives

data/lib/my_representatives/sa/csv_lower.rb

@@ -0,0 +1,120 @@
+module MyRepresentatives
+  module SA
+    class CSVLower
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        @csv_url      = "http://www2.parliament.sa.gov.au/Internet/DesktopModules/CreateMM.aspx?type=ha"
+        @csv_filename = "tmp/csv_sa_lower.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      def csv_from_url
+        create_tmp
+
+        f = open(@csv_url)
+        doc = Nokogiri::HTML(f)
+
+        CSV.open(@csv_filename, "wb") do |csv|
+          csv << ["title", "first", "last", "electorate", "address", "email"]
+          loops = 0
+
+          doc.css("table").css("tr").each do |row|
+            if loops > 0
+              arr = []
+              row.css("td").each do |cell|
+                arr << cell.text
+              end
+              csv << arr
+            end
+            loops += 1
+          end
+        end
+      end
+
+      def people_from_csv
+        CSV.foreach(@csv_filename, headers: true) do |row|
+
+          # Setup the electorate
+          electorate_name = find_electorate(row)
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_sa_lower!
+
+          # Setup the Representative (Person)
+          person = Person.new(electorate)
+
+          person.title                      = find_title(row)
+          person.first_name                 = find_first_name(row)
+          person.last_name                  = find_last_name(row)
+          person.email                      = find_email(row)
+          person.phone                      = find_phone(row)
+          person.party_name                 = find_party(row)
+          person.gender                     = guess_gender(person.title)
+          person.formal_name                = format_formal_name(person)
+          person.physical_address           = format_physical_address(row)
+          person.postal_address             = format_postal_address(row)
+          person.honorifics                 = "MP"
+          person.preferred_name             = nil
+          person.salutation                 = nil
+          person.image_url                  = nil
+          person.homepage_url               = nil
+
+          @people << person
+        end
+      end
+
+      private
+
+        def find_first_name(arr)
+          arr["first"]
+        end
+
+        def find_last_name(arr)
+          arr["last"]
+        end
+
+        def find_title(arr)
+          arr["title"]
+        end
+
+        def find_electorate(arr)
+          if arr && arr["electorate"]
+            arr["electorate"].gsub("Member for ","")
+          else
+            nil
+          end
+        end
+
+        def find_email(arr)
+          arr["email"]
+        end
+
+        def find_party(arr)
+          nil
+        end
+
+        def find_phone(arr)
+          nil
+        end
+
+        def format_formal_name(person)
+          "#{person.title} #{person.first_name} #{person.last_name}"
+        end
+
+        def format_physical_address(arr)
+          "#{arr["address"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["address"]}"
+        end
+
+    end # CSVLower
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/sa/csv_upper.rb

@@ -0,0 +1,115 @@
+module MyRepresentatives
+  module SA
+    class CSVUpper
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        @csv_url      = "http://www2.parliament.sa.gov.au/Internet/DesktopModules/CreateMM.aspx?type=lc"
+        @csv_filename = "tmp/csv_sa_upper.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      def csv_from_url
+        create_tmp
+        f = open(@csv_url)
+        doc = Nokogiri::HTML(f)
+
+        CSV.open(@csv_filename, "wb") do |csv|
+          csv << ["title", "first", "last", "address", "city", "state", "postcode"]
+          loops = 0
+
+          doc.css("table").css("tr").each do |row|
+            if loops > 0
+              arr = []
+              row.css("td").each do |cell|
+                arr << cell.text
+              end
+              csv << arr
+            end
+            loops += 1
+          end
+        end
+      end
+
+      def people_from_csv
+        CSV.foreach(@csv_filename, headers: true) do |row|
+
+          # Setup the electorate
+          electorate_name = find_electorate
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_sa_upper!
+
+          # Setup the Representative (Person)
+          person = Person.new(electorate)
+
+          person.title                      = find_title(row)
+          person.first_name                 = find_first_name(row)
+          person.last_name                  = find_last_name(row)
+          person.email                      = find_email(row)
+          person.phone                      = find_phone(row)
+          person.party_name                 = find_party(row)
+          person.gender                     = guess_gender(person.title)
+          person.formal_name                = format_formal_name(person)
+          person.physical_address           = format_physical_address(row)
+          person.postal_address             = format_postal_address(row)
+          person.honorifics                 = "MLC"
+          person.preferred_name             = nil
+          person.salutation                 = nil
+          person.image_url                  = nil
+          person.homepage_url               = nil
+
+          @people << person
+        end
+      end
+
+      private
+
+        def find_first_name(arr)
+          arr["first"]
+        end
+
+        def find_last_name(arr)
+          arr["last"]
+        end
+
+        def find_title(arr)
+          arr["title"]
+        end
+
+        def find_electorate
+          "All SA"
+        end
+
+        def find_email(arr)
+          arr["email"]
+        end
+
+        def find_party(arr)
+          nil
+        end
+
+        def find_phone(arr)
+          nil
+        end
+
+        def format_formal_name(person)
+          "#{person.title} #{person.first_name} #{person.last_name}"
+        end
+
+        def format_physical_address(arr)
+          "#{arr["address"]}\n#{arr["city"]} #{arr["state"]} #{arr["postcode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["address"]}\n#{arr["city"]} #{arr["state"]} #{arr["postcode"]}"
+        end
+
+    end # CSVUpper
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/sa/merge_lower.rb

@@ -0,0 +1,39 @@
+module MyRepresentatives
+  module SA
+    class MergeLower
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::SA::CSVLower.new.people
+        @web_people   = MyRepresentatives::SA::WebLower.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.electorate.name == person.electorate.name }
+              index = @web_people.find_index { |wp| wp.electorate.name == person.electorate.name}
+
+              if index
+                person.formal_name      = @web_people[index].formal_name
+                person.first_name       = @web_people[index].first_name
+                person.preferred_name   = @web_people[index].preferred_name
+                person.party_name       = @web_people[index].party_name
+                person.email            = @web_people[index].email unless person.email
+                person.phone            = @web_people[index].phone unless person.phone
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+    end # MergeLower
+  end # SA
+end # MyRepresentatives