my_representatives 0.0.2

data/lib/my_representatives/sa/merge_upper.rb

@@ -0,0 +1,39 @@
+module MyRepresentatives
+  module SA
+    class MergeUpper
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::SA::CSVUpper.new.people
+        @web_people   = MyRepresentatives::SA::WebUpper.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.last_name == person.last_name }
+              index = @web_people.find_index { |wp| wp.last_name == person.last_name}
+
+              if index
+                person.formal_name      = @web_people[index].formal_name
+                person.first_name       = @web_people[index].first_name
+                person.preferred_name   = @web_people[index].preferred_name
+                person.party_name       = @web_people[index].party_name
+                person.email            = @web_people[index].email unless person.email
+                person.phone            = @web_people[index].phone unless person.phone
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+    end # MergeUpper
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/sa/web_index_lower.rb

@@ -0,0 +1,42 @@
+module MyRepresentatives
+  module SA
+    class WebIndexLower
+      attr_accessor :index_url, :document, :lower_urls
+
+      def initialize
+        @logger     = Logger.new(STDOUT)
+        @lower_urls = []
+
+        # Lower House
+        @index_url = "http://www2.parliament.sa.gov.au/Internet/DesktopModules/memberlist.aspx?sort=name&house=HA"
+        @document  = find_representatives
+        representative_urls_from_document
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+
+        def representative_urls_from_document
+          begin
+            @document.css("table#list").css("a").each do |link|
+              uri = link.attr("href")
+              @lower_urls << "http://www2.parliament.sa.gov.au/Internet/DesktopModules/#{uri}" if uri.include?("MemberDrill.aspx?pid=")
+            end
+          rescue
+          end
+        end
+
+    end # WebIndexLower
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/sa/web_index_upper.rb

@@ -0,0 +1,43 @@
+module MyRepresentatives
+  module SA
+    class WebIndexUpper
+      attr_accessor :index_url, :document, :upper_urls
+
+      def initialize
+        @logger     = Logger.new(STDOUT)
+        @upper_urls = []
+
+        # Upper House
+        @index_url = "http://www2.parliament.sa.gov.au/Internet/DesktopModules/memberlist.aspx?sort=name&house=LC"
+        @document  = find_representatives
+        representative_urls_from_document
+
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+
+        def representative_urls_from_document
+          begin
+            @document.css("table#list").css("a").each do |link|
+              uri = link.attr("href")
+              @upper_urls << "http://www2.parliament.sa.gov.au/Internet/DesktopModules/#{uri}" if uri.include?("MemberDrill.aspx?pid=")
+            end
+          rescue
+          end
+        end
+
+    end # WebIndex
+  end # QLD
+end # MyRepresentatives

data/lib/my_representatives/sa/web_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module SA
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls   = WebIndexLower.new.lower_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_sa_lower!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.formal_name                = document.formal_name
+          person.title                      = document.title
+          person.first_name                 = document.first_name
+          person.preferred_name             = document.preferred_name
+          person.last_name                  = document.last_name
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.gender                     = guess_gender(document.title)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/sa/web_show.rb

@@ -0,0 +1,158 @@
+module MyRepresentatives
+  module SA
+    class WebShow
+      include MyRepresentatives::Guessable
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          @document.title.strip.gsub("Profile: ","")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def title
+        begin
+          guess_title(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def first_name
+        begin
+          guess_first(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def last_name
+        begin
+          guess_last(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def preferred_name
+        begin
+          guess_preferred(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          @document.at('td.boxedtext:contains("Electorate")').next_element.text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+          @document.at('td.boxedtext:contains("Political Party")').next_element.text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          regex = /Email(.+?@.+?.sa.gov.au)/
+          text = @document.text.strip
+          match = text.match(regex)
+          if match
+            match[1]
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def image_url
+        id = homepage_url.split("=")
+        "http://www2.parliament.sa.gov.au/FormerMembers/Common/Handler/PersonImage.ashx?PersonId=#{id.last}"
+      end
+
+      def phone
+        begin
+          phone = @document.at('b:contains("Electorate Telephone:")')
+          alt_phone = @document.at('b:contains("Telephone:")')
+          if phone
+            phone = phone.parent.text
+            regex = /Electorate Telephone:(08 [0-9]{4} [0-9]{4})/
+            alt_regex = /Electorate Telephone:([0-9]{4} [0-9]{4})/
+            match = phone.match(regex)
+            match_alt = phone.match(alt_regex)
+
+            if match
+              match[1]
+            elsif match_alt
+              "08"+match_alt[1]
+            else
+              nil
+            end
+          elsif alt_phone
+            phone = alt_phone.parent.text
+            regex = /Telephone:(08 [0-9]{4} [0-9]{4})/
+            alt_regex = /Telephone:([0-9]{4} [0-9]{4})/
+            last_attempt = /Telephone:(1800 [0-9]{3} [0-9]{3})/
+            match = phone.match(regex)
+            match_alt = phone.match(alt_regex)
+            match_last = phone.match(last_attempt)
+
+            if match
+              match[1]
+            elsif match_alt
+              "08"+match_alt[1]
+            elsif match_last
+              match_last[1]
+            else
+              nil
+            end
+          else
+            nil
+          end
+        rescue
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+        end
+
+    end # WebShow
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/sa/web_upper.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module SA
+    class WebUpper
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls   = WebIndexUpper.new.upper_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = "All SA"
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_sa_upper!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.formal_name                = document.formal_name
+          person.title                      = document.title
+          person.first_name                 = document.first_name
+          person.preferred_name             = document.preferred_name
+          person.last_name                  = document.last_name
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.gender                     = guess_gender(document.title)
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # SA
+end # MyRepresentatives

data/lib/my_representatives/static.rb

@@ -0,0 +1,5 @@
+module MyRepresentatives
+  module Static
+    STATES = ["ACT","NSW","NT","QLD","SA","TAS","VIC","WA"]
+  end
+end

data/lib/my_representatives/tas/csv_lower.rb

@@ -0,0 +1,125 @@
+module MyRepresentatives
+  module TAS
+    class CSVLower
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        # Tasmania's Parliament website has a word document for their House of Assembly and  Legislative Council
+        # At this time, this gem uses a csv manually created by https://github.com/tennantje/.
+        # https://gist.github.com/tennantje/c94bd62340bd078b9b1ae12168f48388
+        # This gem has opted for this approach given that there is also no scrapable data on the Tasmanian Parliament website,
+        # whereas Northern Territory (for example) has no CSV file, but does at least have a scrapable website.
+        @csv_url      = "https://gist.githubusercontent.com/tennantje/c94bd62340bd078b9b1ae12168f48388/raw/"
+        @csv_filename = "tmp/csv_tas_lower.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name = find_electorate(row)
+            electorate      = Electorate.new(electorate_name)
+            electorate.state_tas_lower!
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.title                      = find_title(row)
+            person.first_name                 = find_first_name(row)
+            person.preferred_name             = find_preferred_name(row)
+            person.last_name                  = find_last_name(row)
+            person.email                      = find_email(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.gender                     = find_gender(row)
+            person.honorifics                 = "MP"
+            person.formal_name                = format_formal_name(person)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.salutation                 = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+
+          end
+        end
+
+        def find_first_name(arr)
+          arr["FirstName"]
+        end
+
+        def find_last_name(arr)
+          arr["LastName"]
+        end
+
+        def find_preferred_name(arr)
+          arr["PreferredName"]
+        end
+
+        def find_title(arr)
+          arr["Title"]
+        end
+
+        def find_electorate(arr)
+          arr["Electorate"]
+        end
+
+        def find_email(arr)
+          arr["Email"]
+        end
+
+        def find_party(arr)
+          arr["Party"]
+        end
+
+        def find_phone(arr)
+          arr["Phone"]
+        end
+
+        def find_gender(arr)
+          arr["Gender"]
+        end
+
+        def format_formal_name(person)
+          if person.preferred_name
+            "#{person.title} #{person.preferred_name} #{person.last_name} #{person.honorifics}"
+          else
+            "#{person.title} #{person.first_name} #{person.last_name} #{person.honorifics}"
+          end
+        end
+
+        def format_physical_address(arr)
+          "#{arr["Address1"]}\n#{arr["Address2"]}\n#{arr["City"]}\n#{arr["State"]} #{arr["Postcode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["Address1"]}\n#{arr["Address2"]}\n#{arr["City"]}\n#{arr["State"]} #{arr["Postcode"]}"
+        end
+
+    end # CSVLower
+  end # TAS
+end # MyRepresentatives

data/lib/my_representatives/tas/csv_upper.rb

@@ -0,0 +1,125 @@
+module MyRepresentatives
+  module TAS
+    class CSVUpper
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        # Tasmania's Parliament website has a word document for their House of Assembly and  Legislative Council.
+        # At this time, this gem uses a csv manually created by https://github.com/tennantje/.
+        # https://gist.github.com/tennantje/c662bd7a2ac44d2e5839ad73d6d0a80e
+        # This gem has opted for this approach given that there is also no scrapable data on the Tasmanian Parliament website,
+        # whereas Northern Territory (for example) has no CSV file, but does at least have a scrapable website.
+        @csv_url      = "https://gist.githubusercontent.com/tennantje/c662bd7a2ac44d2e5839ad73d6d0a80e/raw"
+        @csv_filename = "tmp/csv_tas_upper.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name = find_electorate(row)
+            electorate      = Electorate.new(electorate_name)
+            electorate.state_tas_upper!
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.title                      = find_title(row)
+            person.first_name                 = find_first_name(row)
+            person.preferred_name             = find_preferred_name(row)
+            person.last_name                  = find_last_name(row)
+            person.email                      = find_email(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.gender                     = find_gender(row)
+            person.honorifics                 = "MLC"
+            person.formal_name                = format_formal_name(person)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.salutation                 = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+
+          end
+        end
+
+        def find_first_name(arr)
+          arr["FirstName"]
+        end
+
+        def find_last_name(arr)
+          arr["LastName"]
+        end
+
+        def find_preferred_name(arr)
+          arr["PreferredName"]
+        end
+
+        def find_title(arr)
+          arr["Title"]
+        end
+
+        def find_electorate(arr)
+          arr["Electorate"]
+        end
+
+        def find_email(arr)
+          arr["Email"]
+        end
+
+        def find_party(arr)
+          arr["Party"]
+        end
+
+        def find_phone(arr)
+          arr["Phone"]
+        end
+
+        def find_gender(arr)
+          arr["Gender"]
+        end
+
+        def format_formal_name(person)
+          if person.preferred_name
+            "#{person.title} #{person.preferred_name} #{person.last_name} #{person.honorifics}"
+          else
+            "#{person.title} #{person.first_name} #{person.last_name} #{person.honorifics}"
+          end
+        end
+
+        def format_physical_address(arr)
+          "#{arr["Address1"]}\n#{arr["City"]}\nTAS #{arr["Postcode"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["Address1"]}\n#{arr["City"]}\nTAS #{arr["Postcode"]}"
+        end
+
+    end # CSVUpper
+  end # TAS
+end # MyRepresentatives

data/lib/my_representatives/version.rb

@@ -0,0 +1,5 @@
+module MyRepresentatives
+  unless defined?(MyRepresentatives::VERSION)
+    VERSION = '0.0.2'
+  end
+end