my_representatives 0.0.2

data/lib/my_representatives/nsw/web_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module NSW
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.lower_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_nsw_lower!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.formal_name                = document.formal_name
+          person.first_name                 = document.first_name
+          person.last_name                  = document.last_name
+          person.preferred_name             = document.preferred_name
+          person.title                      = nil
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.gender                     = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # NSW
+end # MyRepresentatives

data/lib/my_representatives/nsw/web_show.rb

@@ -0,0 +1,140 @@
+module MyRepresentatives
+  module NSW
+    class WebShow
+      include MyRepresentatives::Guessable
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          unformatted_name.split(",")[0].titleize
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def first_name
+        begin
+          guess_first(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def preferred_name
+        begin
+          guess_preferred(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def last_name
+        begin
+          guess_last(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def phone
+        begin
+          phone = @document.css(".contact_office").css("th.label_cell")
+          index = phone.find_index { |p| p.text == "P" }
+
+          if index
+            phone[index].next_element.text
+          else
+            nil
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          email = @document.css(".contact_office").at('a:contains("@parliament.nsw.gov.au")')
+          if email.is_a?(Array)
+            email.first.attr('href').strip.gsub("mailto:","")
+          else
+            email.attr('href').strip.gsub("mailto:","")
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          electorate = @document.css(".office").css("div.title")[0].text.strip
+          if electorate.include?("Electorate Office")
+            electorate.gsub!(" Electorate Office","")
+          else
+            "All NSW"
+          end
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def image_url
+        begin
+          "https://www.parliament.nsw.gov.au" + @document.css("div.img_position").css("img").attr('src').value.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+          # if @document && @document.css("div.section") && @document.css("div.section").css("img")
+          #   true
+          # else
+          #   @logger.debug("Failed to find a valid document to scrape")
+          #   false
+          # end
+        end
+
+        def unformatted_name
+          begin
+            @document.css("h1.page-title").text.strip
+          rescue NoMethodError
+            nil
+          end
+        end
+
+    end # WebShow
+  end # NSW
+end # MyRepresentatives

data/lib/my_representatives/nsw/web_upper.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module NSW
+    class WebUpper
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.upper_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_nsw_upper!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.formal_name                = document.formal_name
+          person.first_name                 = document.first_name
+          person.last_name                  = document.last_name
+          person.preferred_name             = document.preferred_name
+          person.title                      = nil
+          person.physical_address           = nil
+          person.postal_address             = nil
+          person.honorifics                 = nil
+          person.gender                     = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebUpper
+  end # NSW
+end # MyRepresentatives

data/lib/my_representatives/nt/web_index.rb

@@ -0,0 +1,46 @@
+module MyRepresentatives
+  module NT
+    class WebIndex
+      attr_accessor :index_url, :document, :lower_urls
+
+      def initialize
+        @logger     = Logger.new(STDOUT)
+        @lower_urls = []
+
+        # Lower House
+        @index_url = "https://parliament.nt.gov.au/members-of-parliament/members-by-name"
+        @document  = find_representatives
+        representative_urls_from_document
+
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+        def representative_urls_from_document
+          begin
+            representatives = @document.css(".content-body").css("h3")
+          rescue
+          end
+
+          representatives.each do |row|
+            begin
+              @lower_urls  << row.css("a").attr("href").value.strip
+            rescue
+            end
+          end
+        end
+
+    end # WebIndex
+  end # NT
+end # MyRepresentatives

data/lib/my_representatives/nt/web_lower.rb

@@ -0,0 +1,54 @@
+module MyRepresentatives
+  module NT
+    class WebLower
+      include MyRepresentatives::Abbreviatable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :urls, :people
+
+      def initialize
+        @logger = Logger.new(STDOUT)
+        @urls = WebIndex.new.lower_urls
+        @people = []
+
+        @urls.each do |url|
+          document = WebShow.new(url)
+          create_person_from_document(document)
+        end
+      end
+
+      private
+
+        def create_person_from_document(document)
+          @logger.info("Attempting to create #{document.formal_name}")
+
+          # Setup the electorate
+          electorate_name = document.electorate_name
+          electorate      = Electorate.new(electorate_name)
+          electorate.state_nt_lower!
+
+          # Setup the Representative (Person)
+          person = MyRepresentatives::Person.new(electorate)
+          person.email                      = document.email
+          person.phone                      = document.phone
+          person.party_name                 = document.party_name
+          person.image_url                  = document.image_url
+          person.homepage_url               = document.homepage_url
+          person.formal_name                = document.formal_name
+          person.first_name                 = document.first_name
+          person.last_name                  = document.last_name
+          person.preferred_name             = document.preferred_name
+          person.title                      = document.title
+          person.physical_address           = document.physical_address
+          person.postal_address             = document.postal_address
+          person.gender                     = guess_gender(document.title)
+          person.honorifics                 = nil
+          person.salutation                 = nil
+
+          @people << person
+
+        end
+
+    end # WebLower
+  end # NT
+end # MyRepresentatives

data/lib/my_representatives/nt/web_show.rb

@@ -0,0 +1,134 @@
+module MyRepresentatives
+  module NT
+    class WebShow
+      include MyRepresentatives::Guessable
+
+      attr_accessor :url, :document, :logger
+
+      def initialize(url)
+        @logger = Logger.new(STDOUT)
+        raise MyRepresentatives::InvalidURLError unless url && url.is_a?(String)
+        @url = url
+        fetch_document
+        raise MyRepresentatives::NokogiriDocumentPropertiesError unless test_document_for_validity
+      end
+
+      def homepage_url
+        @url
+      end
+
+      def formal_name
+        begin
+          @document.css("h1#main").text
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def title
+        begin
+          guess_title(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def first_name
+        begin
+          guess_first(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def preferred_name
+        begin
+          guess_preferred(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def last_name
+        begin
+          guess_last(formal_name)
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def phone
+        begin
+          @document.css("table").at('th:contains("Phone (business hours)")').next_element.text.strip.gsub("+61 ","")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def email
+        begin
+          @document.css("table").at('th:contains("Electorate office")').next_element.text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def electorate_name
+        begin
+          @document.css("table").at('th:contains("Electorate")').next_element.text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def party_name
+        begin
+          @document.css("table").at('th:contains("Party")').next_element.text.strip
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def physical_address
+        begin
+          @document.css("table").at('th:contains("Address")').next_element.text.strip.gsub(", ","\n")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def postal_address
+        begin
+          @document.css("table").at('th:contains("Postal address")').next_element.text.strip.gsub(", ","\n")
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      def image_url
+        begin
+          @document.css("div.thumbnail").css("img").attr('src').value
+        rescue NoMethodError
+          nil
+        end
+      end
+
+      private
+
+        def fetch_document
+          begin
+            request   = open(@url)
+            @document = Nokogiri::HTML(request)
+          rescue => err
+            @logger.debug("Failed to connect to the url: #{@url}")
+            raise err
+          end
+        end
+
+        def test_document_for_validity
+          true
+        end
+
+    end # WebShow
+  end # NT
+end # MyRepresentatives

data/lib/my_representatives/person.rb

@@ -0,0 +1,197 @@
+require 'active_support'
+require 'active_support/inflector'
+
+module MyRepresentatives
+  class Person
+    class Error < StandardError; end
+    class IsNotAnElectorateObjectError < Error; end
+
+    include MyRepresentatives::Hashable
+
+    attr_accessor :electorate, :successful_merge
+    attr_reader :formal_name, :first_name, :last_name, :preferred_name, :salutation, :title, :honorifics, :gender, :email, :phone, :physical_address, :postal_address, :image_url, :homepage_url, :party_name
+
+    def initialize(electorate)
+      raise IsNotAnElectorateObjectError unless electorate && electorate.is_a?(Electorate)
+      @electorate       = electorate
+      @successful_merge = false
+    end
+
+    def to_hash
+      super
+    end
+
+    def formal_name=(value)
+      @formal_name = if value
+        if value.strip.empty?
+          nil
+        else
+          value.strip.gsub(/\s{2,}/," ")
+        end
+      else
+        nil
+      end
+    end
+
+    def first_name=(value)
+      @first_name = if value
+        value.strip
+      else
+        nil
+      end
+    end
+
+    def last_name=(value)
+      @last_name = if value
+        value.strip.gsub(" MP", "").gsub(" MLA", "").gsub(" MLC", "").gsub(" QC", "").gsub(" CSC", "").gsub(" AO", "").gsub(" AOM", "")
+      else
+        nil
+      end
+    end
+
+    def preferred_name=(value)
+      @preferred_name = if value
+        value.strip
+      else
+        nil
+      end
+    end
+
+    def salutation=(value)
+      @salutation = if value
+        value.strip
+      else
+        nil
+      end
+    end
+
+    def title=(value)
+      @title = if value
+        if value.strip.empty?
+          nil
+        else
+          value.strip
+        end
+      else
+        nil
+      end
+    end
+
+    def honorifics=(value)
+      @honorifics = if value
+        value.strip
+      else
+        nil
+      end
+    end
+
+    def gender=(value)
+      @gender = if value
+        value.strip.titleize
+      else
+        nil
+      end
+    end
+
+    def email=(value)
+      @email = if value
+        value.downcase.strip.split(" ")[0]
+      else
+        nil
+      end
+    end
+
+    def phone=(value)
+      @phone = if value
+        value.strip.gsub(/\D/, '')
+      else
+        nil
+      end
+    end
+
+    def physical_address=(value)
+      @physical_address = if value && !value.upcase.include?("PO BOX")
+        if value.strip.empty?
+          nil
+        else
+          value.strip.gsub(/[ ]{2,}/," ").gsub("\n\n","\n").gsub("\n ","\n")
+        end
+      else
+        nil
+      end
+    end
+
+    def postal_address=(value)
+      @postal_address = if value
+        if value.strip.empty?
+          nil
+        else
+          value.strip.gsub(/[ ]{2,}/," ").gsub("\n\n","\n").gsub("\n ","\n")
+        end
+      else
+        nil
+      end
+    end
+
+    def image_url=(value)
+      @image_url = if value
+        value.strip
+      else
+        nil
+      end
+    end
+
+    def homepage_url=(value)
+      @homepage_url = if value
+        value.strip
+      else
+        nil
+      end
+    end
+
+    def party_name=(value)
+      @party_name = if value
+        if value.upcase == "LP" || value.upcase == "LIB"
+          "Liberal Party"
+        elsif value.upcase == "ALP" || value == "Australian Labor Party"
+          "Australian Labor Party"
+        elsif value.upcase == "AG"
+          "Australian Greens"
+        elsif value.upcase == "NATS" || value.upcase == "NAT"
+          "The Nationals"
+        elsif value.upcase == "AUS" || value.upcase == "KAP"
+          "Katter's Australian Party"
+        elsif value.upcase.start_with?("IND")
+          "Independent"
+        elsif value.upcase == "NXT"
+          "Nick Xenophon Team"
+        elsif value.upcase == "JLN"
+          "Jacqui Lambie Network"
+        elsif value.upcase == "AC"
+          "Australan Conservatives"
+        elsif value.upcase == "PHON" || value.upcase == "ONP"
+          "Pauline Hanson's One Nation"
+        elsif value.upcase == "DJHP"
+          "Derryn Hinch's Justice Party"
+        elsif value.upcase == "LDP"
+          "Liberal Democrats"
+        elsif value.upcase == "CLP"
+          "Country Liberal Party"
+        elsif value.upcase == "LNP"
+          "Liberal National Party"
+        elsif value.upcase == "GWA"
+          "Greens Western Australia"
+        elsif value.upcase == "SFF"
+          "Shooters, Fisher & Farmers Party"
+        elsif value.upcase == "NPA"
+          "Greens Western Australia"
+        else
+          value.strip
+        end
+      else
+        nil
+      end
+    end
+
+  end
+end