my_representatives 0.0.2

data/lib/my_representatives/electorate.rb

@@ -0,0 +1,146 @@
+module MyRepresentatives
+  class Electorate
+    class Error < StandardError; end
+    class ElectorateNameMissingError < Error; end
+    class StateNameMissingError < Error; end
+    class StateNameInvalidError < Error; end
+
+    include MyRepresentatives::Hashable
+
+    attr_accessor :name, :house, :state, :lower_house, :state_government
+
+    def initialize(name)
+      raise ElectorateNameMissingError unless name && name.is_a?(String)
+      @name = name
+    end
+
+    def to_hash
+      super
+    end
+
+    def commonwealth_lower!(state)
+      raise StateNameMissingError unless state
+      raise StateNameInvalidError unless MyRepresentatives::Static::STATES.include?(state)
+      self.name             = self.name
+      self.house            = "House of Representatives"
+      self.state            = state
+      self.state_government = false
+      self.lower_house      = true
+    end
+
+    def commonwealth_upper!(state)
+      raise StateNameMissingError unless state
+      raise StateNameInvalidError unless MyRepresentatives::Static::STATES.include?(state)
+      self.name             = self.name
+      self.house            = "Senate"
+      self.state            = state
+      self.state_government = false
+      self.lower_house      = false
+    end
+
+    def state_act_lower!
+      self.name             = self.name
+      self.house            = "Legislative Assembly"
+      self.state            = "ACT"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_nsw_lower!
+      self.name             = self.name
+      self.house            = "Legislative Assembly"
+      self.state            = "NSW"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_nsw_upper!
+      self.name             = self.name
+      self.house            = "Legislative Council"
+      self.state            = "NSW"
+      self.state_government = true
+      self.lower_house      = false
+    end
+
+    def state_nt_lower!
+      self.name             = self.name
+      self.house            = "Legislative Assembly"
+      self.state            = "NT"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_qld_lower!
+      self.name             = self.name
+      self.house            = "Legislative Assembly"
+      self.state            = "QLD"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_sa_lower!
+      self.name             = self.name
+      self.house            = "House of Assembly"
+      self.state            = "SA"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_sa_upper!
+      self.name             = self.name
+      self.house            = "Legislative Council"
+      self.state            = "SA"
+      self.state_government = true
+      self.lower_house      = false
+    end
+
+    def state_tas_lower!
+      self.name             = self.name
+      self.house            = "House of Assembly"
+      self.state            = "TAS"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_tas_upper!
+      self.name             = self.name
+      self.house            = "Legislative Council"
+      self.state            = "TAS"
+      self.state_government = true
+      self.lower_house      = false
+    end
+
+    def state_vic_lower!
+      self.name             = self.name
+      self.house            = "Legislative Assembly"
+      self.state            = "VIC"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_vic_upper!
+      self.name             = self.name
+      self.house            = "Legislative Council"
+      self.state            = "VIC"
+      self.state_government = true
+      self.lower_house      = false
+    end
+
+    def state_wa_lower!
+      self.name             = self.name
+      self.house            = "Legislative Assembly"
+      self.state            = "WA"
+      self.state_government = true
+      self.lower_house      = true
+    end
+
+    def state_wa_upper!
+      self.name             = self.name
+      self.house            = "Legislative Council"
+      self.state            = "WA"
+      self.state_government = true
+      self.lower_house      = false
+    end
+
+  end
+end

data/lib/my_representatives/errors.rb

@@ -0,0 +1,7 @@
+module MyRepresentatives
+  class Error < StandardError; end
+  class NokogiriDocumentExpectedError < Error; end
+  class NokogiriDocumentPropertiesError < Error; end
+  class InvalidURLError < Error; end
+  class GenericPhantomJSError < Error; end
+end

data/lib/my_representatives/fileable.rb

@@ -0,0 +1,11 @@
+require 'fileutils'
+
+module MyRepresentatives
+  module Fileable
+
+    def create_tmp
+      FileUtils::mkdir "tmp" unless File.directory?("tmp")
+    end
+
+  end # Fileable
+end # MyRepresentatives

data/lib/my_representatives/guessable.rb

@@ -0,0 +1,120 @@
+module MyRepresentatives
+  module Guessable
+
+    def guess_first(str)
+      name = str
+      name.gsub!(/\((.+?)\)\s/,"")
+
+      if name
+
+        name = if name.start_with?("Hon ")
+          name.gsub("Hon ", "")
+        elsif name.start_with?("The Hon. Dr ")
+          name.gsub("The Hon. Dr ", "")
+        elsif name.start_with?("Reverend the Hon. ")
+          name.gsub("Reverend the Hon. ", "")
+        elsif name.start_with?("Hon. ")
+          name.gsub("Hon. ", "")
+        elsif name.start_with?("Senator the Hon ")
+          name.gsub("Senator the Hon ", "")
+        elsif name.start_with?("The Hon. ")
+          name.gsub("The Hon. ", "")
+        elsif name.start_with?("The Hon ")
+          name.gsub("The Hon ", "")
+        elsif name.start_with?("Senator ")
+          name.gsub("Senator ", "")
+        elsif name.start_with?("Mrs ")
+          name.gsub("Mrs ", "")
+        elsif name.start_with?("Miss ")
+          name.gsub("Miss ", "")
+        elsif name.start_with?("Ms ")
+          name.gsub("Ms ", "")
+        elsif name.start_with?("Mr ")
+          name.gsub("Mr ", "")
+        elsif name.start_with?("Dr ")
+          name.gsub("Dr ", "")
+        else
+          name
+        end
+
+        name = name.split(" ")
+        name[0]
+
+      else
+        nil
+      end
+    end
+
+    def guess_last(str)
+      name = str
+      if name
+        name.gsub(" MLA", "").gsub(" MLC", "").gsub(" MP", "").gsub(" QC", "").gsub(" AO", "").gsub(" AOM", "").gsub(" OAM", "").gsub(" CSC", "").gsub(",","").strip.split(" ").last
+      else
+        nil
+      end
+    end
+    def guess_preferred(str)
+      name = str
+      regex = /\((.+?)\)/
+      match = name.match(regex)
+
+      if match
+        match[1]
+      else
+        nil
+      end
+
+    end
+
+    def guess_gender(str)
+      if str
+        if str == "Mrs"
+          "Female"
+        elsif str == "Miss"
+          "Female"
+        elsif str == "Ms"
+          "Female"
+        elsif str == "Mr"
+          "Male"
+        else
+          nil
+        end
+      else
+        nil
+      end
+    end
+
+    def guess_title(str)
+      name = str
+      if name
+        name = if name.start_with?("Hon ")
+          "Hon"
+        elsif name.start_with?("The Hon")
+          "The Hon"
+        elsif name.start_with?("Senator the Hon ")
+          "Senator the Hon"
+        elsif name.start_with?("Senator ")
+          "Senator"
+        elsif name.start_with?("Mrs ")
+          "Mrs"
+        elsif name.start_with?("Miss ")
+          "Miss"
+        elsif name.start_with?("Ms ")
+          "Ms"
+        elsif name.start_with?("Mr ")
+          "Mr"
+        elsif name.start_with?("Dr ")
+          "Dr"
+        else
+          ""
+        end
+        name
+      else
+        nil
+      end
+    end
+
+
+
+  end # Guessable
+end # MyRepresentatives

data/lib/my_representatives/hashable.rb

@@ -0,0 +1,11 @@
+module MyRepresentatives
+  module Hashable
+
+    def to_hash
+      hash = {}
+      instance_variables.each {|var| hash[var.to_s.delete("@")] = instance_variable_get(var) }
+      hash
+    end
+
+  end
+end

data/lib/my_representatives/nsw/csv_lower.rb

@@ -0,0 +1,115 @@
+module MyRepresentatives
+  module NSW
+    class CSVLower
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        @csv_url = "https://www.parliament.nsw.gov.au/_layouts/15/NSWParliament/memberlistservice.aspx?members=LA&format=Excel"
+        @csv_filename = "tmp/csv_nsw_lower.csv"
+        @people = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name = find_electorate(row)
+            electorate      = Electorate.new(electorate_name)
+            electorate.state_nsw_lower!
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.first_name                 = find_first_name(row)
+            person.last_name                  = find_last_name(row)
+            person.title                      = find_title(row)
+            person.honorifics                 = find_honorifics(row)
+            person.email                      = find_email(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.salutation                 = find_salutation(row)
+            person.formal_name                = format_formal_name(person)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.gender                     = guess_gender(person.title)
+            person.preferred_name             = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+          end
+        end
+
+        def find_first_name(arr)
+          arr["INITIALS"]
+        end
+
+        def find_last_name(arr)
+          arr["SURNAME"]
+        end
+
+        def find_title(arr)
+          arr["TITLE"]
+        end
+
+        def find_honorifics(arr)
+          arr["POST-NOMINALS"]
+        end
+
+        def find_electorate(arr)
+          arr["ELECTORATE"]
+        end
+
+        def find_email(arr)
+          arr["CONTACT ADDRESS EMAIL"]
+        end
+
+        def find_party(arr)
+          arr["PARTY"]
+        end
+
+        def find_phone(arr)
+          arr["CONTACT ADDRESS PHONE"]
+        end
+
+        def find_salutation(arr)
+          arr["SALUTATION"]
+        end
+
+        def format_formal_name(person)
+          "#{person.title} #{person.first_name} #{person.last_name} #{person.honorifics}"
+        end
+
+        def format_physical_address(arr)
+          "#{arr["CONTACT ADDRESS LINE1"]}\n#{arr["CONTACT ADDRESS LINE2"]}\n#{arr["CONTACT ADDRESS LINE3"]}\n #{arr["CONTACT ADDRESS SUBURB"]} #{arr["CONTACT ADDRESS STATE"]} #{arr["CONTACT ADDRESS POSTCODE"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["CONTACT ADDRESS POBOX"]}\n#{arr["CONTACT ADDRESS POBOX SUBURB"]} #{arr["CONTACT ADDRESS POBOX STATE"]} #{arr["CONTACT ADDRESS POBOX POSTCODE"]}"
+        end
+
+    end # CSVLower
+  end # NSW
+end # MyRepresentatives

data/lib/my_representatives/nsw/csv_upper.rb

@@ -0,0 +1,115 @@
+module MyRepresentatives
+  module NSW
+    class CSVUpper
+      include MyRepresentatives::Fileable
+      include MyRepresentatives::Guessable
+
+      attr_accessor :csv_url, :csv_filename, :people
+
+      def initialize
+        @csv_url      = "https://www.parliament.nsw.gov.au/_layouts/15/NSWParliament/memberlistservice.aspx?members=LC&format=Excel"
+        @csv_filename = "tmp/csv_nsw_upper.csv"
+        @people       = []
+
+        csv_from_url
+        people_from_csv
+      end
+
+      private
+
+        def csv_from_url
+          create_tmp
+          begin
+            open(@csv_filename, "wb") do |file|
+              open(@csv_url) do |uri|
+                 file.write(uri.read)
+              end
+            end
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@csv_url}")
+            raise err
+          end
+        end
+
+        def people_from_csv
+          CSV.foreach(@csv_filename, headers: true) do |row|
+
+            # Setup the electorate
+            electorate_name = find_electorate
+            electorate      = Electorate.new(electorate_name)
+            electorate.state_nsw_upper!
+
+            # Setup the Representative (Person)
+            person = Person.new(electorate)
+
+            person.first_name                 = find_first_name(row)
+            person.last_name                  = find_last_name(row)
+            person.title                      = find_title(row)
+            person.honorifics                 = find_honorifics(row)
+            person.email                      = find_email(row)
+            person.phone                      = find_phone(row)
+            person.party_name                 = find_party(row)
+            person.salutation                 = find_salutation(row)
+            person.formal_name                = format_formal_name(person)
+            person.physical_address           = format_physical_address(row)
+            person.postal_address             = format_postal_address(row)
+            person.gender                     = guess_gender(person.title)
+            person.preferred_name             = nil
+            person.image_url                  = nil
+            person.homepage_url               = nil
+
+            @people << person
+          end
+        end
+
+        def find_first_name(arr)
+          arr["INITIALS"]
+        end
+
+        def find_last_name(arr)
+          arr["SURNAME"]
+        end
+
+        def find_title(arr)
+          arr["TITLE"]
+        end
+
+        def find_honorifics(arr)
+          arr["POST-NOMINALS"]
+        end
+
+        def find_electorate
+          "All NSW"
+        end
+
+        def find_email(arr)
+          arr["CONTACT ADDRESS EMAIL"]
+        end
+
+        def find_party(arr)
+          arr["PARTY"]
+        end
+
+        def find_phone(arr)
+          arr["CONTACT ADDRESS PHONE"]
+        end
+
+        def find_salutation(arr)
+          arr["SALUTATION"]
+        end
+
+        def format_formal_name(person)
+          "#{person.title} #{person.first_name} #{person.last_name} #{person.honorifics}"
+        end
+
+        def format_physical_address(arr)
+          "#{arr["CONTACT ADDRESS LINE1"]}\n#{arr["CONTACT ADDRESS LINE2"]}\n#{arr["CONTACT ADDRESS LINE3"]}\n#{arr["CONTACT ADDRESS SUBURB"]} #{arr["CONTACT ADDRESS STATE"]} #{arr["CONTACT ADDRESS POSTCODE"]}"
+        end
+
+        def format_postal_address(arr)
+          "#{arr["CONTACT ADDRESS LINE1"]}\n#{arr["CONTACT ADDRESS LINE2"]}\n#{arr["CONTACT ADDRESS LINE3"]}\n#{arr["CONTACT ADDRESS SUBURB"]} #{arr["CONTACT ADDRESS STATE"]} #{arr["CONTACT ADDRESS POSTCODE"]}"
+        end
+
+    end # CSVUpper
+  end # NSW
+end #MyRepresentatives

data/lib/my_representatives/nsw/merge_lower.rb

@@ -0,0 +1,41 @@
+module MyRepresentatives
+  module NSW
+    class MergeLower
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::NSW::CSVLower.new.people
+        @web_people   = MyRepresentatives::NSW::WebLower.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.phone == person.phone} || @web_people.any? { |wp| wp.email == person.email}
+              index = @web_people.find_index {|wp| wp.phone == person.phone}
+
+              if !index
+                index = @web_people.find_index { |wp| wp.email == person.email }
+              end
+
+              if index
+                person.formal_name      = @web_people[index].formal_name
+                person.first_name       = @web_people[index].first_name
+                person.preferred_name   = @web_people[index].preferred_name
+                person.email            = @web_people[index].email unless person.email
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+    end # MergeLower
+  end # NSW
+end # MyRepresentatives

data/lib/my_representatives/nsw/merge_upper.rb

@@ -0,0 +1,41 @@
+module MyRepresentatives
+  module NSW
+    class MergeUpper
+      attr_accessor :csv_people, :web_people, :people
+
+      def initialize
+        @csv_people   = MyRepresentatives::NSW::CSVUpper.new.people
+        @web_people   = MyRepresentatives::NSW::WebUpper.new.people
+        @people = []
+        check_and_update_person
+      end
+
+      private
+
+        def check_and_update_person
+          @csv_people.each do |person|
+            if @web_people.any? { |wp| wp.phone == person.phone} || @web_people.any? { |wp| wp.email == person.email}
+              index = @web_people.find_index {|wp| wp.phone == person.phone}
+
+              if !index
+                index = @web_people.find_index { |wp| wp.email == person.email }
+              end
+
+              if index
+                person.formal_name      = @web_people[index].formal_name
+                person.first_name       = @web_people[index].first_name
+                person.preferred_name   = @web_people[index].preferred_name
+                person.email            = @web_people[index].email unless person.email
+                person.homepage_url     = @web_people[index].homepage_url
+                person.image_url        = @web_people[index].image_url
+                person.successful_merge = true
+              end
+
+            end
+            @people << person
+          end
+        end
+
+    end # MergeUpper
+  end # NSW
+end # MyRepresentatives

data/lib/my_representatives/nsw/web_index.rb

@@ -0,0 +1,56 @@
+module MyRepresentatives
+  module NSW
+    class WebIndex
+      attr_accessor :index_url, :document, :upper_urls, :lower_urls
+
+      def initialize
+        @logger     = Logger.new(STDOUT)
+        @index_url  = nil
+        @document   = nil
+        @lower_urls = []
+        @upper_urls = []
+
+        # Lower House
+        @index_url = "https://www.parliament.nsw.gov.au/members/pages/all-members.aspx?house=LA"
+        @document  = find_representatives
+        representative_urls_from_document( { lower_house: true } )
+
+        # Upper House
+        @index_url = "https://www.parliament.nsw.gov.au/members/pages/all-members.aspx?house=LC"
+        @document  = find_representatives
+        representative_urls_from_document( { lower_house: false } )
+
+      end
+
+
+      private
+
+        def find_representatives
+          begin
+            request = open(@index_url)
+            Nokogiri::HTML(request)
+          rescue SocketError => err
+            @logger.debug("Unable to connect to #{@index_url}")
+            raise err
+          end
+        end
+
+        def representative_urls_from_document(options = { lower_house: true } )
+          begin
+            representatives = @document.css("table").css("a.prl-name-link.green") if options[:lower_house]
+            representatives = @document.css("table").css("a.prl-name-link.maroon") if !options[:lower_house]
+          rescue
+          end
+
+          representatives.each do |row|
+            begin
+              @lower_urls  << "https://www.parliament.nsw.gov.au" + row.attr("href") if options[:lower_house]
+              @upper_urls << "https://www.parliament.nsw.gov.au" + row.attr("href") if !options[:lower_house]
+            rescue
+            end
+          end
+        end
+
+    end # WebIndex
+  end # NSW
+end # MyRepresentatives