governator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 745347ec6c0ff7ff406f0625162db6ef4a632906
+  data.tar.gz: 9106f4daff8952ddac44dbb35c34e766a1941f55
+SHA512:
+  metadata.gz: b4567506b6c7d8d6a648308471ba6e754c2348c162f4377087d216221b78dc7b78aba972c084964e9093a963cdf2c042ee9a5f9b2ec8567616994664371adb76
+  data.tar.gz: 898e473b417465ad3b712cfb2169abb4d6b4c644e3f5908541cf317f6f9c72b65f077c7896b9723150abf5fb5094a185ec7ce647c50b9ada64ef2f9a9ab89dcb

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.idea/
+/vendor/

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in governator.gemspec
+gemspec
+
+group :development, :test do
+  gem 'pry'
+  gem 'rspec'
+  gem 'rubocop'
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 M. Simon Borg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Governator
+
+Scraper for the data on US Governors.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'governator'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install governator
+
+## Usage
+
+```
+Governator.scrape
+
+governors = Governator.governors
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/msimonborg/governator.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'governator'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start
+
+# require "irb"
+# IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/governator.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'governator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'governator'
+  spec.version       = Governator::VERSION
+  spec.authors       = ['M. Simon Borg']
+  spec.email         = ['msimonborg@gmail.com']
+
+  spec.summary       = 'Scraper for data on US Governors'
+  spec.description   = 'Scraper for data on US Governors'
+  spec.homepage      = 'https://github.com/msimonborg/governator'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'faraday'
+  spec.add_dependency 'twitter'
+  spec.add_dependency 'nokogiri'
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'rake', '~> 10.0'
+end

data/lib/governator/bio_page.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'governator/page_scraper'
+
+class Governator
+  class BioPage < PageScraper
+    attr_reader :uri
+
+    def initialize(uri)
+      @uri = uri
+      @raw = Nokogiri::HTML(CONN.get(uri).body)
+      check_for_alt_office
+    end
+
+    def check_for_alt_office
+      @alt_office_present = if raw.css('address')[2].to_s.match?(/Phone|Address|Fax/)
+                              true
+                            else
+                              false
+                            end
+    end
+
+    def alt_office_present?
+      @alt_office_present
+    end
+
+    def website
+      @_website ||= raw.css('.ullist-wrap a').first['href']
+    end
+
+    def party_panel
+      @_party_panel ||= if alt_office_present?
+                          raw.css('address')[3]
+                        else
+                          raw.css('address')[2]
+                        end
+    end
+
+    def party_paragraph
+      @_party_paragraph ||= party_panel.css('p').detect do |p|
+        p.text.include?('Party')
+      end
+    end
+
+    def party
+      @_party ||= party_paragraph.text.delete("\t\n#{nbsp}").sub('Party:', '')
+    end
+
+    def address_panel
+      @_address_panel ||= raw.css('address')[1]
+    end
+
+    def address_array
+      @_address_array ||=
+        address_panel.css('p').first.text.delete("\t\n").sub('Address:', '').split(' ') - [' ']
+    end
+
+    def address
+      @_address ||= address_array.join(' ')
+    end
+
+    def city
+      @_city ||= address_panel.css('p')[1].text.delete(',')
+    end
+
+    def state
+      @_state ||= address_panel.css('p')[2]&.text
+    end
+
+    def zip
+      @_zip ||= address_panel.css('p')[3]&.text
+    end
+
+    def phone
+      @_phone ||= address_panel.css('p')[4].text.delete("\t\nPhone: ").strip.sub('/', '-')
+    end
+
+    def fax
+      @_fax ||= address_panel.css('p')[5]&.text&.delete("\t\nFax:")&.strip&.sub('/', '-')
+    end
+
+    def office_type
+      'capitol'
+    end
+
+    def alt_address_panel
+      @_alt_address_panel ||= raw.css('address')[2] if alt_office_present?
+    end
+
+    def alt_address_array
+      return unless alt_office_present?
+      @_alt_address_array ||=
+        alt_address_panel.css('p').first.text.delete("\t\n").sub('Address:', '').split(' ') - [' ']
+    end
+
+    def alt_building
+      @_alt_building ||= alt_address_array[0..7].join(' ') if alt_office_present?
+    end
+
+    def alt_address
+      @alt_address ||= alt_address_array[8..11].join(' ') if alt_office_present?
+    end
+
+    def alt_suite
+      @alt_suite ||= alt_address_array[13..14].join(' ') if alt_office_present?
+    end
+
+    def alt_city
+      @_alt_city ||= alt_address_panel.css('p')[1].text.delete(',') if alt_office_present?
+    end
+
+    def alt_state
+      @_alt_state ||= alt_address_panel.css('p')[2].text if alt_office_present?
+    end
+
+    def alt_zip
+      @_alt_zip ||= alt_address_panel.css('p')[3].text if alt_office_present?
+    end
+
+    def alt_phone
+      return unless alt_office_present?
+      @_alt_phone ||= alt_address_panel.css('p')[4].text.delete("\t\nPhone: ").strip.sub('/', '-')
+    end
+
+    def alt_fax
+      return unless alt_office_present?
+      @_alt_fax ||= alt_address_panel.css('p')[5]&.text&.delete("\t\nFax:")&.strip&.sub('/', '-')
+    end
+
+    def alt_office_type
+      return unless alt_office_present?
+      alt_state == 'DC' ? 'dc' : 'district'
+    end
+  end
+end

data/lib/governator/name_parser.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class Governator
+  class NameParser
+    attr_reader :official_full, :first, :last, :middle, :nickname, :suffix
+
+    def initialize(official_full)
+      @official_full = official_full
+    end
+
+    def parse
+      split_name
+      [first, last, middle, nickname, suffix]
+    end
+
+    def name_array
+      @_name_array ||= official_full.split(' ')
+    end
+
+    def split_name
+      detect_nickname
+      detect_suffix
+      if name_array.length == 2
+        set_first_and_last
+      elsif name_array[0].include?('.') && name_array[1].include?('.')
+        set_name_with_initialed_first
+      elsif name_array.length >= 3
+        set_first_last_and_middle
+      end
+    end
+
+    def set_first_and_last
+      @first = name_array.first
+      @last  = name_array.last
+    end
+
+    def set_name_with_initialed_first
+      @first  = "#{name_array.shift} #{name_array.shift}"
+      @last   = name_array.pop
+      @middle = name_array.pop
+    end
+
+    def set_first_last_and_middle
+      @first  = name_array.shift
+      @last   = name_array.pop
+      @middle = name_array.join(' ')
+    end
+
+    def detect_nickname
+      @nickname = name_array.detect { |name| name.include?('"') }
+      name_array.reject! { |name| name.include?('"') }
+    end
+
+    def detect_suffix
+      @suffix = if name_array[-2].include?(',')
+                  name_array[-2].delete(',')
+                  name_array.pop
+                end
+    end
+  end
+end

data/lib/governator/page_scraper.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class Governator
+  class PageScraper
+    attr_reader :raw
+
+    def initialize(raw)
+      @raw = raw
+    end
+
+    def nbsp
+      @_nbsp ||= Nokogiri::HTML(' ').text
+    end
+  end
+end

data/lib/governator/panel.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'governator/page_scraper'
+
+class Governator
+  class Panel < PageScraper
+    def image
+      @_image ||= raw.css('.governors-img img').first['src']
+    end
+
+    def bio_page
+      @_bio_page ||= raw.css('.governors-state a').first['href']
+    end
+
+    def governor_name
+      @_governor_name ||= raw.css('.governors-state a')
+                             .first
+                             .text
+                             .sub('Governor ', '')
+                             .gsub('  ', ' ')
+    end
+
+    def state
+      state = raw.css('.governors-state h3').first.text
+      case state
+      when 'Northern Mariana Islands' then 'Commonwealth of the ' + state
+      when 'Virgin Islands' then 'United States ' + state
+      else state
+      end
+    end
+  end
+end

data/lib/governator/version.rb

@@ -0,0 +1,3 @@
+class Governator
+  VERSION = '0.1.0'.freeze
+end

data/lib/governator.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'nokogiri'
+require 'twitter'
+
+require 'governator/bio_page'
+require 'governator/name_parser'
+require 'governator/panel'
+require 'governator/version'
+
+class Governator
+  BASE_URI = 'https://www.nga.org'.freeze
+  CONN = Faraday.new(url: BASE_URI)
+
+  def self.governors
+    @_governors ||= []
+  end
+
+  def self.index_page
+    @_index_page ||= Nokogiri::HTML(CONN.get('/cms/governors/bios').body)
+  end
+
+  def self.panels
+    @_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
+      Panel.new(panel)
+    end
+  end
+
+  def self.scrape!
+    governors.clear
+    panels.each do |panel|
+      governor = create(panel)
+      puts "Scraped #{governor.official_full} of #{governor.state_name}"
+    end
+    
+    governors
+  end
+
+  def self.create(panel)
+    new(panel).tap do |g|
+      g.build
+      g.save
+    end
+  end
+
+  def self.to_json
+    governors.map(&:to_h)
+  end
+
+  attr_reader :panel, :state_name, :bio_page, :official_full, :first, :last,
+              :middle, :nickname, :suffix, :url, :party, :office_locations
+
+  def initialize(panel)
+    @panel = panel
+  end
+
+  def build
+    @bio_page      = BioPage.new(panel.bio_page)
+    @state_name    = panel.state
+    @official_full = panel.governor_name
+    @url           = bio_page.website
+    @party         = bio_page.party
+
+    @first, @last, @middle, @nickname, @suffix = NameParser.new(official_full).parse
+    build_office_locations
+    self
+  end
+
+  def photo_url
+    @_photo_url ||= "#{BASE_URI}#{panel.image}"
+  end
+
+  def build_office_locations
+    @office_locations = [primary_office]
+    @office_locations << secondary_office if bio_page.alt_office_present?
+  end
+
+  def primary_office
+    { address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
+      phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
+  end
+
+  def secondary_office
+    { address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
+      zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
+      office_type: bio_page.alt_office_type }
+  end
+
+  def save
+    self.class.governors << self
+    self
+  end
+
+  def to_h
+    {
+      photo_url: photo_url,
+      state_name: state_name,
+      official_full: official_full,
+      url: url,
+      party: party,
+      office_locations: office_locations
+    }
+  end
+
+  def inspect
+    "#<Governator panel=#{panel} photo_url=\"#{photo_url}\" "\
+      "state_name=\"#{state_name}\" bio_page=#{bio_page} "\
+      "official_full=\"#{official_full}\" url=\"#{url}\" "\
+      "party=\"#{party}\" office_locations=\"#{office_locations}\">"
+  end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: governator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- M. Simon Borg
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Scraper for data on US Governors
+email:
+- msimonborg@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- governator.gemspec
+- lib/governator.rb
+- lib/governator/bio_page.rb
+- lib/governator/name_parser.rb
+- lib/governator/page_scraper.rb
+- lib/governator/panel.rb
+- lib/governator/version.rb
+homepage: https://github.com/msimonborg/governator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Scraper for data on US Governors
+test_files: []