governator 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -0
- data/Gemfile +2 -0
- data/README.md +19 -0
- data/Rakefile +2 -0
- data/bin/console +10 -2
- data/governator.gemspec +1 -0
- data/lib/governator/twitter_client.rb +37 -0
- data/lib/governator/version.rb +3 -1
- data/lib/governator.rb +92 -55
- metadata +4 -2
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA1:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: e89dcfc60bccb89d0080d03ca2b0b606192a19d9
         | 
| 4 | 
            +
              data.tar.gz: 322faf0a64187c9b9fda0889a98a9adbe5648e50
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 2600672ed050b4e7431e01c94dd369f494e61a9c93088238f16e05620a8cff1b66df83c27d5a676bb7070a2e83f8f41d1a59de52ea4c0c28db5ef6fc36e01200
         | 
| 7 | 
            +
              data.tar.gz: 2f6d3be6510ae7087b63fa865498d77d06b7b3f03e7cb31a4c1b9d287dd6fc0cc851d9182e66e7e989f90f0aab4c686b0e7997e385bb30c0423a2b45f0fe04fa
         | 
    
        data/.rubocop.yml
    ADDED
    
    
    
        data/Gemfile
    CHANGED
    
    
    
        data/README.md
    CHANGED
    
    | @@ -26,6 +26,25 @@ Governator.scrape! | |
| 26 26 | 
             
            governors = Governator.governors
         | 
| 27 27 | 
             
            ```
         | 
| 28 28 |  | 
| 29 | 
            +
            ## Twitter
         | 
| 30 | 
            +
             | 
| 31 | 
            +
            If you want to scrape Twitter handles you will need to initialize the client. Add this to your code (in a Rails app it should probably go in a file called `governator.rb` in `./config/initializers/`)
         | 
| 32 | 
            +
             | 
| 33 | 
            +
            ```ruby
         | 
| 34 | 
            +
            Governator.config do |config|
         | 
| 35 | 
            +
              config.use_twitter = true
         | 
| 36 | 
            +
              
         | 
| 37 | 
            +
              config.twitter do |twitter|
         | 
| 38 | 
            +
                twitter.consumer_key        = YOUR_CONSUMER_KEY
         | 
| 39 | 
            +
                twitter.consumer_secret     = YOUR_TWITTER_CONSUMER_SECRET
         | 
| 40 | 
            +
                twitter.access_token        = YOUR_ACCESS_TOKEN
         | 
| 41 | 
            +
                twitter.access_token_secret = YOUR_ACCESS_TOKEN_SECRET
         | 
| 42 | 
            +
              end
         | 
| 43 | 
            +
            end
         | 
| 44 | 
            +
            ```
         | 
| 45 | 
            +
             | 
| 46 | 
            +
            As with everything, secrets should never be stored anywhere public, like version control. Set these values as variables on your system.
         | 
| 47 | 
            +
             | 
| 29 48 | 
             
            ## Development
         | 
| 30 49 |  | 
| 31 50 | 
             
            After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
         | 
    
        data/Rakefile
    CHANGED
    
    
    
        data/bin/console
    CHANGED
    
    | @@ -1,10 +1,18 @@ | |
| 1 1 | 
             
            #!/usr/bin/env ruby
         | 
| 2 | 
            +
            # frozen_string_literal: true
         | 
| 2 3 |  | 
| 3 4 | 
             
            require 'bundler/setup'
         | 
| 4 5 | 
             
            require 'governator'
         | 
| 5 6 |  | 
| 6 | 
            -
             | 
| 7 | 
            -
             | 
| 7 | 
            +
            Governator.config do |config|
         | 
| 8 | 
            +
              config.use_twitter = true
         | 
| 9 | 
            +
              config.twitter do |twitter|
         | 
| 10 | 
            +
                twitter.consumer_key        = ENV['TWITTER_CONSUMER_KEY']
         | 
| 11 | 
            +
                twitter.consumer_secret     = ENV['TWITTER_CONSUMER_SECRET']
         | 
| 12 | 
            +
                twitter.access_token        = ENV['ACCESS_TOKEN']
         | 
| 13 | 
            +
                twitter.access_token_secret = ENV['ACCESS_TOKEN_SECRET']
         | 
| 14 | 
            +
              end
         | 
| 15 | 
            +
            end
         | 
| 8 16 |  | 
| 9 17 | 
             
            # (If you use this, don't forget to add pry to your Gemfile!)
         | 
| 10 18 | 
             
            require 'pry'
         | 
    
        data/governator.gemspec
    CHANGED
    
    
| @@ -0,0 +1,37 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            class Governator
         | 
| 4 | 
            +
              class TwitterClient
         | 
| 5 | 
            +
                def self.client(&block)
         | 
| 6 | 
            +
                  @_client ||= Twitter::REST::Client.new(&block)
         | 
| 7 | 
            +
                end
         | 
| 8 | 
            +
             | 
| 9 | 
            +
                def self.governors
         | 
| 10 | 
            +
                  @_governors ||= nga_list_members + rga_list_members + cspan_list_members
         | 
| 11 | 
            +
                end
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                def self.nga_list
         | 
| 14 | 
            +
                  @_nga_list ||= client.lists('NatlGovsAssoc').detect { |l| l.name == 'Governors' }
         | 
| 15 | 
            +
                end
         | 
| 16 | 
            +
             | 
| 17 | 
            +
                def self.nga_list_members
         | 
| 18 | 
            +
                  @_nga_list_members ||= client.list_members(nga_list, count: 100).attrs[:users]
         | 
| 19 | 
            +
                end
         | 
| 20 | 
            +
             | 
| 21 | 
            +
                def self.rga_list
         | 
| 22 | 
            +
                  @_rga_list ||= client.lists('The_RGA').detect { |l| l.name == 'GOP Governors' }
         | 
| 23 | 
            +
                end
         | 
| 24 | 
            +
             | 
| 25 | 
            +
                def self.rga_list_members
         | 
| 26 | 
            +
                  @_rga_list_members ||= client.list_members(rga_list, count: 100).attrs[:users]
         | 
| 27 | 
            +
                end
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                def self.cspan_list
         | 
| 30 | 
            +
                  @_cspan_list ||= client.lists('cspan').detect { |l| l.name == 'Governors' }
         | 
| 31 | 
            +
                end
         | 
| 32 | 
            +
             | 
| 33 | 
            +
                def self.cspan_list_members
         | 
| 34 | 
            +
                  @_cspan_list_members ||= client.list_members(cspan_list, count: 100).attrs[:users]
         | 
| 35 | 
            +
                end
         | 
| 36 | 
            +
              end
         | 
| 37 | 
            +
            end
         | 
    
        data/lib/governator/version.rb
    CHANGED
    
    
    
        data/lib/governator.rb
    CHANGED
    
    | @@ -7,54 +7,110 @@ require 'twitter' | |
| 7 7 | 
             
            require 'governator/bio_page'
         | 
| 8 8 | 
             
            require 'governator/name_parser'
         | 
| 9 9 | 
             
            require 'governator/panel'
         | 
| 10 | 
            +
            require 'governator/twitter_client'
         | 
| 10 11 | 
             
            require 'governator/version'
         | 
| 11 12 |  | 
| 13 | 
            +
            # Governator.scrape!
         | 
| 14 | 
            +
            # governors = Governor.governors
         | 
| 12 15 | 
             
            class Governator
         | 
| 13 | 
            -
              BASE_URI = 'https://www.nga.org' | 
| 16 | 
            +
              BASE_URI = 'https://www.nga.org'
         | 
| 14 17 | 
             
              CONN = Faraday.new(url: BASE_URI)
         | 
| 15 18 |  | 
| 16 | 
            -
               | 
| 17 | 
            -
                 | 
| 18 | 
            -
              end
         | 
| 19 | 
            +
              class << self # <-- Begin class methods
         | 
| 20 | 
            +
                attr_reader :use_twitter
         | 
| 19 21 |  | 
| 20 | 
            -
             | 
| 21 | 
            -
             | 
| 22 | 
            -
             | 
| 22 | 
            +
                def scrape!
         | 
| 23 | 
            +
                  governors.clear
         | 
| 24 | 
            +
                  panels.each do |panel|
         | 
| 25 | 
            +
                    governor = create(panel)
         | 
| 26 | 
            +
                    puts "Scraped #{governor.official_full} of #{governor.state_name}"
         | 
| 27 | 
            +
                  end
         | 
| 23 28 |  | 
| 24 | 
            -
             | 
| 25 | 
            -
                @_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
         | 
| 26 | 
            -
                  Panel.new(panel)
         | 
| 29 | 
            +
                  governors
         | 
| 27 30 | 
             
                end
         | 
| 28 | 
            -
              end
         | 
| 29 31 |  | 
| 30 | 
            -
             | 
| 31 | 
            -
             | 
| 32 | 
            -
                panels.each do |panel|
         | 
| 33 | 
            -
                  governor = create(panel)
         | 
| 34 | 
            -
                  puts "Scraped #{governor.official_full} of #{governor.state_name}"
         | 
| 32 | 
            +
                def governors
         | 
| 33 | 
            +
                  @_governors ||= []
         | 
| 35 34 | 
             
                end
         | 
| 36 35 |  | 
| 37 | 
            -
                 | 
| 38 | 
            -
             | 
| 36 | 
            +
                def to_json
         | 
| 37 | 
            +
                  governors.map(&:to_h)
         | 
| 38 | 
            +
                end
         | 
| 39 | 
            +
                alias serialize to_json
         | 
| 39 40 |  | 
| 40 | 
            -
             | 
| 41 | 
            -
             | 
| 42 | 
            -
                  g.build
         | 
| 43 | 
            -
                  g.save
         | 
| 41 | 
            +
                def config
         | 
| 42 | 
            +
                  yield self
         | 
| 44 43 | 
             
                end
         | 
| 45 | 
            -
              end
         | 
| 46 44 |  | 
| 47 | 
            -
             | 
| 48 | 
            -
             | 
| 49 | 
            -
             | 
| 45 | 
            +
                def twitter(&block)
         | 
| 46 | 
            +
                  TwitterClient.client(&block)
         | 
| 47 | 
            +
                end
         | 
| 48 | 
            +
             | 
| 49 | 
            +
                def use_twitter=(boolean)
         | 
| 50 | 
            +
                  raise ArgumentError, 'value must be Boolean value' unless [true, false].include? boolean
         | 
| 51 | 
            +
                  @use_twitter = boolean
         | 
| 52 | 
            +
                end
         | 
| 53 | 
            +
             | 
| 54 | 
            +
                private # private class methods
         | 
| 55 | 
            +
             | 
| 56 | 
            +
                def index_page
         | 
| 57 | 
            +
                  @_index_page ||= Nokogiri::HTML(CONN.get('/cms/governors/bios').body)
         | 
| 58 | 
            +
                end
         | 
| 59 | 
            +
             | 
| 60 | 
            +
                def panels
         | 
| 61 | 
            +
                  @_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
         | 
| 62 | 
            +
                    Panel.new(panel)
         | 
| 63 | 
            +
                  end
         | 
| 64 | 
            +
                end
         | 
| 65 | 
            +
             | 
| 66 | 
            +
                def create(panel)
         | 
| 67 | 
            +
                  new(panel).tap do |g|
         | 
| 68 | 
            +
                    g.send :build
         | 
| 69 | 
            +
                    g.send :save
         | 
| 70 | 
            +
                  end
         | 
| 71 | 
            +
                end
         | 
| 72 | 
            +
              end # <-- End class methods
         | 
| 50 73 |  | 
| 51 74 | 
             
              attr_reader :panel, :state_name, :bio_page, :official_full, :first, :last,
         | 
| 52 | 
            -
                          :middle, :nickname, :suffix, :url, :party, :office_locations
         | 
| 75 | 
            +
                          :middle, :nickname, :suffix, :url, :party, :office_locations,
         | 
| 76 | 
            +
                          :twitter
         | 
| 53 77 |  | 
| 54 78 | 
             
              def initialize(panel)
         | 
| 55 79 | 
             
                @panel = panel
         | 
| 56 80 | 
             
              end
         | 
| 57 81 |  | 
| 82 | 
            +
              def to_h
         | 
| 83 | 
            +
                {
         | 
| 84 | 
            +
                  photo_url: photo_url,
         | 
| 85 | 
            +
                  state_name: state_name,
         | 
| 86 | 
            +
                  official_full: official_full,
         | 
| 87 | 
            +
                  url: url,
         | 
| 88 | 
            +
                  party: party,
         | 
| 89 | 
            +
                  office_locations: office_locations
         | 
| 90 | 
            +
                }
         | 
| 91 | 
            +
              end
         | 
| 92 | 
            +
             | 
| 93 | 
            +
              def inspect
         | 
| 94 | 
            +
                "#<Governator #{official_full}>"
         | 
| 95 | 
            +
              end
         | 
| 96 | 
            +
             | 
| 97 | 
            +
              def secondary_office
         | 
| 98 | 
            +
                { address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
         | 
| 99 | 
            +
                  zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
         | 
| 100 | 
            +
                  office_type: bio_page.alt_office_type }
         | 
| 101 | 
            +
              end
         | 
| 102 | 
            +
             | 
| 103 | 
            +
              def primary_office
         | 
| 104 | 
            +
                { address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
         | 
| 105 | 
            +
                  phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
         | 
| 106 | 
            +
              end
         | 
| 107 | 
            +
             | 
| 108 | 
            +
              def photo_url
         | 
| 109 | 
            +
                @_photo_url ||= "#{BASE_URI}#{panel.image}"
         | 
| 110 | 
            +
              end
         | 
| 111 | 
            +
             | 
| 112 | 
            +
              private
         | 
| 113 | 
            +
             | 
| 58 114 | 
             
              def build
         | 
| 59 115 | 
             
                @bio_page      = BioPage.new(panel.bio_page)
         | 
| 60 116 | 
             
                @state_name    = panel.state
         | 
| @@ -64,11 +120,18 @@ class Governator | |
| 64 120 |  | 
| 65 121 | 
             
                @first, @last, @middle, @nickname, @suffix = NameParser.new(official_full).parse
         | 
| 66 122 | 
             
                build_office_locations
         | 
| 123 | 
            +
                fetch_twitter_handle if self.class.use_twitter == true
         | 
| 67 124 | 
             
                self
         | 
| 68 125 | 
             
              end
         | 
| 69 126 |  | 
| 70 | 
            -
              def  | 
| 71 | 
            -
                 | 
| 127 | 
            +
              def fetch_twitter_handle
         | 
| 128 | 
            +
                twitter_governor = TwitterClient.governors.detect do |tg|
         | 
| 129 | 
            +
                  tg[:name].match?(last) &&
         | 
| 130 | 
            +
                    tg[:location].match?(state_name) ||
         | 
| 131 | 
            +
                    tg[:description].match?(state_name)
         | 
| 132 | 
            +
                end
         | 
| 133 | 
            +
             | 
| 134 | 
            +
                @twitter = twitter_governor[:screen_name] if twitter_governor
         | 
| 72 135 | 
             
              end
         | 
| 73 136 |  | 
| 74 137 | 
             
              def build_office_locations
         | 
| @@ -76,34 +139,8 @@ class Governator | |
| 76 139 | 
             
                @office_locations << secondary_office if bio_page.alt_office_present?
         | 
| 77 140 | 
             
              end
         | 
| 78 141 |  | 
| 79 | 
            -
              def primary_office
         | 
| 80 | 
            -
                { address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
         | 
| 81 | 
            -
                  phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
         | 
| 82 | 
            -
              end
         | 
| 83 | 
            -
             | 
| 84 | 
            -
              def secondary_office
         | 
| 85 | 
            -
                { address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
         | 
| 86 | 
            -
                  zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
         | 
| 87 | 
            -
                  office_type: bio_page.alt_office_type }
         | 
| 88 | 
            -
              end
         | 
| 89 | 
            -
             | 
| 90 142 | 
             
              def save
         | 
| 91 143 | 
             
                self.class.governors << self
         | 
| 92 144 | 
             
                self
         | 
| 93 145 | 
             
              end
         | 
| 94 | 
            -
             | 
| 95 | 
            -
              def to_h
         | 
| 96 | 
            -
                {
         | 
| 97 | 
            -
                  photo_url: photo_url,
         | 
| 98 | 
            -
                  state_name: state_name,
         | 
| 99 | 
            -
                  official_full: official_full,
         | 
| 100 | 
            -
                  url: url,
         | 
| 101 | 
            -
                  party: party,
         | 
| 102 | 
            -
                  office_locations: office_locations
         | 
| 103 | 
            -
                }
         | 
| 104 | 
            -
              end
         | 
| 105 | 
            -
             | 
| 106 | 
            -
              def inspect
         | 
| 107 | 
            -
                "#<Governator #{official_full}>"
         | 
| 108 | 
            -
              end
         | 
| 109 146 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: governator
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0.1. | 
| 4 | 
            +
              version: 0.1.2
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - M. Simon Borg
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: exe
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2017-06- | 
| 11 | 
            +
            date: 2017-06-21 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: faraday
         | 
| @@ -88,6 +88,7 @@ extensions: [] | |
| 88 88 | 
             
            extra_rdoc_files: []
         | 
| 89 89 | 
             
            files:
         | 
| 90 90 | 
             
            - ".gitignore"
         | 
| 91 | 
            +
            - ".rubocop.yml"
         | 
| 91 92 | 
             
            - Gemfile
         | 
| 92 93 | 
             
            - LICENSE.txt
         | 
| 93 94 | 
             
            - README.md
         | 
| @@ -100,6 +101,7 @@ files: | |
| 100 101 | 
             
            - lib/governator/name_parser.rb
         | 
| 101 102 | 
             
            - lib/governator/page_scraper.rb
         | 
| 102 103 | 
             
            - lib/governator/panel.rb
         | 
| 104 | 
            +
            - lib/governator/twitter_client.rb
         | 
| 103 105 | 
             
            - lib/governator/version.rb
         | 
| 104 106 | 
             
            homepage: https://github.com/msimonborg/governator
         | 
| 105 107 | 
             
            licenses:
         |