governator 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -0
- data/Gemfile +2 -0
- data/README.md +19 -0
- data/Rakefile +2 -0
- data/bin/console +10 -2
- data/governator.gemspec +1 -0
- data/lib/governator/twitter_client.rb +37 -0
- data/lib/governator/version.rb +3 -1
- data/lib/governator.rb +92 -55
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e89dcfc60bccb89d0080d03ca2b0b606192a19d9
|
4
|
+
data.tar.gz: 322faf0a64187c9b9fda0889a98a9adbe5648e50
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2600672ed050b4e7431e01c94dd369f494e61a9c93088238f16e05620a8cff1b66df83c27d5a676bb7070a2e83f8f41d1a59de52ea4c0c28db5ef6fc36e01200
|
7
|
+
data.tar.gz: 2f6d3be6510ae7087b63fa865498d77d06b7b3f03e7cb31a4c1b9d287dd6fc0cc851d9182e66e7e989f90f0aab4c686b0e7997e385bb30c0423a2b45f0fe04fa
|
data/.rubocop.yml
ADDED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -26,6 +26,25 @@ Governator.scrape!
|
|
26
26
|
governors = Governator.governors
|
27
27
|
```
|
28
28
|
|
29
|
+
## Twitter
|
30
|
+
|
31
|
+
If you want to scrape Twitter handles you will need to initialize the client. Add this to your code (in a Rails app it should probably go in a file called `governator.rb` in `./config/initializers/`)
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
Governator.config do |config|
|
35
|
+
config.use_twitter = true
|
36
|
+
|
37
|
+
config.twitter do |twitter|
|
38
|
+
twitter.consumer_key = YOUR_CONSUMER_KEY
|
39
|
+
twitter.consumer_secret = YOUR_TWITTER_CONSUMER_SECRET
|
40
|
+
twitter.access_token = YOUR_ACCESS_TOKEN
|
41
|
+
twitter.access_token_secret = YOUR_ACCESS_TOKEN_SECRET
|
42
|
+
end
|
43
|
+
end
|
44
|
+
```
|
45
|
+
|
46
|
+
As with everything, secrets should never be stored anywhere public, like version control. Set these values as variables on your system.
|
47
|
+
|
29
48
|
## Development
|
30
49
|
|
31
50
|
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
@@ -1,10 +1,18 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
|
3
4
|
require 'bundler/setup'
|
4
5
|
require 'governator'
|
5
6
|
|
6
|
-
|
7
|
-
|
7
|
+
Governator.config do |config|
|
8
|
+
config.use_twitter = true
|
9
|
+
config.twitter do |twitter|
|
10
|
+
twitter.consumer_key = ENV['TWITTER_CONSUMER_KEY']
|
11
|
+
twitter.consumer_secret = ENV['TWITTER_CONSUMER_SECRET']
|
12
|
+
twitter.access_token = ENV['ACCESS_TOKEN']
|
13
|
+
twitter.access_token_secret = ENV['ACCESS_TOKEN_SECRET']
|
14
|
+
end
|
15
|
+
end
|
8
16
|
|
9
17
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
18
|
require 'pry'
|
data/governator.gemspec
CHANGED
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Governator
|
4
|
+
class TwitterClient
|
5
|
+
def self.client(&block)
|
6
|
+
@_client ||= Twitter::REST::Client.new(&block)
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.governors
|
10
|
+
@_governors ||= nga_list_members + rga_list_members + cspan_list_members
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.nga_list
|
14
|
+
@_nga_list ||= client.lists('NatlGovsAssoc').detect { |l| l.name == 'Governors' }
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.nga_list_members
|
18
|
+
@_nga_list_members ||= client.list_members(nga_list, count: 100).attrs[:users]
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.rga_list
|
22
|
+
@_rga_list ||= client.lists('The_RGA').detect { |l| l.name == 'GOP Governors' }
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.rga_list_members
|
26
|
+
@_rga_list_members ||= client.list_members(rga_list, count: 100).attrs[:users]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.cspan_list
|
30
|
+
@_cspan_list ||= client.lists('cspan').detect { |l| l.name == 'Governors' }
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.cspan_list_members
|
34
|
+
@_cspan_list_members ||= client.list_members(cspan_list, count: 100).attrs[:users]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/governator/version.rb
CHANGED
data/lib/governator.rb
CHANGED
@@ -7,54 +7,110 @@ require 'twitter'
|
|
7
7
|
require 'governator/bio_page'
|
8
8
|
require 'governator/name_parser'
|
9
9
|
require 'governator/panel'
|
10
|
+
require 'governator/twitter_client'
|
10
11
|
require 'governator/version'
|
11
12
|
|
13
|
+
# Governator.scrape!
|
14
|
+
# governors = Governor.governors
|
12
15
|
class Governator
|
13
|
-
BASE_URI = 'https://www.nga.org'
|
16
|
+
BASE_URI = 'https://www.nga.org'
|
14
17
|
CONN = Faraday.new(url: BASE_URI)
|
15
18
|
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
+
class << self # <-- Begin class methods
|
20
|
+
attr_reader :use_twitter
|
19
21
|
|
20
|
-
|
21
|
-
|
22
|
-
|
22
|
+
def scrape!
|
23
|
+
governors.clear
|
24
|
+
panels.each do |panel|
|
25
|
+
governor = create(panel)
|
26
|
+
puts "Scraped #{governor.official_full} of #{governor.state_name}"
|
27
|
+
end
|
23
28
|
|
24
|
-
|
25
|
-
@_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
|
26
|
-
Panel.new(panel)
|
29
|
+
governors
|
27
30
|
end
|
28
|
-
end
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
-
panels.each do |panel|
|
33
|
-
governor = create(panel)
|
34
|
-
puts "Scraped #{governor.official_full} of #{governor.state_name}"
|
32
|
+
def governors
|
33
|
+
@_governors ||= []
|
35
34
|
end
|
36
35
|
|
37
|
-
|
38
|
-
|
36
|
+
def to_json
|
37
|
+
governors.map(&:to_h)
|
38
|
+
end
|
39
|
+
alias serialize to_json
|
39
40
|
|
40
|
-
|
41
|
-
|
42
|
-
g.build
|
43
|
-
g.save
|
41
|
+
def config
|
42
|
+
yield self
|
44
43
|
end
|
45
|
-
end
|
46
44
|
|
47
|
-
|
48
|
-
|
49
|
-
|
45
|
+
def twitter(&block)
|
46
|
+
TwitterClient.client(&block)
|
47
|
+
end
|
48
|
+
|
49
|
+
def use_twitter=(boolean)
|
50
|
+
raise ArgumentError, 'value must be Boolean value' unless [true, false].include? boolean
|
51
|
+
@use_twitter = boolean
|
52
|
+
end
|
53
|
+
|
54
|
+
private # private class methods
|
55
|
+
|
56
|
+
def index_page
|
57
|
+
@_index_page ||= Nokogiri::HTML(CONN.get('/cms/governors/bios').body)
|
58
|
+
end
|
59
|
+
|
60
|
+
def panels
|
61
|
+
@_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
|
62
|
+
Panel.new(panel)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def create(panel)
|
67
|
+
new(panel).tap do |g|
|
68
|
+
g.send :build
|
69
|
+
g.send :save
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end # <-- End class methods
|
50
73
|
|
51
74
|
attr_reader :panel, :state_name, :bio_page, :official_full, :first, :last,
|
52
|
-
:middle, :nickname, :suffix, :url, :party, :office_locations
|
75
|
+
:middle, :nickname, :suffix, :url, :party, :office_locations,
|
76
|
+
:twitter
|
53
77
|
|
54
78
|
def initialize(panel)
|
55
79
|
@panel = panel
|
56
80
|
end
|
57
81
|
|
82
|
+
def to_h
|
83
|
+
{
|
84
|
+
photo_url: photo_url,
|
85
|
+
state_name: state_name,
|
86
|
+
official_full: official_full,
|
87
|
+
url: url,
|
88
|
+
party: party,
|
89
|
+
office_locations: office_locations
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
def inspect
|
94
|
+
"#<Governator #{official_full}>"
|
95
|
+
end
|
96
|
+
|
97
|
+
def secondary_office
|
98
|
+
{ address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
|
99
|
+
zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
|
100
|
+
office_type: bio_page.alt_office_type }
|
101
|
+
end
|
102
|
+
|
103
|
+
def primary_office
|
104
|
+
{ address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
|
105
|
+
phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
|
106
|
+
end
|
107
|
+
|
108
|
+
def photo_url
|
109
|
+
@_photo_url ||= "#{BASE_URI}#{panel.image}"
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
58
114
|
def build
|
59
115
|
@bio_page = BioPage.new(panel.bio_page)
|
60
116
|
@state_name = panel.state
|
@@ -64,11 +120,18 @@ class Governator
|
|
64
120
|
|
65
121
|
@first, @last, @middle, @nickname, @suffix = NameParser.new(official_full).parse
|
66
122
|
build_office_locations
|
123
|
+
fetch_twitter_handle if self.class.use_twitter == true
|
67
124
|
self
|
68
125
|
end
|
69
126
|
|
70
|
-
def
|
71
|
-
|
127
|
+
def fetch_twitter_handle
|
128
|
+
twitter_governor = TwitterClient.governors.detect do |tg|
|
129
|
+
tg[:name].match?(last) &&
|
130
|
+
tg[:location].match?(state_name) ||
|
131
|
+
tg[:description].match?(state_name)
|
132
|
+
end
|
133
|
+
|
134
|
+
@twitter = twitter_governor[:screen_name] if twitter_governor
|
72
135
|
end
|
73
136
|
|
74
137
|
def build_office_locations
|
@@ -76,34 +139,8 @@ class Governator
|
|
76
139
|
@office_locations << secondary_office if bio_page.alt_office_present?
|
77
140
|
end
|
78
141
|
|
79
|
-
def primary_office
|
80
|
-
{ address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
|
81
|
-
phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
|
82
|
-
end
|
83
|
-
|
84
|
-
def secondary_office
|
85
|
-
{ address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
|
86
|
-
zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
|
87
|
-
office_type: bio_page.alt_office_type }
|
88
|
-
end
|
89
|
-
|
90
142
|
def save
|
91
143
|
self.class.governors << self
|
92
144
|
self
|
93
145
|
end
|
94
|
-
|
95
|
-
def to_h
|
96
|
-
{
|
97
|
-
photo_url: photo_url,
|
98
|
-
state_name: state_name,
|
99
|
-
official_full: official_full,
|
100
|
-
url: url,
|
101
|
-
party: party,
|
102
|
-
office_locations: office_locations
|
103
|
-
}
|
104
|
-
end
|
105
|
-
|
106
|
-
def inspect
|
107
|
-
"#<Governator #{official_full}>"
|
108
|
-
end
|
109
146
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: governator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- M. Simon Borg
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -88,6 +88,7 @@ extensions: []
|
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
90
|
- ".gitignore"
|
91
|
+
- ".rubocop.yml"
|
91
92
|
- Gemfile
|
92
93
|
- LICENSE.txt
|
93
94
|
- README.md
|
@@ -100,6 +101,7 @@ files:
|
|
100
101
|
- lib/governator/name_parser.rb
|
101
102
|
- lib/governator/page_scraper.rb
|
102
103
|
- lib/governator/panel.rb
|
104
|
+
- lib/governator/twitter_client.rb
|
103
105
|
- lib/governator/version.rb
|
104
106
|
homepage: https://github.com/msimonborg/governator
|
105
107
|
licenses:
|