governator 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -0
- data/Gemfile +2 -0
- data/README.md +19 -0
- data/Rakefile +2 -0
- data/bin/console +10 -2
- data/governator.gemspec +1 -0
- data/lib/governator/twitter_client.rb +37 -0
- data/lib/governator/version.rb +3 -1
- data/lib/governator.rb +92 -55
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e89dcfc60bccb89d0080d03ca2b0b606192a19d9
|
4
|
+
data.tar.gz: 322faf0a64187c9b9fda0889a98a9adbe5648e50
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2600672ed050b4e7431e01c94dd369f494e61a9c93088238f16e05620a8cff1b66df83c27d5a676bb7070a2e83f8f41d1a59de52ea4c0c28db5ef6fc36e01200
|
7
|
+
data.tar.gz: 2f6d3be6510ae7087b63fa865498d77d06b7b3f03e7cb31a4c1b9d287dd6fc0cc851d9182e66e7e989f90f0aab4c686b0e7997e385bb30c0423a2b45f0fe04fa
|
data/.rubocop.yml
ADDED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -26,6 +26,25 @@ Governator.scrape!
|
|
26
26
|
governors = Governator.governors
|
27
27
|
```
|
28
28
|
|
29
|
+
## Twitter
|
30
|
+
|
31
|
+
If you want to scrape Twitter handles you will need to initialize the client. Add this to your code (in a Rails app it should probably go in a file called `governator.rb` in `./config/initializers/`)
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
Governator.config do |config|
|
35
|
+
config.use_twitter = true
|
36
|
+
|
37
|
+
config.twitter do |twitter|
|
38
|
+
twitter.consumer_key = YOUR_CONSUMER_KEY
|
39
|
+
twitter.consumer_secret = YOUR_TWITTER_CONSUMER_SECRET
|
40
|
+
twitter.access_token = YOUR_ACCESS_TOKEN
|
41
|
+
twitter.access_token_secret = YOUR_ACCESS_TOKEN_SECRET
|
42
|
+
end
|
43
|
+
end
|
44
|
+
```
|
45
|
+
|
46
|
+
As with everything, secrets should never be stored anywhere public, like version control. Set these values as variables on your system.
|
47
|
+
|
29
48
|
## Development
|
30
49
|
|
31
50
|
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
@@ -1,10 +1,18 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
2
3
|
|
3
4
|
require 'bundler/setup'
|
4
5
|
require 'governator'
|
5
6
|
|
6
|
-
|
7
|
-
|
7
|
+
Governator.config do |config|
|
8
|
+
config.use_twitter = true
|
9
|
+
config.twitter do |twitter|
|
10
|
+
twitter.consumer_key = ENV['TWITTER_CONSUMER_KEY']
|
11
|
+
twitter.consumer_secret = ENV['TWITTER_CONSUMER_SECRET']
|
12
|
+
twitter.access_token = ENV['ACCESS_TOKEN']
|
13
|
+
twitter.access_token_secret = ENV['ACCESS_TOKEN_SECRET']
|
14
|
+
end
|
15
|
+
end
|
8
16
|
|
9
17
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
18
|
require 'pry'
|
data/governator.gemspec
CHANGED
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Governator
|
4
|
+
class TwitterClient
|
5
|
+
def self.client(&block)
|
6
|
+
@_client ||= Twitter::REST::Client.new(&block)
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.governors
|
10
|
+
@_governors ||= nga_list_members + rga_list_members + cspan_list_members
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.nga_list
|
14
|
+
@_nga_list ||= client.lists('NatlGovsAssoc').detect { |l| l.name == 'Governors' }
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.nga_list_members
|
18
|
+
@_nga_list_members ||= client.list_members(nga_list, count: 100).attrs[:users]
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.rga_list
|
22
|
+
@_rga_list ||= client.lists('The_RGA').detect { |l| l.name == 'GOP Governors' }
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.rga_list_members
|
26
|
+
@_rga_list_members ||= client.list_members(rga_list, count: 100).attrs[:users]
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.cspan_list
|
30
|
+
@_cspan_list ||= client.lists('cspan').detect { |l| l.name == 'Governors' }
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.cspan_list_members
|
34
|
+
@_cspan_list_members ||= client.list_members(cspan_list, count: 100).attrs[:users]
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/governator/version.rb
CHANGED
data/lib/governator.rb
CHANGED
@@ -7,54 +7,110 @@ require 'twitter'
|
|
7
7
|
require 'governator/bio_page'
|
8
8
|
require 'governator/name_parser'
|
9
9
|
require 'governator/panel'
|
10
|
+
require 'governator/twitter_client'
|
10
11
|
require 'governator/version'
|
11
12
|
|
13
|
+
# Governator.scrape!
|
14
|
+
# governors = Governor.governors
|
12
15
|
class Governator
|
13
|
-
BASE_URI = 'https://www.nga.org'
|
16
|
+
BASE_URI = 'https://www.nga.org'
|
14
17
|
CONN = Faraday.new(url: BASE_URI)
|
15
18
|
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
+
class << self # <-- Begin class methods
|
20
|
+
attr_reader :use_twitter
|
19
21
|
|
20
|
-
|
21
|
-
|
22
|
-
|
22
|
+
def scrape!
|
23
|
+
governors.clear
|
24
|
+
panels.each do |panel|
|
25
|
+
governor = create(panel)
|
26
|
+
puts "Scraped #{governor.official_full} of #{governor.state_name}"
|
27
|
+
end
|
23
28
|
|
24
|
-
|
25
|
-
@_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
|
26
|
-
Panel.new(panel)
|
29
|
+
governors
|
27
30
|
end
|
28
|
-
end
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
-
panels.each do |panel|
|
33
|
-
governor = create(panel)
|
34
|
-
puts "Scraped #{governor.official_full} of #{governor.state_name}"
|
32
|
+
def governors
|
33
|
+
@_governors ||= []
|
35
34
|
end
|
36
35
|
|
37
|
-
|
38
|
-
|
36
|
+
def to_json
|
37
|
+
governors.map(&:to_h)
|
38
|
+
end
|
39
|
+
alias serialize to_json
|
39
40
|
|
40
|
-
|
41
|
-
|
42
|
-
g.build
|
43
|
-
g.save
|
41
|
+
def config
|
42
|
+
yield self
|
44
43
|
end
|
45
|
-
end
|
46
44
|
|
47
|
-
|
48
|
-
|
49
|
-
|
45
|
+
def twitter(&block)
|
46
|
+
TwitterClient.client(&block)
|
47
|
+
end
|
48
|
+
|
49
|
+
def use_twitter=(boolean)
|
50
|
+
raise ArgumentError, 'value must be Boolean value' unless [true, false].include? boolean
|
51
|
+
@use_twitter = boolean
|
52
|
+
end
|
53
|
+
|
54
|
+
private # private class methods
|
55
|
+
|
56
|
+
def index_page
|
57
|
+
@_index_page ||= Nokogiri::HTML(CONN.get('/cms/governors/bios').body)
|
58
|
+
end
|
59
|
+
|
60
|
+
def panels
|
61
|
+
@_panels ||= index_page.css('.panel.panel-default.governors').map do |panel|
|
62
|
+
Panel.new(panel)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def create(panel)
|
67
|
+
new(panel).tap do |g|
|
68
|
+
g.send :build
|
69
|
+
g.send :save
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end # <-- End class methods
|
50
73
|
|
51
74
|
attr_reader :panel, :state_name, :bio_page, :official_full, :first, :last,
|
52
|
-
:middle, :nickname, :suffix, :url, :party, :office_locations
|
75
|
+
:middle, :nickname, :suffix, :url, :party, :office_locations,
|
76
|
+
:twitter
|
53
77
|
|
54
78
|
def initialize(panel)
|
55
79
|
@panel = panel
|
56
80
|
end
|
57
81
|
|
82
|
+
def to_h
|
83
|
+
{
|
84
|
+
photo_url: photo_url,
|
85
|
+
state_name: state_name,
|
86
|
+
official_full: official_full,
|
87
|
+
url: url,
|
88
|
+
party: party,
|
89
|
+
office_locations: office_locations
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
def inspect
|
94
|
+
"#<Governator #{official_full}>"
|
95
|
+
end
|
96
|
+
|
97
|
+
def secondary_office
|
98
|
+
{ address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
|
99
|
+
zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
|
100
|
+
office_type: bio_page.alt_office_type }
|
101
|
+
end
|
102
|
+
|
103
|
+
def primary_office
|
104
|
+
{ address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
|
105
|
+
phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
|
106
|
+
end
|
107
|
+
|
108
|
+
def photo_url
|
109
|
+
@_photo_url ||= "#{BASE_URI}#{panel.image}"
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
58
114
|
def build
|
59
115
|
@bio_page = BioPage.new(panel.bio_page)
|
60
116
|
@state_name = panel.state
|
@@ -64,11 +120,18 @@ class Governator
|
|
64
120
|
|
65
121
|
@first, @last, @middle, @nickname, @suffix = NameParser.new(official_full).parse
|
66
122
|
build_office_locations
|
123
|
+
fetch_twitter_handle if self.class.use_twitter == true
|
67
124
|
self
|
68
125
|
end
|
69
126
|
|
70
|
-
def
|
71
|
-
|
127
|
+
def fetch_twitter_handle
|
128
|
+
twitter_governor = TwitterClient.governors.detect do |tg|
|
129
|
+
tg[:name].match?(last) &&
|
130
|
+
tg[:location].match?(state_name) ||
|
131
|
+
tg[:description].match?(state_name)
|
132
|
+
end
|
133
|
+
|
134
|
+
@twitter = twitter_governor[:screen_name] if twitter_governor
|
72
135
|
end
|
73
136
|
|
74
137
|
def build_office_locations
|
@@ -76,34 +139,8 @@ class Governator
|
|
76
139
|
@office_locations << secondary_office if bio_page.alt_office_present?
|
77
140
|
end
|
78
141
|
|
79
|
-
def primary_office
|
80
|
-
{ address: bio_page.address, city: bio_page.city, state: bio_page.state, zip: bio_page.zip,
|
81
|
-
phone: bio_page.phone, fax: bio_page.fax, office_type: bio_page.office_type }
|
82
|
-
end
|
83
|
-
|
84
|
-
def secondary_office
|
85
|
-
{ address: bio_page.alt_address, city: bio_page.alt_city, state: bio_page.alt_state,
|
86
|
-
zip: bio_page.alt_zip, phone: bio_page.alt_phone, fax: bio_page.alt_fax,
|
87
|
-
office_type: bio_page.alt_office_type }
|
88
|
-
end
|
89
|
-
|
90
142
|
def save
|
91
143
|
self.class.governors << self
|
92
144
|
self
|
93
145
|
end
|
94
|
-
|
95
|
-
def to_h
|
96
|
-
{
|
97
|
-
photo_url: photo_url,
|
98
|
-
state_name: state_name,
|
99
|
-
official_full: official_full,
|
100
|
-
url: url,
|
101
|
-
party: party,
|
102
|
-
office_locations: office_locations
|
103
|
-
}
|
104
|
-
end
|
105
|
-
|
106
|
-
def inspect
|
107
|
-
"#<Governator #{official_full}>"
|
108
|
-
end
|
109
146
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: governator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- M. Simon Borg
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -88,6 +88,7 @@ extensions: []
|
|
88
88
|
extra_rdoc_files: []
|
89
89
|
files:
|
90
90
|
- ".gitignore"
|
91
|
+
- ".rubocop.yml"
|
91
92
|
- Gemfile
|
92
93
|
- LICENSE.txt
|
93
94
|
- README.md
|
@@ -100,6 +101,7 @@ files:
|
|
100
101
|
- lib/governator/name_parser.rb
|
101
102
|
- lib/governator/page_scraper.rb
|
102
103
|
- lib/governator/panel.rb
|
104
|
+
- lib/governator/twitter_client.rb
|
103
105
|
- lib/governator/version.rb
|
104
106
|
homepage: https://github.com/msimonborg/governator
|
105
107
|
licenses:
|