govkit 0.5.0 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- data/{README.markdown → README.md} +1 -0
- data/Rakefile +3 -2
- data/VERSION +1 -1
- data/govkit.gemspec +63 -61
- data/lib/generators/govkit/templates/govkit.rb +1 -1
- data/lib/gov_kit/configuration.rb +5 -1
- data/lib/gov_kit/follow_the_money.rb +24 -26
- data/lib/gov_kit/open_congress.rb +1 -1
- data/lib/gov_kit/open_states.rb +3 -3
- data/lib/gov_kit/resource.rb +5 -7
- data/lib/gov_kit/search_engines/google_blog.rb +8 -9
- data/lib/gov_kit/search_engines/google_news.rb +6 -7
- data/lib/gov_kit/search_engines/technorati.rb +1 -2
- data/lib/gov_kit/search_engines/wikipedia.rb +5 -3
- data/lib/gov_kit/transparency_data.rb +43 -0
- data/lib/gov_kit.rb +4 -2
- metadata +32 -16
- data/.gitignore +0 -21
@@ -6,6 +6,7 @@ Govkit is a Ruby gem that provides simple access to open government APIs around
|
|
6
6
|
* [The Open States project](http://fiftystates-dev.sunlightlabs.com/), which has a RESTful API for accessing data about state legislators, bills, votes, etc.
|
7
7
|
* [Project Vote Smart](http://www.votesmart.org/services_api.php), which has an API with congressional addresses, etc.
|
8
8
|
* [Follow The Money](http://www.followthemoney.org/), whose API reveals campaign contribution data for state officials.
|
9
|
+
* [TransparencyData API](http://transparencydata.com/api) (campaign contributions only)
|
9
10
|
|
10
11
|
# Installation
|
11
12
|
|
data/Rakefile
CHANGED
@@ -19,12 +19,13 @@ begin
|
|
19
19
|
gem.name = "govkit"
|
20
20
|
gem.summary = %Q{Simple access to open government APIs around the web}
|
21
21
|
gem.description = %Q{Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.}
|
22
|
-
gem.email = "
|
22
|
+
gem.email = "develop@opencongress.org"
|
23
23
|
gem.homepage = "http://github.com/opengovernment/govkit"
|
24
24
|
gem.authors = ["Participatory Politics Foundation", "Srinivas Aki", "Carl Tashian"]
|
25
25
|
gem.add_dependency('httparty', '>= 0.5.2')
|
26
26
|
gem.add_dependency('json', '>= 1.4.3')
|
27
|
-
gem.add_dependency('
|
27
|
+
gem.add_dependency('nokogiri', '>= 1.4.4')
|
28
|
+
gem.add_dependency('fastercsv', '>= 1.5.3')
|
28
29
|
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
29
30
|
end
|
30
31
|
Jeweler::GemcutterTasks.new
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.1
|
data/govkit.gemspec
CHANGED
@@ -1,83 +1,82 @@
|
|
1
1
|
# Generated by jeweler
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{govkit}
|
8
|
-
s.version = "0.5.
|
8
|
+
s.version = "0.5.1"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Participatory Politics Foundation", "Srinivas Aki", "Carl Tashian"]
|
12
|
-
s.date = %q{2010-
|
12
|
+
s.date = %q{2010-12-21}
|
13
13
|
s.description = %q{Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.}
|
14
|
-
s.email = %q{
|
14
|
+
s.email = %q{develop@opencongress.org}
|
15
15
|
s.extra_rdoc_files = [
|
16
16
|
"LICENSE",
|
17
|
-
|
17
|
+
"README.md"
|
18
18
|
]
|
19
19
|
s.files = [
|
20
20
|
".document",
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
21
|
+
"LICENSE",
|
22
|
+
"README.md",
|
23
|
+
"Rakefile",
|
24
|
+
"USAGE",
|
25
|
+
"VERSION",
|
26
|
+
"generators/govkit/govkit_generator.rb",
|
27
|
+
"generators/govkit/templates/govkit.rb",
|
28
|
+
"govkit.gemspec",
|
29
|
+
"init.rb",
|
30
|
+
"lib/generators/govkit/govkit_generator.rb",
|
31
|
+
"lib/generators/govkit/templates/govkit.rb",
|
32
|
+
"lib/gov_kit.rb",
|
33
|
+
"lib/gov_kit/acts_as_noteworthy.rb",
|
34
|
+
"lib/gov_kit/configuration.rb",
|
35
|
+
"lib/gov_kit/follow_the_money.rb",
|
36
|
+
"lib/gov_kit/open_congress.rb",
|
37
|
+
"lib/gov_kit/open_congress/bill.rb",
|
38
|
+
"lib/gov_kit/open_congress/blog_post.rb",
|
39
|
+
"lib/gov_kit/open_congress/news_post.rb",
|
40
|
+
"lib/gov_kit/open_congress/person.rb",
|
41
|
+
"lib/gov_kit/open_congress/person_stat.rb",
|
42
|
+
"lib/gov_kit/open_congress/roll_call.rb",
|
43
|
+
"lib/gov_kit/open_congress/roll_call_comparison.rb",
|
44
|
+
"lib/gov_kit/open_congress/voting_comparison.rb",
|
45
|
+
"lib/gov_kit/open_states.rb",
|
46
|
+
"lib/gov_kit/railtie.rb",
|
47
|
+
"lib/gov_kit/resource.rb",
|
48
|
+
"lib/gov_kit/search_engines.rb",
|
49
|
+
"lib/gov_kit/search_engines/google_blog.rb",
|
50
|
+
"lib/gov_kit/search_engines/google_news.rb",
|
51
|
+
"lib/gov_kit/search_engines/technorati.rb",
|
52
|
+
"lib/gov_kit/search_engines/wikipedia.rb",
|
53
|
+
"lib/gov_kit/transparency_data.rb",
|
54
|
+
"lib/gov_kit/vote_smart.rb",
|
55
|
+
"lib/govkit.rb",
|
56
|
+
"spec/fixtures/follow_the_money/business-page0.response",
|
57
|
+
"spec/fixtures/follow_the_money/business-page1.response",
|
58
|
+
"spec/fixtures/follow_the_money/contribution.response",
|
59
|
+
"spec/fixtures/follow_the_money/unauthorized.response",
|
60
|
+
"spec/fixtures/open_states/401.response",
|
61
|
+
"spec/fixtures/open_states/410.response",
|
62
|
+
"spec/fixtures/open_states/bill.response",
|
63
|
+
"spec/fixtures/open_states/bill_query.response",
|
64
|
+
"spec/fixtures/open_states/legislator.response",
|
65
|
+
"spec/fixtures/open_states/legislator_query.response",
|
66
|
+
"spec/fixtures/open_states/state.response",
|
67
|
+
"spec/follow_the_money_spec.rb",
|
68
|
+
"spec/open_states_spec.rb",
|
69
|
+
"spec/spec.opts",
|
70
|
+
"spec/spec_helper.rb"
|
71
71
|
]
|
72
72
|
s.homepage = %q{http://github.com/opengovernment/govkit}
|
73
|
-
s.rdoc_options = ["--charset=UTF-8"]
|
74
73
|
s.require_paths = ["lib"]
|
75
74
|
s.rubygems_version = %q{1.3.7}
|
76
75
|
s.summary = %q{Simple access to open government APIs around the web}
|
77
76
|
s.test_files = [
|
78
77
|
"spec/follow_the_money_spec.rb",
|
79
|
-
|
80
|
-
|
78
|
+
"spec/open_states_spec.rb",
|
79
|
+
"spec/spec_helper.rb"
|
81
80
|
]
|
82
81
|
|
83
82
|
if s.respond_to? :specification_version then
|
@@ -87,16 +86,19 @@ Gem::Specification.new do |s|
|
|
87
86
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
88
87
|
s.add_runtime_dependency(%q<httparty>, [">= 0.5.2"])
|
89
88
|
s.add_runtime_dependency(%q<json>, [">= 1.4.3"])
|
90
|
-
s.add_runtime_dependency(%q<
|
89
|
+
s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
|
90
|
+
s.add_runtime_dependency(%q<fastercsv>, [">= 1.5.3"])
|
91
91
|
else
|
92
92
|
s.add_dependency(%q<httparty>, [">= 0.5.2"])
|
93
93
|
s.add_dependency(%q<json>, [">= 1.4.3"])
|
94
|
-
s.add_dependency(%q<
|
94
|
+
s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
|
95
|
+
s.add_dependency(%q<fastercsv>, [">= 1.5.3"])
|
95
96
|
end
|
96
97
|
else
|
97
98
|
s.add_dependency(%q<httparty>, [">= 0.5.2"])
|
98
99
|
s.add_dependency(%q<json>, [">= 1.4.3"])
|
99
|
-
s.add_dependency(%q<
|
100
|
+
s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
|
101
|
+
s.add_dependency(%q<fastercsv>, [">= 1.5.3"])
|
100
102
|
end
|
101
103
|
end
|
102
104
|
|
@@ -2,7 +2,7 @@ if defined? GovKit
|
|
2
2
|
GovKit.configure do |config|
|
3
3
|
# Get an API key for Sunlight's Open States project here:
|
4
4
|
# http://services.sunlightlabs.com/accounts/register/
|
5
|
-
config.
|
5
|
+
config.sunlight_apikey = 'YOUR_SUNLIGHT_API_KEY'
|
6
6
|
|
7
7
|
##API key for Votesmart
|
8
8
|
# http://votesmart.org/services_api.php
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module GovKit
|
2
2
|
class Configuration
|
3
|
-
attr_accessor :
|
3
|
+
attr_accessor :sunlight_apikey, :openstates_base_url, :transparency_data_base_url, :transparency_data_categories_url
|
4
4
|
attr_accessor :votesmart_apikey, :votesmart_base_url
|
5
5
|
attr_accessor :ftm_apikey, :ftm_base_url
|
6
6
|
attr_accessor :opencongress_apikey, :opencongress_base_url
|
@@ -11,6 +11,7 @@ module GovKit
|
|
11
11
|
def initialize
|
12
12
|
@openstates_apikey = @votesmart_apikey = @ftm_apikey = ''
|
13
13
|
@openstates_base_url = 'openstates.sunlightlabs.com/api/v1/'
|
14
|
+
@transparency_data_base_url = 'transparencydata.com/api/1.0/'
|
14
15
|
@votesmart_base_url = 'api.votesmart.org/'
|
15
16
|
@ftm_base_url = 'api.followthemoney.org/'
|
16
17
|
@opencongress_base_url = 'www.opencongress.org/'
|
@@ -18,6 +19,9 @@ module GovKit
|
|
18
19
|
@google_blog_base_url = 'blogsearch.google.com'
|
19
20
|
@google_news_base_url = 'news.google.com'
|
20
21
|
@wikipedia_base_url = 'en.wikipedia.org'
|
22
|
+
|
23
|
+
# Permant home for contribution category mappings
|
24
|
+
@transparency_data_categories_url = 'assets.transparencydata.org.s3.amazonaws.com/docs/catcodes.csv'
|
21
25
|
end
|
22
26
|
end
|
23
27
|
|
@@ -4,14 +4,13 @@ module GovKit
|
|
4
4
|
base_uri GovKit::configuration.ftm_base_url
|
5
5
|
|
6
6
|
def self.get_xml(path, options)
|
7
|
-
|
8
|
-
doc = Hpricot::XML(response)
|
7
|
+
doc = Nokogiri::XML(get(path, options))
|
9
8
|
|
10
9
|
e = doc.search("//error")
|
11
10
|
|
12
11
|
# Deal with whatever error comes back
|
13
12
|
if e.size > 0
|
14
|
-
raise case e.first.attributes['code']
|
13
|
+
raise case e.first.attributes['code'].value
|
15
14
|
when "100"
|
16
15
|
GovKit::NotAuthorized
|
17
16
|
when "300"
|
@@ -20,11 +19,17 @@ module GovKit
|
|
20
19
|
GovKit::ResourceNotFound
|
21
20
|
else
|
22
21
|
GovKit::InvalidRequest
|
23
|
-
end, e.first.attributes['text']
|
22
|
+
end, e.first.attributes['text'].value
|
24
23
|
end
|
25
24
|
|
26
25
|
doc
|
27
26
|
end
|
27
|
+
|
28
|
+
def self.stringify_values_of(result)
|
29
|
+
# result is an array of hashes, but all the values are Nokogiri::XML::Attr objects, not strings.
|
30
|
+
# We want them to be strings.
|
31
|
+
result.collect! { |r| r.inject({}) {|h, (k, v)| h[k] = v.to_s; h } }
|
32
|
+
end
|
28
33
|
end
|
29
34
|
|
30
35
|
module FollowTheMoney
|
@@ -37,15 +42,14 @@ module GovKit
|
|
37
42
|
|
38
43
|
doc = get_xml("/base_level.industries.list.php", :query => {:page => page_num})
|
39
44
|
|
40
|
-
next_page = doc.
|
45
|
+
next_page = doc.children.first.attributes['next_page'].value
|
41
46
|
|
42
47
|
page_num += 1
|
43
48
|
|
44
|
-
result += doc.search('//business_detail').collect
|
45
|
-
business.attributes.to_hash
|
46
|
-
end
|
49
|
+
result += doc.search('//business_detail').collect { |x| x.attributes }
|
47
50
|
end
|
48
51
|
|
52
|
+
stringify_values_of(result)
|
49
53
|
parse(result)
|
50
54
|
end
|
51
55
|
end
|
@@ -57,24 +61,22 @@ module GovKit
|
|
57
61
|
until next_page != "yes"
|
58
62
|
doc = get_xml("/candidates.contributions.php", :query => {"imsp_candidate_id" => nimsp_id, :page => page_num})
|
59
63
|
|
60
|
-
next_page = doc.
|
64
|
+
next_page = doc.children.first.attributes['next_page'].value
|
61
65
|
|
62
66
|
page_num += 1
|
63
67
|
|
64
|
-
result
|
65
|
-
contribution.attributes.to_hash
|
66
|
-
end
|
68
|
+
result += doc.search('//contribution').collect { |x| x.attributes }
|
67
69
|
end
|
70
|
+
|
71
|
+
stringify_values_of(result)
|
68
72
|
parse(result)
|
69
73
|
end
|
70
74
|
|
71
75
|
def self.top(nimsp_id)
|
72
76
|
doc = get_xml("/candidates.top_contributor.php", :query => {"imsp_candidate_id" => nimsp_id})
|
77
|
+
result = doc.search('//top_contributor').collect { |x| x.attributes }
|
73
78
|
|
74
|
-
result
|
75
|
-
contribution.attributes.to_hash
|
76
|
-
end
|
77
|
-
|
79
|
+
stringify_values_of(result)
|
78
80
|
parse(result)
|
79
81
|
end
|
80
82
|
end
|
@@ -82,11 +84,9 @@ module GovKit
|
|
82
84
|
class IndustryContribution < Contribution
|
83
85
|
def self.find(nimsp_id)
|
84
86
|
doc = get_xml("/candidates.industries.php", :query => {"imsp_candidate_id" => nimsp_id})
|
87
|
+
result = doc.search('//candidate_industry').collect { |x| x.attributes }
|
85
88
|
|
86
|
-
result
|
87
|
-
contribution.attributes.to_hash
|
88
|
-
end
|
89
|
-
|
89
|
+
stringify_values_of(result)
|
90
90
|
parse(result)
|
91
91
|
end
|
92
92
|
end
|
@@ -95,10 +95,9 @@ module GovKit
|
|
95
95
|
def self.find(nimsp_id)
|
96
96
|
doc = get_xml("/candidates.sectors.php", :query => {"imsp_candidate_id" => nimsp_id})
|
97
97
|
|
98
|
-
result = doc.search('//candidate_sector').collect
|
99
|
-
contribution.attributes.to_hash
|
100
|
-
end
|
98
|
+
result = doc.search('//candidate_sector').collect { |x| x.attributes }
|
101
99
|
|
100
|
+
stringify_values_of(result)
|
102
101
|
parse(result)
|
103
102
|
end
|
104
103
|
end
|
@@ -107,10 +106,9 @@ module GovKit
|
|
107
106
|
def self.find(nimsp_id)
|
108
107
|
doc = get_xml("/candidates.businesses.php", :query => {"imsp_candidate_id" => nimsp_id})
|
109
108
|
|
110
|
-
result = doc.search('//candidate_business').collect
|
111
|
-
contribution.attributes.to_hash
|
112
|
-
end
|
109
|
+
result = doc.search('//candidate_business').collect { |x| x.attributes }
|
113
110
|
|
111
|
+
stringify_values_of(result)
|
114
112
|
parse(result)
|
115
113
|
end
|
116
114
|
end
|
data/lib/gov_kit/open_states.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module GovKit
|
2
2
|
class OpenStatesResource < Resource
|
3
|
-
default_params :output => 'json', :apikey => GovKit::configuration.
|
3
|
+
default_params :output => 'json', :apikey => GovKit::configuration.sunlight_apikey
|
4
4
|
base_uri GovKit::configuration.openstates_base_url
|
5
5
|
end
|
6
6
|
|
@@ -29,8 +29,8 @@ module GovKit
|
|
29
29
|
parse(response)
|
30
30
|
end
|
31
31
|
|
32
|
-
def self.latest(updated_since,
|
33
|
-
response = get('/bills/
|
32
|
+
def self.latest(updated_since, ops = {})
|
33
|
+
response = get('/bills/', :query => {:updated_since => updated_since.to_s}.merge(ops))
|
34
34
|
parse(response)
|
35
35
|
end
|
36
36
|
end
|
data/lib/gov_kit/resource.rb
CHANGED
@@ -17,7 +17,6 @@ module GovKit
|
|
17
17
|
# This method handles the basic responses we might get back from
|
18
18
|
# Net::HTTP. But if a service returns something other than a 404 when an object is not found,
|
19
19
|
# you'll need to handle that in the subclass.
|
20
|
-
|
21
20
|
raise ResourceNotFound, "Resource not found" unless !response.blank?
|
22
21
|
|
23
22
|
if response.class == HTTParty::Response
|
@@ -33,11 +32,10 @@ module GovKit
|
|
33
32
|
end
|
34
33
|
|
35
34
|
def self.instantiate(record)
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
instantiate_record(record)
|
35
|
+
if record.is_a?(Array)
|
36
|
+
instantiate_collection(record)
|
37
|
+
else
|
38
|
+
instantiate_record(record)
|
41
39
|
end
|
42
40
|
end
|
43
41
|
|
@@ -88,7 +86,7 @@ module GovKit
|
|
88
86
|
end
|
89
87
|
|
90
88
|
def find_or_create_resource_for(name)
|
91
|
-
resource_name = name.to_s.gsub(/^[_+]/,'').gsub(/^(
|
89
|
+
resource_name = name.to_s.gsub(/^[_+]/,'').gsub(/^(\-?\d)/, "n#{$1}").gsub(/\s/, '').camelize
|
92
90
|
if self.class.parents.size > 1
|
93
91
|
find_resource_in_modules(resource_name, self.class.parents)
|
94
92
|
else
|
@@ -6,10 +6,9 @@ module GovKit
|
|
6
6
|
host = GovKit::configuration.google_blog_base_url
|
7
7
|
path = "/blogsearch?hl=en&q=#{URI::encode(query)}&btnG=Search+Blogs&num=50"
|
8
8
|
|
9
|
-
|
10
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312',html))
|
9
|
+
doc = Nokogiri::HTML(make_request(host, path))
|
11
10
|
stories = doc.search("td.j")
|
12
|
-
titles = (doc/"a").select { |a| (a.attributes["id"] && a.attributes["id"].match(/p-(.*)/)) }
|
11
|
+
titles = (doc/"a").select { |a| (a.attributes["id"] && a.attributes["id"].value.match(/p-(.*)/)) }
|
13
12
|
|
14
13
|
mentions = []
|
15
14
|
|
@@ -17,12 +16,12 @@ module GovKit
|
|
17
16
|
mention = GovKit::Mention.new
|
18
17
|
t = titles.shift
|
19
18
|
|
20
|
-
mention.title =
|
21
|
-
mention.url = t.attributes["href"] if t
|
22
|
-
mention.date = story.at("font:nth(
|
23
|
-
mention.excerpt = (story.at("br + font").
|
24
|
-
mention.source = story.at("a.f1").
|
25
|
-
mention.url = story.at("a.f1").attributes["href"]
|
19
|
+
mention.title = t.text if t #.unpack("C*").pack("U*") if t
|
20
|
+
# mention.url = t.attributes["href"].value if t
|
21
|
+
mention.date = story.at("font:nth(1)").text.strip
|
22
|
+
mention.excerpt = (story.at("br + font").text) #.unpack("C*").pack("U*")
|
23
|
+
mention.source = story.at("a.f1").text
|
24
|
+
mention.url = story.at("a.f1").attributes["href"].value
|
26
25
|
|
27
26
|
mentions << mention
|
28
27
|
end
|
@@ -9,8 +9,7 @@ module GovKit
|
|
9
9
|
host = GovKit::configuration.google_news_base_url
|
10
10
|
path = "/news?hl=en&ned=us&q=#{URI::encode(query)}&btnG=Search+News&num=50"
|
11
11
|
|
12
|
-
|
13
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312',html))
|
12
|
+
doc = Nokogiri::HTML(make_request(host, path))
|
14
13
|
stories = doc.search("div.search-results > div.story")
|
15
14
|
|
16
15
|
mentions = []
|
@@ -18,11 +17,11 @@ module GovKit
|
|
18
17
|
stories.each do |story|
|
19
18
|
mention = GovKit::Mention.new
|
20
19
|
|
21
|
-
mention.title = story.at("h2.title a").
|
22
|
-
mention.url = story.at("h2.title a").attributes["href"]
|
23
|
-
mention.date = story.at("div.sub-title > span.date").
|
24
|
-
mention.source = story.at("div.sub-title > span.source").
|
25
|
-
mention.excerpt = story.at("div.body > div.snippet").
|
20
|
+
mention.title = story.at("h2.title a").text
|
21
|
+
mention.url = story.at("h2.title a").attributes["href"].value
|
22
|
+
mention.date = story.at("div.sub-title > span.date").text
|
23
|
+
mention.source = story.at("div.sub-title > span.source").text
|
24
|
+
mention.excerpt = story.at("div.body > div.snippet").text
|
26
25
|
|
27
26
|
mentions << mention
|
28
27
|
end
|
@@ -6,8 +6,7 @@ module GovKit
|
|
6
6
|
host = GovKit::configuration.technorati_base_url
|
7
7
|
path = "/search?key=#{GovKit::configuration.technorati_apikey}&limit=50&language=en&query=#{URI::encode(query)}"
|
8
8
|
|
9
|
-
|
10
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312',html))
|
9
|
+
doc = Nokogiri::HTML(make_request(host, path))
|
11
10
|
|
12
11
|
mentions = []
|
13
12
|
# doc.search("tapi/document/item").each do |i|
|
@@ -7,10 +7,12 @@ module GovKit
|
|
7
7
|
headers 'User-Agent' => 'GovKit +http://ppolitics.org'
|
8
8
|
|
9
9
|
def self.search(query, options={})
|
10
|
-
|
11
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312', response))
|
10
|
+
doc = Nokogiri::HTML(get("/wiki/#{query}"))
|
12
11
|
|
13
|
-
bio = doc.at('#bodyContent > p:first').
|
12
|
+
bio = doc.at('#bodyContent > p:first').text rescue ""
|
13
|
+
|
14
|
+
# Convert HTML => text.
|
15
|
+
# bio = Loofah.fragment(bio).text
|
14
16
|
|
15
17
|
return "" if bio =~ /may refer to:/
|
16
18
|
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'fastercsv'
|
2
|
+
|
3
|
+
module GovKit
|
4
|
+
class TransparencyDataResource < Resource
|
5
|
+
default_params :apikey => GovKit::configuration.sunlight_apikey
|
6
|
+
base_uri GovKit::configuration.transparency_data_base_url
|
7
|
+
end
|
8
|
+
|
9
|
+
module TransparencyData
|
10
|
+
# See http://transparencydata.com/api/contributions/
|
11
|
+
# for complete query options.
|
12
|
+
class Contribution < TransparencyDataResource
|
13
|
+
def self.find(ops = {})
|
14
|
+
response = get('/contributions.json', :query => ops)
|
15
|
+
parse(response)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class Entity < TransparencyDataResource
|
20
|
+
def self.find_by_id(id)
|
21
|
+
response = get("/entities/#{id}.json")
|
22
|
+
parse(response)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class Categories
|
27
|
+
# Contribution category code mapping table, in CSV format
|
28
|
+
# Returns an array of hashes.
|
29
|
+
def self.all
|
30
|
+
categories = []
|
31
|
+
open(GovKit::configuration.transparency_data_categories_url) do |f|
|
32
|
+
f.each_line do |line|
|
33
|
+
FasterCSV.parse(line) do |row|
|
34
|
+
categories << row.fields
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
categories
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
data/lib/gov_kit.rb
CHANGED
@@ -1,21 +1,23 @@
|
|
1
1
|
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
|
2
2
|
|
3
3
|
require 'active_support'
|
4
|
-
require '
|
4
|
+
require 'nokogiri'
|
5
5
|
require 'iconv'
|
6
6
|
require 'httparty'
|
7
|
+
require 'open-uri'
|
7
8
|
require 'json'
|
8
9
|
require 'gov_kit/configuration'
|
9
10
|
|
10
11
|
module GovKit
|
11
12
|
autoload :Resource, 'gov_kit/resource'
|
12
13
|
autoload :OpenStates, 'gov_kit/open_states'
|
14
|
+
autoload :TransparencyData, 'gov_kit/transparency_data'
|
13
15
|
autoload :VoteSmart, 'gov_kit/vote_smart'
|
14
16
|
autoload :ActsAsNoteworthy, 'gov_kit/acts_as_noteworthy'
|
15
17
|
autoload :FollowTheMoney, 'gov_kit/follow_the_money'
|
16
18
|
autoload :OpenCongress, 'gov_kit/open_congress'
|
17
19
|
autoload :SearchEngines, 'gov_kit/search_engines'
|
18
|
-
|
20
|
+
|
19
21
|
class Mention
|
20
22
|
attr_accessor :url, :excerpt, :title, :source, :date, :weight
|
21
23
|
end
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: govkit
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 9
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 5
|
9
|
-
-
|
10
|
-
version: 0.5.
|
9
|
+
- 1
|
10
|
+
version: 0.5.1
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Participatory Politics Foundation
|
@@ -17,7 +17,7 @@ autorequire:
|
|
17
17
|
bindir: bin
|
18
18
|
cert_chain: []
|
19
19
|
|
20
|
-
date: 2010-
|
20
|
+
date: 2010-12-21 00:00:00 -08:00
|
21
21
|
default_executable:
|
22
22
|
dependencies:
|
23
23
|
- !ruby/object:Gem::Dependency
|
@@ -53,35 +53,50 @@ dependencies:
|
|
53
53
|
type: :runtime
|
54
54
|
version_requirements: *id002
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: nokogiri
|
57
57
|
prerelease: false
|
58
58
|
requirement: &id003 !ruby/object:Gem::Requirement
|
59
59
|
none: false
|
60
60
|
requirements:
|
61
61
|
- - ">="
|
62
62
|
- !ruby/object:Gem::Version
|
63
|
-
hash:
|
63
|
+
hash: 15
|
64
64
|
segments:
|
65
|
-
-
|
66
|
-
-
|
67
|
-
-
|
68
|
-
version:
|
65
|
+
- 1
|
66
|
+
- 4
|
67
|
+
- 4
|
68
|
+
version: 1.4.4
|
69
69
|
type: :runtime
|
70
70
|
version_requirements: *id003
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: fastercsv
|
73
|
+
prerelease: false
|
74
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
hash: 5
|
80
|
+
segments:
|
81
|
+
- 1
|
82
|
+
- 5
|
83
|
+
- 3
|
84
|
+
version: 1.5.3
|
85
|
+
type: :runtime
|
86
|
+
version_requirements: *id004
|
71
87
|
description: Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.
|
72
|
-
email:
|
88
|
+
email: develop@opencongress.org
|
73
89
|
executables: []
|
74
90
|
|
75
91
|
extensions: []
|
76
92
|
|
77
93
|
extra_rdoc_files:
|
78
94
|
- LICENSE
|
79
|
-
- README.
|
95
|
+
- README.md
|
80
96
|
files:
|
81
97
|
- .document
|
82
|
-
- .gitignore
|
83
98
|
- LICENSE
|
84
|
-
- README.
|
99
|
+
- README.md
|
85
100
|
- Rakefile
|
86
101
|
- USAGE
|
87
102
|
- VERSION
|
@@ -112,6 +127,7 @@ files:
|
|
112
127
|
- lib/gov_kit/search_engines/google_news.rb
|
113
128
|
- lib/gov_kit/search_engines/technorati.rb
|
114
129
|
- lib/gov_kit/search_engines/wikipedia.rb
|
130
|
+
- lib/gov_kit/transparency_data.rb
|
115
131
|
- lib/gov_kit/vote_smart.rb
|
116
132
|
- lib/govkit.rb
|
117
133
|
- spec/fixtures/follow_the_money/business-page0.response
|
@@ -134,8 +150,8 @@ homepage: http://github.com/opengovernment/govkit
|
|
134
150
|
licenses: []
|
135
151
|
|
136
152
|
post_install_message:
|
137
|
-
rdoc_options:
|
138
|
-
|
153
|
+
rdoc_options: []
|
154
|
+
|
139
155
|
require_paths:
|
140
156
|
- lib
|
141
157
|
required_ruby_version: !ruby/object:Gem::Requirement
|