govkit 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/{README.markdown → README.md} +1 -0
- data/Rakefile +3 -2
- data/VERSION +1 -1
- data/govkit.gemspec +63 -61
- data/lib/generators/govkit/templates/govkit.rb +1 -1
- data/lib/gov_kit/configuration.rb +5 -1
- data/lib/gov_kit/follow_the_money.rb +24 -26
- data/lib/gov_kit/open_congress.rb +1 -1
- data/lib/gov_kit/open_states.rb +3 -3
- data/lib/gov_kit/resource.rb +5 -7
- data/lib/gov_kit/search_engines/google_blog.rb +8 -9
- data/lib/gov_kit/search_engines/google_news.rb +6 -7
- data/lib/gov_kit/search_engines/technorati.rb +1 -2
- data/lib/gov_kit/search_engines/wikipedia.rb +5 -3
- data/lib/gov_kit/transparency_data.rb +43 -0
- data/lib/gov_kit.rb +4 -2
- metadata +32 -16
- data/.gitignore +0 -21
|
@@ -6,6 +6,7 @@ Govkit is a Ruby gem that provides simple access to open government APIs around
|
|
|
6
6
|
* [The Open States project](http://fiftystates-dev.sunlightlabs.com/), which has a RESTful API for accessing data about state legislators, bills, votes, etc.
|
|
7
7
|
* [Project Vote Smart](http://www.votesmart.org/services_api.php), which has an API with congressional addresses, etc.
|
|
8
8
|
* [Follow The Money](http://www.followthemoney.org/), whose API reveals campaign contribution data for state officials.
|
|
9
|
+
* [TransparencyData API](http://transparencydata.com/api) (campaign contributions only)
|
|
9
10
|
|
|
10
11
|
# Installation
|
|
11
12
|
|
data/Rakefile
CHANGED
|
@@ -19,12 +19,13 @@ begin
|
|
|
19
19
|
gem.name = "govkit"
|
|
20
20
|
gem.summary = %Q{Simple access to open government APIs around the web}
|
|
21
21
|
gem.description = %Q{Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.}
|
|
22
|
-
gem.email = "
|
|
22
|
+
gem.email = "develop@opencongress.org"
|
|
23
23
|
gem.homepage = "http://github.com/opengovernment/govkit"
|
|
24
24
|
gem.authors = ["Participatory Politics Foundation", "Srinivas Aki", "Carl Tashian"]
|
|
25
25
|
gem.add_dependency('httparty', '>= 0.5.2')
|
|
26
26
|
gem.add_dependency('json', '>= 1.4.3')
|
|
27
|
-
gem.add_dependency('
|
|
27
|
+
gem.add_dependency('nokogiri', '>= 1.4.4')
|
|
28
|
+
gem.add_dependency('fastercsv', '>= 1.5.3')
|
|
28
29
|
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
|
29
30
|
end
|
|
30
31
|
Jeweler::GemcutterTasks.new
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.5.
|
|
1
|
+
0.5.1
|
data/govkit.gemspec
CHANGED
|
@@ -1,83 +1,82 @@
|
|
|
1
1
|
# Generated by jeweler
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
|
-
# Instead, edit Jeweler::Tasks in Rakefile, and run
|
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |s|
|
|
7
7
|
s.name = %q{govkit}
|
|
8
|
-
s.version = "0.5.
|
|
8
|
+
s.version = "0.5.1"
|
|
9
9
|
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
11
|
s.authors = ["Participatory Politics Foundation", "Srinivas Aki", "Carl Tashian"]
|
|
12
|
-
s.date = %q{2010-
|
|
12
|
+
s.date = %q{2010-12-21}
|
|
13
13
|
s.description = %q{Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.}
|
|
14
|
-
s.email = %q{
|
|
14
|
+
s.email = %q{develop@opencongress.org}
|
|
15
15
|
s.extra_rdoc_files = [
|
|
16
16
|
"LICENSE",
|
|
17
|
-
|
|
17
|
+
"README.md"
|
|
18
18
|
]
|
|
19
19
|
s.files = [
|
|
20
20
|
".document",
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
21
|
+
"LICENSE",
|
|
22
|
+
"README.md",
|
|
23
|
+
"Rakefile",
|
|
24
|
+
"USAGE",
|
|
25
|
+
"VERSION",
|
|
26
|
+
"generators/govkit/govkit_generator.rb",
|
|
27
|
+
"generators/govkit/templates/govkit.rb",
|
|
28
|
+
"govkit.gemspec",
|
|
29
|
+
"init.rb",
|
|
30
|
+
"lib/generators/govkit/govkit_generator.rb",
|
|
31
|
+
"lib/generators/govkit/templates/govkit.rb",
|
|
32
|
+
"lib/gov_kit.rb",
|
|
33
|
+
"lib/gov_kit/acts_as_noteworthy.rb",
|
|
34
|
+
"lib/gov_kit/configuration.rb",
|
|
35
|
+
"lib/gov_kit/follow_the_money.rb",
|
|
36
|
+
"lib/gov_kit/open_congress.rb",
|
|
37
|
+
"lib/gov_kit/open_congress/bill.rb",
|
|
38
|
+
"lib/gov_kit/open_congress/blog_post.rb",
|
|
39
|
+
"lib/gov_kit/open_congress/news_post.rb",
|
|
40
|
+
"lib/gov_kit/open_congress/person.rb",
|
|
41
|
+
"lib/gov_kit/open_congress/person_stat.rb",
|
|
42
|
+
"lib/gov_kit/open_congress/roll_call.rb",
|
|
43
|
+
"lib/gov_kit/open_congress/roll_call_comparison.rb",
|
|
44
|
+
"lib/gov_kit/open_congress/voting_comparison.rb",
|
|
45
|
+
"lib/gov_kit/open_states.rb",
|
|
46
|
+
"lib/gov_kit/railtie.rb",
|
|
47
|
+
"lib/gov_kit/resource.rb",
|
|
48
|
+
"lib/gov_kit/search_engines.rb",
|
|
49
|
+
"lib/gov_kit/search_engines/google_blog.rb",
|
|
50
|
+
"lib/gov_kit/search_engines/google_news.rb",
|
|
51
|
+
"lib/gov_kit/search_engines/technorati.rb",
|
|
52
|
+
"lib/gov_kit/search_engines/wikipedia.rb",
|
|
53
|
+
"lib/gov_kit/transparency_data.rb",
|
|
54
|
+
"lib/gov_kit/vote_smart.rb",
|
|
55
|
+
"lib/govkit.rb",
|
|
56
|
+
"spec/fixtures/follow_the_money/business-page0.response",
|
|
57
|
+
"spec/fixtures/follow_the_money/business-page1.response",
|
|
58
|
+
"spec/fixtures/follow_the_money/contribution.response",
|
|
59
|
+
"spec/fixtures/follow_the_money/unauthorized.response",
|
|
60
|
+
"spec/fixtures/open_states/401.response",
|
|
61
|
+
"spec/fixtures/open_states/410.response",
|
|
62
|
+
"spec/fixtures/open_states/bill.response",
|
|
63
|
+
"spec/fixtures/open_states/bill_query.response",
|
|
64
|
+
"spec/fixtures/open_states/legislator.response",
|
|
65
|
+
"spec/fixtures/open_states/legislator_query.response",
|
|
66
|
+
"spec/fixtures/open_states/state.response",
|
|
67
|
+
"spec/follow_the_money_spec.rb",
|
|
68
|
+
"spec/open_states_spec.rb",
|
|
69
|
+
"spec/spec.opts",
|
|
70
|
+
"spec/spec_helper.rb"
|
|
71
71
|
]
|
|
72
72
|
s.homepage = %q{http://github.com/opengovernment/govkit}
|
|
73
|
-
s.rdoc_options = ["--charset=UTF-8"]
|
|
74
73
|
s.require_paths = ["lib"]
|
|
75
74
|
s.rubygems_version = %q{1.3.7}
|
|
76
75
|
s.summary = %q{Simple access to open government APIs around the web}
|
|
77
76
|
s.test_files = [
|
|
78
77
|
"spec/follow_the_money_spec.rb",
|
|
79
|
-
|
|
80
|
-
|
|
78
|
+
"spec/open_states_spec.rb",
|
|
79
|
+
"spec/spec_helper.rb"
|
|
81
80
|
]
|
|
82
81
|
|
|
83
82
|
if s.respond_to? :specification_version then
|
|
@@ -87,16 +86,19 @@ Gem::Specification.new do |s|
|
|
|
87
86
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
|
88
87
|
s.add_runtime_dependency(%q<httparty>, [">= 0.5.2"])
|
|
89
88
|
s.add_runtime_dependency(%q<json>, [">= 1.4.3"])
|
|
90
|
-
s.add_runtime_dependency(%q<
|
|
89
|
+
s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
|
|
90
|
+
s.add_runtime_dependency(%q<fastercsv>, [">= 1.5.3"])
|
|
91
91
|
else
|
|
92
92
|
s.add_dependency(%q<httparty>, [">= 0.5.2"])
|
|
93
93
|
s.add_dependency(%q<json>, [">= 1.4.3"])
|
|
94
|
-
s.add_dependency(%q<
|
|
94
|
+
s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
|
|
95
|
+
s.add_dependency(%q<fastercsv>, [">= 1.5.3"])
|
|
95
96
|
end
|
|
96
97
|
else
|
|
97
98
|
s.add_dependency(%q<httparty>, [">= 0.5.2"])
|
|
98
99
|
s.add_dependency(%q<json>, [">= 1.4.3"])
|
|
99
|
-
s.add_dependency(%q<
|
|
100
|
+
s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
|
|
101
|
+
s.add_dependency(%q<fastercsv>, [">= 1.5.3"])
|
|
100
102
|
end
|
|
101
103
|
end
|
|
102
104
|
|
|
@@ -2,7 +2,7 @@ if defined? GovKit
|
|
|
2
2
|
GovKit.configure do |config|
|
|
3
3
|
# Get an API key for Sunlight's Open States project here:
|
|
4
4
|
# http://services.sunlightlabs.com/accounts/register/
|
|
5
|
-
config.
|
|
5
|
+
config.sunlight_apikey = 'YOUR_SUNLIGHT_API_KEY'
|
|
6
6
|
|
|
7
7
|
##API key for Votesmart
|
|
8
8
|
# http://votesmart.org/services_api.php
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
module GovKit
|
|
2
2
|
class Configuration
|
|
3
|
-
attr_accessor :
|
|
3
|
+
attr_accessor :sunlight_apikey, :openstates_base_url, :transparency_data_base_url, :transparency_data_categories_url
|
|
4
4
|
attr_accessor :votesmart_apikey, :votesmart_base_url
|
|
5
5
|
attr_accessor :ftm_apikey, :ftm_base_url
|
|
6
6
|
attr_accessor :opencongress_apikey, :opencongress_base_url
|
|
@@ -11,6 +11,7 @@ module GovKit
|
|
|
11
11
|
def initialize
|
|
12
12
|
@openstates_apikey = @votesmart_apikey = @ftm_apikey = ''
|
|
13
13
|
@openstates_base_url = 'openstates.sunlightlabs.com/api/v1/'
|
|
14
|
+
@transparency_data_base_url = 'transparencydata.com/api/1.0/'
|
|
14
15
|
@votesmart_base_url = 'api.votesmart.org/'
|
|
15
16
|
@ftm_base_url = 'api.followthemoney.org/'
|
|
16
17
|
@opencongress_base_url = 'www.opencongress.org/'
|
|
@@ -18,6 +19,9 @@ module GovKit
|
|
|
18
19
|
@google_blog_base_url = 'blogsearch.google.com'
|
|
19
20
|
@google_news_base_url = 'news.google.com'
|
|
20
21
|
@wikipedia_base_url = 'en.wikipedia.org'
|
|
22
|
+
|
|
23
|
+
# Permant home for contribution category mappings
|
|
24
|
+
@transparency_data_categories_url = 'assets.transparencydata.org.s3.amazonaws.com/docs/catcodes.csv'
|
|
21
25
|
end
|
|
22
26
|
end
|
|
23
27
|
|
|
@@ -4,14 +4,13 @@ module GovKit
|
|
|
4
4
|
base_uri GovKit::configuration.ftm_base_url
|
|
5
5
|
|
|
6
6
|
def self.get_xml(path, options)
|
|
7
|
-
|
|
8
|
-
doc = Hpricot::XML(response)
|
|
7
|
+
doc = Nokogiri::XML(get(path, options))
|
|
9
8
|
|
|
10
9
|
e = doc.search("//error")
|
|
11
10
|
|
|
12
11
|
# Deal with whatever error comes back
|
|
13
12
|
if e.size > 0
|
|
14
|
-
raise case e.first.attributes['code']
|
|
13
|
+
raise case e.first.attributes['code'].value
|
|
15
14
|
when "100"
|
|
16
15
|
GovKit::NotAuthorized
|
|
17
16
|
when "300"
|
|
@@ -20,11 +19,17 @@ module GovKit
|
|
|
20
19
|
GovKit::ResourceNotFound
|
|
21
20
|
else
|
|
22
21
|
GovKit::InvalidRequest
|
|
23
|
-
end, e.first.attributes['text']
|
|
22
|
+
end, e.first.attributes['text'].value
|
|
24
23
|
end
|
|
25
24
|
|
|
26
25
|
doc
|
|
27
26
|
end
|
|
27
|
+
|
|
28
|
+
def self.stringify_values_of(result)
|
|
29
|
+
# result is an array of hashes, but all the values are Nokogiri::XML::Attr objects, not strings.
|
|
30
|
+
# We want them to be strings.
|
|
31
|
+
result.collect! { |r| r.inject({}) {|h, (k, v)| h[k] = v.to_s; h } }
|
|
32
|
+
end
|
|
28
33
|
end
|
|
29
34
|
|
|
30
35
|
module FollowTheMoney
|
|
@@ -37,15 +42,14 @@ module GovKit
|
|
|
37
42
|
|
|
38
43
|
doc = get_xml("/base_level.industries.list.php", :query => {:page => page_num})
|
|
39
44
|
|
|
40
|
-
next_page = doc.
|
|
45
|
+
next_page = doc.children.first.attributes['next_page'].value
|
|
41
46
|
|
|
42
47
|
page_num += 1
|
|
43
48
|
|
|
44
|
-
result += doc.search('//business_detail').collect
|
|
45
|
-
business.attributes.to_hash
|
|
46
|
-
end
|
|
49
|
+
result += doc.search('//business_detail').collect { |x| x.attributes }
|
|
47
50
|
end
|
|
48
51
|
|
|
52
|
+
stringify_values_of(result)
|
|
49
53
|
parse(result)
|
|
50
54
|
end
|
|
51
55
|
end
|
|
@@ -57,24 +61,22 @@ module GovKit
|
|
|
57
61
|
until next_page != "yes"
|
|
58
62
|
doc = get_xml("/candidates.contributions.php", :query => {"imsp_candidate_id" => nimsp_id, :page => page_num})
|
|
59
63
|
|
|
60
|
-
next_page = doc.
|
|
64
|
+
next_page = doc.children.first.attributes['next_page'].value
|
|
61
65
|
|
|
62
66
|
page_num += 1
|
|
63
67
|
|
|
64
|
-
result
|
|
65
|
-
contribution.attributes.to_hash
|
|
66
|
-
end
|
|
68
|
+
result += doc.search('//contribution').collect { |x| x.attributes }
|
|
67
69
|
end
|
|
70
|
+
|
|
71
|
+
stringify_values_of(result)
|
|
68
72
|
parse(result)
|
|
69
73
|
end
|
|
70
74
|
|
|
71
75
|
def self.top(nimsp_id)
|
|
72
76
|
doc = get_xml("/candidates.top_contributor.php", :query => {"imsp_candidate_id" => nimsp_id})
|
|
77
|
+
result = doc.search('//top_contributor').collect { |x| x.attributes }
|
|
73
78
|
|
|
74
|
-
result
|
|
75
|
-
contribution.attributes.to_hash
|
|
76
|
-
end
|
|
77
|
-
|
|
79
|
+
stringify_values_of(result)
|
|
78
80
|
parse(result)
|
|
79
81
|
end
|
|
80
82
|
end
|
|
@@ -82,11 +84,9 @@ module GovKit
|
|
|
82
84
|
class IndustryContribution < Contribution
|
|
83
85
|
def self.find(nimsp_id)
|
|
84
86
|
doc = get_xml("/candidates.industries.php", :query => {"imsp_candidate_id" => nimsp_id})
|
|
87
|
+
result = doc.search('//candidate_industry').collect { |x| x.attributes }
|
|
85
88
|
|
|
86
|
-
result
|
|
87
|
-
contribution.attributes.to_hash
|
|
88
|
-
end
|
|
89
|
-
|
|
89
|
+
stringify_values_of(result)
|
|
90
90
|
parse(result)
|
|
91
91
|
end
|
|
92
92
|
end
|
|
@@ -95,10 +95,9 @@ module GovKit
|
|
|
95
95
|
def self.find(nimsp_id)
|
|
96
96
|
doc = get_xml("/candidates.sectors.php", :query => {"imsp_candidate_id" => nimsp_id})
|
|
97
97
|
|
|
98
|
-
result = doc.search('//candidate_sector').collect
|
|
99
|
-
contribution.attributes.to_hash
|
|
100
|
-
end
|
|
98
|
+
result = doc.search('//candidate_sector').collect { |x| x.attributes }
|
|
101
99
|
|
|
100
|
+
stringify_values_of(result)
|
|
102
101
|
parse(result)
|
|
103
102
|
end
|
|
104
103
|
end
|
|
@@ -107,10 +106,9 @@ module GovKit
|
|
|
107
106
|
def self.find(nimsp_id)
|
|
108
107
|
doc = get_xml("/candidates.businesses.php", :query => {"imsp_candidate_id" => nimsp_id})
|
|
109
108
|
|
|
110
|
-
result = doc.search('//candidate_business').collect
|
|
111
|
-
contribution.attributes.to_hash
|
|
112
|
-
end
|
|
109
|
+
result = doc.search('//candidate_business').collect { |x| x.attributes }
|
|
113
110
|
|
|
111
|
+
stringify_values_of(result)
|
|
114
112
|
parse(result)
|
|
115
113
|
end
|
|
116
114
|
end
|
data/lib/gov_kit/open_states.rb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
module GovKit
|
|
2
2
|
class OpenStatesResource < Resource
|
|
3
|
-
default_params :output => 'json', :apikey => GovKit::configuration.
|
|
3
|
+
default_params :output => 'json', :apikey => GovKit::configuration.sunlight_apikey
|
|
4
4
|
base_uri GovKit::configuration.openstates_base_url
|
|
5
5
|
end
|
|
6
6
|
|
|
@@ -29,8 +29,8 @@ module GovKit
|
|
|
29
29
|
parse(response)
|
|
30
30
|
end
|
|
31
31
|
|
|
32
|
-
def self.latest(updated_since,
|
|
33
|
-
response = get('/bills/
|
|
32
|
+
def self.latest(updated_since, ops = {})
|
|
33
|
+
response = get('/bills/', :query => {:updated_since => updated_since.to_s}.merge(ops))
|
|
34
34
|
parse(response)
|
|
35
35
|
end
|
|
36
36
|
end
|
data/lib/gov_kit/resource.rb
CHANGED
|
@@ -17,7 +17,6 @@ module GovKit
|
|
|
17
17
|
# This method handles the basic responses we might get back from
|
|
18
18
|
# Net::HTTP. But if a service returns something other than a 404 when an object is not found,
|
|
19
19
|
# you'll need to handle that in the subclass.
|
|
20
|
-
|
|
21
20
|
raise ResourceNotFound, "Resource not found" unless !response.blank?
|
|
22
21
|
|
|
23
22
|
if response.class == HTTParty::Response
|
|
@@ -33,11 +32,10 @@ module GovKit
|
|
|
33
32
|
end
|
|
34
33
|
|
|
35
34
|
def self.instantiate(record)
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
instantiate_record(record)
|
|
35
|
+
if record.is_a?(Array)
|
|
36
|
+
instantiate_collection(record)
|
|
37
|
+
else
|
|
38
|
+
instantiate_record(record)
|
|
41
39
|
end
|
|
42
40
|
end
|
|
43
41
|
|
|
@@ -88,7 +86,7 @@ module GovKit
|
|
|
88
86
|
end
|
|
89
87
|
|
|
90
88
|
def find_or_create_resource_for(name)
|
|
91
|
-
resource_name = name.to_s.gsub(/^[_+]/,'').gsub(/^(
|
|
89
|
+
resource_name = name.to_s.gsub(/^[_+]/,'').gsub(/^(\-?\d)/, "n#{$1}").gsub(/\s/, '').camelize
|
|
92
90
|
if self.class.parents.size > 1
|
|
93
91
|
find_resource_in_modules(resource_name, self.class.parents)
|
|
94
92
|
else
|
|
@@ -6,10 +6,9 @@ module GovKit
|
|
|
6
6
|
host = GovKit::configuration.google_blog_base_url
|
|
7
7
|
path = "/blogsearch?hl=en&q=#{URI::encode(query)}&btnG=Search+Blogs&num=50"
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312',html))
|
|
9
|
+
doc = Nokogiri::HTML(make_request(host, path))
|
|
11
10
|
stories = doc.search("td.j")
|
|
12
|
-
titles = (doc/"a").select { |a| (a.attributes["id"] && a.attributes["id"].match(/p-(.*)/)) }
|
|
11
|
+
titles = (doc/"a").select { |a| (a.attributes["id"] && a.attributes["id"].value.match(/p-(.*)/)) }
|
|
13
12
|
|
|
14
13
|
mentions = []
|
|
15
14
|
|
|
@@ -17,12 +16,12 @@ module GovKit
|
|
|
17
16
|
mention = GovKit::Mention.new
|
|
18
17
|
t = titles.shift
|
|
19
18
|
|
|
20
|
-
mention.title =
|
|
21
|
-
mention.url = t.attributes["href"] if t
|
|
22
|
-
mention.date = story.at("font:nth(
|
|
23
|
-
mention.excerpt = (story.at("br + font").
|
|
24
|
-
mention.source = story.at("a.f1").
|
|
25
|
-
mention.url = story.at("a.f1").attributes["href"]
|
|
19
|
+
mention.title = t.text if t #.unpack("C*").pack("U*") if t
|
|
20
|
+
# mention.url = t.attributes["href"].value if t
|
|
21
|
+
mention.date = story.at("font:nth(1)").text.strip
|
|
22
|
+
mention.excerpt = (story.at("br + font").text) #.unpack("C*").pack("U*")
|
|
23
|
+
mention.source = story.at("a.f1").text
|
|
24
|
+
mention.url = story.at("a.f1").attributes["href"].value
|
|
26
25
|
|
|
27
26
|
mentions << mention
|
|
28
27
|
end
|
|
@@ -9,8 +9,7 @@ module GovKit
|
|
|
9
9
|
host = GovKit::configuration.google_news_base_url
|
|
10
10
|
path = "/news?hl=en&ned=us&q=#{URI::encode(query)}&btnG=Search+News&num=50"
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312',html))
|
|
12
|
+
doc = Nokogiri::HTML(make_request(host, path))
|
|
14
13
|
stories = doc.search("div.search-results > div.story")
|
|
15
14
|
|
|
16
15
|
mentions = []
|
|
@@ -18,11 +17,11 @@ module GovKit
|
|
|
18
17
|
stories.each do |story|
|
|
19
18
|
mention = GovKit::Mention.new
|
|
20
19
|
|
|
21
|
-
mention.title = story.at("h2.title a").
|
|
22
|
-
mention.url = story.at("h2.title a").attributes["href"]
|
|
23
|
-
mention.date = story.at("div.sub-title > span.date").
|
|
24
|
-
mention.source = story.at("div.sub-title > span.source").
|
|
25
|
-
mention.excerpt = story.at("div.body > div.snippet").
|
|
20
|
+
mention.title = story.at("h2.title a").text
|
|
21
|
+
mention.url = story.at("h2.title a").attributes["href"].value
|
|
22
|
+
mention.date = story.at("div.sub-title > span.date").text
|
|
23
|
+
mention.source = story.at("div.sub-title > span.source").text
|
|
24
|
+
mention.excerpt = story.at("div.body > div.snippet").text
|
|
26
25
|
|
|
27
26
|
mentions << mention
|
|
28
27
|
end
|
|
@@ -6,8 +6,7 @@ module GovKit
|
|
|
6
6
|
host = GovKit::configuration.technorati_base_url
|
|
7
7
|
path = "/search?key=#{GovKit::configuration.technorati_apikey}&limit=50&language=en&query=#{URI::encode(query)}"
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312',html))
|
|
9
|
+
doc = Nokogiri::HTML(make_request(host, path))
|
|
11
10
|
|
|
12
11
|
mentions = []
|
|
13
12
|
# doc.search("tapi/document/item").each do |i|
|
|
@@ -7,10 +7,12 @@ module GovKit
|
|
|
7
7
|
headers 'User-Agent' => 'GovKit +http://ppolitics.org'
|
|
8
8
|
|
|
9
9
|
def self.search(query, options={})
|
|
10
|
-
|
|
11
|
-
doc = Hpricot(Iconv.conv('utf-8//IGNORE', 'gb2312', response))
|
|
10
|
+
doc = Nokogiri::HTML(get("/wiki/#{query}"))
|
|
12
11
|
|
|
13
|
-
bio = doc.at('#bodyContent > p:first').
|
|
12
|
+
bio = doc.at('#bodyContent > p:first').text rescue ""
|
|
13
|
+
|
|
14
|
+
# Convert HTML => text.
|
|
15
|
+
# bio = Loofah.fragment(bio).text
|
|
14
16
|
|
|
15
17
|
return "" if bio =~ /may refer to:/
|
|
16
18
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
require 'fastercsv'
|
|
2
|
+
|
|
3
|
+
module GovKit
|
|
4
|
+
class TransparencyDataResource < Resource
|
|
5
|
+
default_params :apikey => GovKit::configuration.sunlight_apikey
|
|
6
|
+
base_uri GovKit::configuration.transparency_data_base_url
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
module TransparencyData
|
|
10
|
+
# See http://transparencydata.com/api/contributions/
|
|
11
|
+
# for complete query options.
|
|
12
|
+
class Contribution < TransparencyDataResource
|
|
13
|
+
def self.find(ops = {})
|
|
14
|
+
response = get('/contributions.json', :query => ops)
|
|
15
|
+
parse(response)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
class Entity < TransparencyDataResource
|
|
20
|
+
def self.find_by_id(id)
|
|
21
|
+
response = get("/entities/#{id}.json")
|
|
22
|
+
parse(response)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
class Categories
|
|
27
|
+
# Contribution category code mapping table, in CSV format
|
|
28
|
+
# Returns an array of hashes.
|
|
29
|
+
def self.all
|
|
30
|
+
categories = []
|
|
31
|
+
open(GovKit::configuration.transparency_data_categories_url) do |f|
|
|
32
|
+
f.each_line do |line|
|
|
33
|
+
FasterCSV.parse(line) do |row|
|
|
34
|
+
categories << row.fields
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
categories
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
end
|
data/lib/gov_kit.rb
CHANGED
|
@@ -1,21 +1,23 @@
|
|
|
1
1
|
$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__))) unless $LOAD_PATH.include?(File.expand_path(File.dirname(__FILE__)))
|
|
2
2
|
|
|
3
3
|
require 'active_support'
|
|
4
|
-
require '
|
|
4
|
+
require 'nokogiri'
|
|
5
5
|
require 'iconv'
|
|
6
6
|
require 'httparty'
|
|
7
|
+
require 'open-uri'
|
|
7
8
|
require 'json'
|
|
8
9
|
require 'gov_kit/configuration'
|
|
9
10
|
|
|
10
11
|
module GovKit
|
|
11
12
|
autoload :Resource, 'gov_kit/resource'
|
|
12
13
|
autoload :OpenStates, 'gov_kit/open_states'
|
|
14
|
+
autoload :TransparencyData, 'gov_kit/transparency_data'
|
|
13
15
|
autoload :VoteSmart, 'gov_kit/vote_smart'
|
|
14
16
|
autoload :ActsAsNoteworthy, 'gov_kit/acts_as_noteworthy'
|
|
15
17
|
autoload :FollowTheMoney, 'gov_kit/follow_the_money'
|
|
16
18
|
autoload :OpenCongress, 'gov_kit/open_congress'
|
|
17
19
|
autoload :SearchEngines, 'gov_kit/search_engines'
|
|
18
|
-
|
|
20
|
+
|
|
19
21
|
class Mention
|
|
20
22
|
attr_accessor :url, :excerpt, :title, :source, :date, :weight
|
|
21
23
|
end
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: govkit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
hash:
|
|
4
|
+
hash: 9
|
|
5
5
|
prerelease: false
|
|
6
6
|
segments:
|
|
7
7
|
- 0
|
|
8
8
|
- 5
|
|
9
|
-
-
|
|
10
|
-
version: 0.5.
|
|
9
|
+
- 1
|
|
10
|
+
version: 0.5.1
|
|
11
11
|
platform: ruby
|
|
12
12
|
authors:
|
|
13
13
|
- Participatory Politics Foundation
|
|
@@ -17,7 +17,7 @@ autorequire:
|
|
|
17
17
|
bindir: bin
|
|
18
18
|
cert_chain: []
|
|
19
19
|
|
|
20
|
-
date: 2010-
|
|
20
|
+
date: 2010-12-21 00:00:00 -08:00
|
|
21
21
|
default_executable:
|
|
22
22
|
dependencies:
|
|
23
23
|
- !ruby/object:Gem::Dependency
|
|
@@ -53,35 +53,50 @@ dependencies:
|
|
|
53
53
|
type: :runtime
|
|
54
54
|
version_requirements: *id002
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
|
-
name:
|
|
56
|
+
name: nokogiri
|
|
57
57
|
prerelease: false
|
|
58
58
|
requirement: &id003 !ruby/object:Gem::Requirement
|
|
59
59
|
none: false
|
|
60
60
|
requirements:
|
|
61
61
|
- - ">="
|
|
62
62
|
- !ruby/object:Gem::Version
|
|
63
|
-
hash:
|
|
63
|
+
hash: 15
|
|
64
64
|
segments:
|
|
65
|
-
-
|
|
66
|
-
-
|
|
67
|
-
-
|
|
68
|
-
version:
|
|
65
|
+
- 1
|
|
66
|
+
- 4
|
|
67
|
+
- 4
|
|
68
|
+
version: 1.4.4
|
|
69
69
|
type: :runtime
|
|
70
70
|
version_requirements: *id003
|
|
71
|
+
- !ruby/object:Gem::Dependency
|
|
72
|
+
name: fastercsv
|
|
73
|
+
prerelease: false
|
|
74
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
|
75
|
+
none: false
|
|
76
|
+
requirements:
|
|
77
|
+
- - ">="
|
|
78
|
+
- !ruby/object:Gem::Version
|
|
79
|
+
hash: 5
|
|
80
|
+
segments:
|
|
81
|
+
- 1
|
|
82
|
+
- 5
|
|
83
|
+
- 3
|
|
84
|
+
version: 1.5.3
|
|
85
|
+
type: :runtime
|
|
86
|
+
version_requirements: *id004
|
|
71
87
|
description: Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.
|
|
72
|
-
email:
|
|
88
|
+
email: develop@opencongress.org
|
|
73
89
|
executables: []
|
|
74
90
|
|
|
75
91
|
extensions: []
|
|
76
92
|
|
|
77
93
|
extra_rdoc_files:
|
|
78
94
|
- LICENSE
|
|
79
|
-
- README.
|
|
95
|
+
- README.md
|
|
80
96
|
files:
|
|
81
97
|
- .document
|
|
82
|
-
- .gitignore
|
|
83
98
|
- LICENSE
|
|
84
|
-
- README.
|
|
99
|
+
- README.md
|
|
85
100
|
- Rakefile
|
|
86
101
|
- USAGE
|
|
87
102
|
- VERSION
|
|
@@ -112,6 +127,7 @@ files:
|
|
|
112
127
|
- lib/gov_kit/search_engines/google_news.rb
|
|
113
128
|
- lib/gov_kit/search_engines/technorati.rb
|
|
114
129
|
- lib/gov_kit/search_engines/wikipedia.rb
|
|
130
|
+
- lib/gov_kit/transparency_data.rb
|
|
115
131
|
- lib/gov_kit/vote_smart.rb
|
|
116
132
|
- lib/govkit.rb
|
|
117
133
|
- spec/fixtures/follow_the_money/business-page0.response
|
|
@@ -134,8 +150,8 @@ homepage: http://github.com/opengovernment/govkit
|
|
|
134
150
|
licenses: []
|
|
135
151
|
|
|
136
152
|
post_install_message:
|
|
137
|
-
rdoc_options:
|
|
138
|
-
|
|
153
|
+
rdoc_options: []
|
|
154
|
+
|
|
139
155
|
require_paths:
|
|
140
156
|
- lib
|
|
141
157
|
required_ruby_version: !ruby/object:Gem::Requirement
|