govkit 0.6.1 → 0.7.0

data/.rspec

@@ -0,0 +1,3 @@
+--colour
+--format nested
+--backtrace

data/Gemfile

@@ -0,0 +1,13 @@
+source 'http://rubygems.org'
+
+gemspec
+
+gem "active_support"
+gem "nokogiri"
+gem "httparty"
+gem "i18n"
+
+group :test do
+  gem "rspec"
+  gem "fakeweb"
+end

data/README.md

@@ -59,9 +59,4 @@ For Canadian open government data, our friends up north have created a [govkit-c
 
 Let us know if you'd like to build a govkit for your region! We'd love to link to you. Your gem should be called, for example, govkit-uk (ISO 3166 country code), and your methods should live in the GovKit::GB namespace (for example).
 
-# TODOs
-
-  * Guaranteed Eachability: If an API call is expected to return zero or more records, then GovKit should always return an array or nil to the caller, not a single object, even if there's only one record returned in this particular API call.
-  * Migration & documentation for acts_as_noteworthy
-
 Copyright (c) 2010 Participatory Politics Foundation, released under the MIT license

data/Rakefile

@@ -22,7 +22,7 @@ begin
     gem.email = "develop@opencongress.org"
     gem.homepage = "http://github.com/opengovernment/govkit"
     gem.authors = ["Participatory Politics Foundation", "Srinivas Aki", "Carl Tashian"]
-    gem.add_dependency('httparty', '>= 0.5.2')
+    gem.add_dependency('httparty', '>= 0.7.4')
     gem.add_dependency('json', '>= 1.4.3')
     gem.add_dependency('nokogiri', '>= 1.4.4')
     gem.add_dependency('fastercsv', '>= 1.5.3')

data/VERSION

@@ -1 +1 @@
-0.6.1
+0.7.0

data/generators/govkit/templates/govkit.rb

@@ -19,6 +19,9 @@ if defined? GovKit
     # Technorati API key
     config.technorati_apikey = 'YOUR_TECHNORATI_APIKEY'
     
+    # Bing App ID
+    config.bing_appid = 'YOUR_BING_APPID'
+    
     # Other things you could set here include alternate URLs for
     # the APIs. See GovKit::Configuration for available attributes.
   end

data/govkit.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{govkit}
-  s.version = "0.6.1"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Participatory Politics Foundation", "Srinivas Aki", "Carl Tashian"]
-  s.date = %q{2011-03-03}
+  s.date = %q{2011-06-29}
   s.description = %q{Govkit lets you quickly get encapsulated Ruby objects for common open government APIs. We're starting with Sunlight's Open States API and the Project Vote Smart API.}
   s.email = %q{develop@opencongress.org}
   s.extra_rdoc_files = [
@@ -18,6 +18,8 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".document",
+    ".rspec",
+    "Gemfile",
     "LICENSE",
     "README.md",
     "Rakefile",
@@ -46,6 +48,7 @@ Gem::Specification.new do |s|
     "lib/gov_kit/railtie.rb",
     "lib/gov_kit/resource.rb",
     "lib/gov_kit/search_engines.rb",
+    "lib/gov_kit/search_engines/bing.rb",
     "lib/gov_kit/search_engines/google_blog.rb",
     "lib/gov_kit/search_engines/google_news.rb",
     "lib/gov_kit/search_engines/technorati.rb",
@@ -53,48 +56,73 @@ Gem::Specification.new do |s|
     "lib/gov_kit/transparency_data.rb",
     "lib/gov_kit/vote_smart.rb",
     "lib/govkit.rb",
+    "spec/fixtures/bing/news_search.response",
+    "spec/fixtures/bing/no_results.response",
     "spec/fixtures/follow_the_money/business-page0.response",
     "spec/fixtures/follow_the_money/business-page1.response",
     "spec/fixtures/follow_the_money/contribution.response",
     "spec/fixtures/follow_the_money/unauthorized.response",
+    "spec/fixtures/open_congress/person.response",
     "spec/fixtures/open_states/401.response",
+    "spec/fixtures/open_states/404.response",
     "spec/fixtures/open_states/410.response",
     "spec/fixtures/open_states/bill.response",
     "spec/fixtures/open_states/bill_query.response",
+    "spec/fixtures/open_states/committee_find.response",
+    "spec/fixtures/open_states/committee_query.response",
     "spec/fixtures/open_states/legislator.response",
     "spec/fixtures/open_states/legislator_query.response",
     "spec/fixtures/open_states/state.response",
+    "spec/fixtures/search_engines/google_news.response",
+    "spec/fixtures/transparency_data/contributions.response",
+    "spec/fixtures/transparency_data/entities_search.response",
+    "spec/fixtures/transparency_data/entities_search_limit_0.response",
+    "spec/fixtures/transparency_data/entities_search_limit_1.response",
+    "spec/fixtures/transparency_data/grants_find_all.response",
+    "spec/fixtures/transparency_data/lobbyists_find_all.response",
     "spec/follow_the_money_spec.rb",
+    "spec/open_congress_spec.rb",
     "spec/open_states_spec.rb",
-    "spec/spec.opts",
-    "spec/spec_helper.rb"
+    "spec/search_engines_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/transparency_data_spec.rb"
   ]
   s.homepage = %q{http://github.com/opengovernment/govkit}
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.5.2}
+  s.rubygems_version = %q{1.6.2}
   s.summary = %q{Simple access to open government APIs around the web}
-  s.test_files = [
-    "spec/follow_the_money_spec.rb",
-    "spec/open_states_spec.rb",
-    "spec/spec_helper.rb"
-  ]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<httparty>, [">= 0.5.2"])
+      s.add_runtime_dependency(%q<govkit>, [">= 0"])
+      s.add_runtime_dependency(%q<active_support>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_runtime_dependency(%q<httparty>, [">= 0"])
+      s.add_runtime_dependency(%q<i18n>, [">= 0"])
+      s.add_runtime_dependency(%q<httparty>, [">= 0.7.4"])
       s.add_runtime_dependency(%q<json>, [">= 1.4.3"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_runtime_dependency(%q<fastercsv>, [">= 1.5.3"])
     else
-      s.add_dependency(%q<httparty>, [">= 0.5.2"])
+      s.add_dependency(%q<govkit>, [">= 0"])
+      s.add_dependency(%q<active_support>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<httparty>, [">= 0"])
+      s.add_dependency(%q<i18n>, [">= 0"])
+      s.add_dependency(%q<httparty>, [">= 0.7.4"])
       s.add_dependency(%q<json>, [">= 1.4.3"])
       s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
       s.add_dependency(%q<fastercsv>, [">= 1.5.3"])
     end
   else
-    s.add_dependency(%q<httparty>, [">= 0.5.2"])
+    s.add_dependency(%q<govkit>, [">= 0"])
+    s.add_dependency(%q<active_support>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<httparty>, [">= 0"])
+    s.add_dependency(%q<i18n>, [">= 0"])
+    s.add_dependency(%q<httparty>, [">= 0.7.4"])
     s.add_dependency(%q<json>, [">= 1.4.3"])
     s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
     s.add_dependency(%q<fastercsv>, [">= 1.5.3"])

data/lib/gov_kit/acts_as_noteworthy.rb

@@ -6,19 +6,18 @@ module GovKit::ActsAsNoteworthy
 
   module ActMethods
     def acts_as_noteworthy(options={})
-      options[:keywords] ||= []
-
       class_inheritable_accessor :options
       self.options = options
 
       unless included_modules.include? InstanceMethods
         instance_eval do
-          has_many :mentions, :as => :owner
+          has_many :mentions, :as => :owner, :order => 'date desc'
 
           with_options :as => :owner, :class_name => "Mention" do |c|
             c.has_many :google_news_mentions, :conditions => {:search_source => "Google News"}, :order => 'date desc'
             c.has_many :google_blog_mentions, :conditions => {:search_source => "Google Blogs"}, :order => 'date desc'
             c.has_many :technorati_mentions, :conditions => {:search_source => "Technorati"}, :order => 'date desc'
+            c.has_many :bing_mentions, :conditions => {:search_source => "Bing"}, :order => 'date desc'
           end
         end
 
@@ -34,16 +33,23 @@ module GovKit::ActsAsNoteworthy
   module InstanceMethods
 
     def raw_mentions
-      params = self.options[:keywords].clone
-      attributes = self.options[:with].clone
+      opts = self.options.clone
+      attributes = opts.delete(:with)
+
+      if opts[:geo]
+        opts[:geo] = self.instance_eval("#{opts[:geo]}")
+      end
 
+      query = []
       attributes.each do |attr|
-        params << self.instance_eval("#{attr}")
+        query << self.instance_eval("#{attr}")
       end
+
       {
-        :google_news => GovKit::SearchEngines::GoogleNews.search(params),
-        :google_blogs => GovKit::SearchEngines::GoogleBlog.search(params),
-        :technorati => GovKit::SearchEngines::Technorati.search(params)
+        :google_news => GovKit::SearchEngines::GoogleNews.search(query, opts),
+        :google_blogs => GovKit::SearchEngines::GoogleBlog.search(query, opts),
+#        :technorati => GovKit::SearchEngines::Technorati.search(query),
+        :bing => GovKit::SearchEngines::Bing.search(query, opts)
       }
     end
   end

data/lib/gov_kit/configuration.rb

@@ -2,11 +2,12 @@ module GovKit
   class Configuration
     attr_accessor :sunlight_apikey, :openstates_base_url, :transparency_data_base_url, :transparency_data_categories_url
     attr_accessor :votesmart_apikey, :votesmart_base_url
-    attr_accessor :ftm_apikey, :ftm_base_url
+    attr_accessor :openstates_apikey, :ftm_apikey, :ftm_base_url
     attr_accessor :opencongress_apikey, :opencongress_base_url
     attr_accessor :technorati_apikey, :technorati_base_url
     attr_accessor :google_blog_base_url, :google_news_base_url
     attr_accessor :wikipedia_base_url
+    attr_accessor :bing_appid, :bing_base_url
 
     def initialize
       @openstates_apikey = @votesmart_apikey = @ftm_apikey = ''
@@ -16,6 +17,7 @@ module GovKit
       @ftm_base_url = 'api.followthemoney.org/'
       @opencongress_base_url = 'www.opencongress.org/'
       @technorati_base_url = 'api.technorati.com'
+      @bing_base_url = 'api.search.live.net'
       @google_blog_base_url = 'blogsearch.google.com'
       @google_news_base_url = 'news.google.com'
       @wikipedia_base_url = 'en.wikipedia.org'

data/lib/gov_kit/open_congress/bill.rb

@@ -2,13 +2,11 @@ module GovKit
   module OpenCongress
     class Bill < OpenCongressObject
       attr_accessor :bill_type, :id, :introduced, :last_speech, :last_vote_date, :last_vote_roll, :last_vote_where, :last_action, :number, :plain_language_summary, :session, :sponsor, :co_sponsors, :title_full_common, :status, :most_recent_actions, :bill_titles, :recent_blogs, :recent_news, :ident
-    
+      
       def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if Bill.instance_methods.include? key
-        end      
+        super Bill, params
       end
-    
+      
       def ident
         "#{session}-#{bill_type}#{number}"
       end

data/lib/gov_kit/open_congress/blog_post.rb

@@ -4,14 +4,10 @@ module GovKit
     class BlogPost < OpenCongressObject
     
       attr_accessor :title, :date, :url, :source_url, :excerpt, :source, :average_rating
-    
-    
-      def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if BlogPost.instance_methods.include? key
-        end      
-      end      
       
+      def initialize(params)
+        super BlogPost, params
+      end
       
     end
   

data/lib/gov_kit/open_congress/news_post.rb

@@ -4,14 +4,10 @@ module GovKit
     class NewsPost < OpenCongressObject
     
       attr_accessor :title, :date, :url, :source_url, :excerpt, :source, :average_rating
-    
-    
-      def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if NewsPost.instance_methods.include? key
-        end      
-      end      
       
+      def initialize(params)
+        super NewsPost, params
+      end
       
     end
   

data/lib/gov_kit/open_congress/person.rb

@@ -7,11 +7,9 @@ module GovKit
                     :name, :nickname, :osid, :party, :religion, :state, :title, :unaccented_name, :url, :user_approval,
                     :youtube_id, :oc_user_comments, :oc_users_tracking, :abstains_percentage, :with_party_percentage, :recent_news,
                     :recent_blogs, :person_stats
-    
+      
       def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if Person.instance_methods.include? key
-        end      
+        super Person, params
       end
     
       def self.find(params)

data/lib/gov_kit/open_congress/person_stat.rb

@@ -4,14 +4,9 @@ module GovKit
     
       attr_accessor :votes_most_often_with_id, :opposing_party_votes_most_often_with_id, :votes_least_often_with_id, :same_party_votes_least_often_with_id, :party_votes_percentage, :abstains_percentage, :abstains_percentage_rank, :party_votes_percentage_rank, :sponsored_bills, :cosponsored_bills, :abstains, :sponsored_bills_passed_rank, :cosponsored_bills_passed_rank, :sponsored_bills_passed, :cosponsored_bills_passed, :sponsored_bills_rank, :cosponsored_bills_rank
     
-    
       def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if PersonStat.instance_methods.include? key
-        end      
-      end      
-      
-      
+        super PersonStat, params
+      end
     end
   
   end

data/lib/gov_kit/open_congress/roll_call.rb

@@ -4,13 +4,11 @@ module GovKit
     
       attr_accessor :abstains, :presents, :roll_type, :title, :question, :republican_position, :democratic_position,
                     :amendment_id, :ayes, :nays, :bill, :date, :number, :id, :required, :where
-    
+       
       def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if RollCall.instance_methods.include? key
-        end      
-      end    
-   
+        super RollCall, params
+      end
+      
     end
   end
 end

data/lib/gov_kit/open_congress/roll_call_comparison.rb

@@ -5,10 +5,7 @@ module GovKit
       attr_accessor :roll_call, :person1, :person2
     
       def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if RollCallComparison.instance_methods.include? key
-        end
-      
+        super RollCallComparison, params
       
         set_people
         set_roll_call

data/lib/gov_kit/open_congress/voting_comparison.rb

@@ -7,9 +7,7 @@ module GovKit
                     :other_votes
     
       def initialize(params)
-        params.each do |key, value|
-          instance_variable_set("@#{key}", value) if VotingComparison.instance_methods.include? key
-        end
+        super VotingComparison, params
         
         set_people
         set_votes

data/lib/gov_kit/open_congress.rb

@@ -1,6 +1,7 @@
 require 'nokogiri'
 require 'open-uri'
 require 'json'
+require 'CGI'
 
 module GovKit::OpenCongress
   autoload :Bill,             'gov_kit/open_congress/bill'
@@ -12,6 +13,13 @@ module GovKit::OpenCongress
   autoload :PersonStat,       'gov_kit/open_congress/person_stat'
 
   class OpenCongressObject
+    
+    def initialize(obj, params)
+      params.each do |key, value|
+        key = key.to_sym if RUBY_VERSION[0,3] == "1.9"
+        instance_variable_set("@#{key}", value) if obj.instance_methods.include? key
+      end      
+    end
 
     def self.construct_url(api_method, params)
       url = nil

data/lib/gov_kit/open_states.rb

@@ -1,34 +1,76 @@
 module GovKit
+
+  # Parent class for OpenStates resources
+  # See http://openstates.sunlightlabs.com/api/
   class OpenStatesResource < Resource
+
+    # Uses default_params from the HTTParty gem.
+    # See HTTParty::ClassMethods:
+    # http://rubydoc.info/gems/httparty/0.7.4/HTTParty/ClassMethods#default_params-instance_method
     default_params :output => 'json', :apikey => GovKit::configuration.sunlight_apikey
     base_uri GovKit::configuration.openstates_base_url
+
+    # Do a GET query, with optional parameters.
+    #
+    # OpenStates returns a 404 error when a query
+    # returns nothing.
+    #
+    # So, if a query result is a resource not found error,
+    # we return an empty set.
+    def self.get_uri(uri, options={})
+      begin
+        response = get(uri, options)
+        result = parse(response)
+      rescue ResourceNotFound
+        return []
+      end
+      result
+    end
+
   end
 
+  # Ruby module for interacting with the Open States Project API
+  # See http://openstates.sunlightlabs.com/api/
+  # Most +find+ and +search+ methods: 
+  # * call HTTParty::ClassMethods#get
+  # * which returns an HTTParty::Response object
+  # * which is passed to GovKit::Resource#parse
+  # * which uses the response to populate a Resource
+  #
   module OpenStates
     ROLE_MEMBER = "member"
     ROLE_COMMITTEE_MEMBER = "committee member"
     CHAMBER_UPPER = "upper"
     CHAMBER_LOWER = "lower"
 
+    # The State class represents the state data returned from Open States.
+    #
+    # For details about fields returned, see the Open States documentation, at 
+    # http://openstates.sunlightlabs.com/api/metadata/, 
+    #
     class State < OpenStatesResource
       def self.find_by_abbreviation(abbreviation)
-        response = get("/metadata/#{abbreviation}/")
-        parse(response)
+        get_uri("/metadata/#{abbreviation}/")
       end
     end
 
+    # The Bill class represents the bill data returned from Open States.
+    #
+    # For details about fields returned, see the Open States documentation, at 
+    # http://openstates.sunlightlabs.com/api/bills/, 
+    #
     class Bill < OpenStatesResource
       # http://openstates.sunlightlabs.com/api/v1/bills/ca/20092010/AB 667/
       def self.find(state_abbrev, session, bill_id, chamber = '')
         escaped_bill_id = bill_id.gsub(/ /, '%20')
         escaped_session = session.gsub(/ /, '%20')
-        response = get("/bills/#{state_abbrev.downcase}/#{escaped_session}/#{chamber.blank? ? '' : chamber + '/'}#{escaped_bill_id}/")
-        parse(response)
+
+        get_uri("/bills/#{state_abbrev.downcase}/#{escaped_session}/#{chamber.blank? ? '' : chamber + '/'}#{escaped_bill_id}/")
       end
 
       def self.search(query, options = {})
-        response = get('/bills/', :query => {:q => query}.merge(options))
-        parse(response)
+        result = get_uri('/bills/', :query => {:q => query}.merge(options))
+        return Array(result)
       end
 
       def self.latest(updated_since, ops = {})
@@ -37,27 +79,34 @@ module GovKit
       end
     end
 
+    # The Legislator class represents the legislator data returned from Open States.
+    #
+    # For details about fields returned, see the Open States documentation, at 
+    # http://openstates.sunlightlabs.com/api/legislators/, 
+    #
     class Legislator < OpenStatesResource
       def self.find(legislator_id)
-        response = get("/legislators/#{legislator_id}/")
-        parse(response)
+        get_uri("/legislators/#{legislator_id}/")
       end
 
       def self.search(options = {})
-        response = get('/legislators/', :query => options)
-        parse(response)
+        result = get_uri('/legislators/', :query => options)
+        return Array(result)
       end
     end
     
+    # The Committee class represents the committee data returned from Open States.
+    #
+    # For details about fields returned, see the Open States documentation, at 
+    # http://openstates.sunlightlabs.com/api/committees/, 
+    #
     class Committee < OpenStatesResource
       def self.find(committee_id)
-        response = get("/committees/#{committee_id}/")
-        parse(response)
+        get_uri("/committees/#{committee_id}/")
       end
 
       def self.search(options = {})
-        response = get('/committees/', :query => options)
-        parse(response)
+        get_uri('/committees/', :query => options)
       end
     end
     

data/lib/gov_kit/railtie.rb

@@ -12,6 +12,10 @@ module GovKit
     end
   end
 
+  # This class exists in order to run its insert method while
+  # Rails is loading.
+  # This then adds GovKit::ActsAsNoteworthy to ActiveRecord::Base.
+  # See http://api.rubyonrails.org/classes/Rails/Railtie.html
   class Railtie
     def self.insert
       ActiveRecord::Base.send(:include, GovKit::ActsAsNoteworthy)