statement 1.0.9 → 1.0.10

data/Rakefile

@@ -5,7 +5,7 @@ Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
+  test.libs << 'lib' << 'spec'
   test.pattern = 'spec/*.rb'
   test.verbose = true
 end

data/lib/statement/scraper.rb

@@ -29,7 +29,7 @@ module Statement
     end
     
     def self.member_methods
-      [:capuano, :cold_fusion, :conaway, :susandavis, :faleomavaega, :freshman_senators, :klobuchar, :lujan, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :mccain, :vitter, :donnelly, :inhofe, :levin, :reid, :palazzo, :document_query, :farenthold, :swalwell, :fischer]
+      [:capuano, :cold_fusion, :conaway, :susandavis, :freshman_senators, :klobuchar, :lujan, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :mccain, :vitter, :donnelly, :inhofe, :levin, :reid, :palazzo, :document_query, :farenthold, :swalwell, :fischer]
     end
     
     def self.committee_methods
@@ -38,7 +38,7 @@ module Statement
     
     def self.member_scrapers
       year = Date.today.year
-      results = [freshman_senators, capuano, cold_fusion(year, 0), conaway, susandavis, faleomavaega, klobuchar, lujan, palazzo(page=1), billnelson(year=year), 
+      results = [freshman_senators, capuano, cold_fusion(year, 0), conaway, susandavis, klobuchar, lujan, palazzo(page=1), billnelson(year=year), 
         document_query(page=1), document_query(page=2), farenthold(year), swalwell(page=1), donnelly(year=year), crapo, coburn, boxer(start=1), mccain(year=year), 
         vitter(year=year), inhofe(year=year), reid, fischer].flatten
       Utils.remove_generic_urls!(results)
@@ -305,20 +305,9 @@ module Statement
       results
     end
     
-    def self.faleomavaega
-      results = []
-      base_url = "http://www.house.gov/faleomavaega/news-press.shtml"
-      doc = open_html(base_url)
-      return if doc.nil?
-      doc.xpath("//li[@type='disc']").each do |row|
-        results << { :source => base_url, :url => "http://www.house.gov/" + row.children[0]['href'], :title => row.children[0].text.gsub(/[u201cu201d]/, '').split('Washington, D.C.').last, :date => Date.parse(row.children[1].text), :domain => "house.gov/faleomavaega" }
-      end
-      results
-    end
-    
     def self.freshman_senators
       results = []
-      ['heinrich','murphy','cruz'].each do |senator|
+      ['murphy','cruz'].each do |senator|
         base_url = "http://www.#{senator}.senate.gov/"
         doc = open_html(base_url+'press.cfm?maxrows=200&startrow=1&&type=1')
         return if doc.nil?
@@ -334,11 +323,12 @@ module Statement
       results = []
       base_url = "http://www.klobuchar.senate.gov/"
       [2012,2013].each do |year|
-        year_url = base_url + "newsreleases.cfm?year=#{year}"
+        year_url = base_url + "public/news-releases?MonthDisplay=0&YearDisplay=#{year}"
         doc = open_html(year_url)
         return if doc.nil?
-        doc.xpath("//dt").each do |row|
-          results << { :source => year_url, :url => base_url + row.next.children[0]['href'], :title => row.next.text.strip.gsub(/[u201cu201d]/, '').split.join(' '), :date => Date.strptime(row.text, "%m/%d/%y"), :domain => "klobuchar.senate.gov" }
+        doc.xpath("//tr")[1..-1].each do |row|
+          next if row.children[2].children[0].text.strip == 'Title'
+          results << { :source => year_url, :url => row.children[2].children[0]['href'], :title => row.children[2].children[0].text.strip, :date => Date.strptime(row.children[0].text, "%m/%d/%y"), :domain => "klobuchar.senate.gov" }
         end
       end
       results

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "1.0.9"
+  VERSION = "1.0.10"
 end

data/spec/statement_spec.rb

@@ -6,28 +6,28 @@ include Statement
 describe Statement do
   it "parses an rss feed" do
     @feed_url = "http://ruiz.house.gov/rss.xml"
-    stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "ruiz_rss.xml")), :status => 200)
+    WebMock.stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "ruiz_rss.xml")), :status => 200)
     @results = Feed.from_rss(@feed_url)
     @results.first[:domain].must_equal "ruiz.house.gov"
   end
   
   it "parses House GOP press release page" do
     @feed_url = "http://www.gop.gov/republicans/news?offset=03/29/13"
-    stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "house_gop_releases.html")), :status => 200)
+    WebMock.stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "house_gop_releases.html")), :status => 200)
     @results = Scraper.house_gop(@feed_url)
     @results.first[:source].must_equal @feed_url
   end
   
   it "does not attempt to parse dates when none are present" do
     @feed_url = "http://culberson.house.gov/feed/rss/"
-    stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "culberson_rss.xml")), :status => 200)
+    WebMock.stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "culberson_rss.xml")), :status => 200)
     @results = Feed.from_rss(@feed_url)
     @results.first[:date].must_equal nil
   end
   
   it "parses invalid RSS" do
     @feed_url = "http://www.burr.senate.gov/public/index.cfm?FuseAction=RSS.Feed"
-    stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "richard_burr.xml")), :status => 200)
+    WebMock.stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "richard_burr.xml")), :status => 200)
     @results = Feed.from_rss(@feed_url)
     @results.first[:url].must_equal "http://www.burr.senate.gov/public/index.cfm?FuseAction=PressOffice.PressReleases&Type=Press Release&ContentRecord_id=65dbea38-d64c-6208-ef8f-2b000e899b3a"
     @results.first[:date].to_s.must_equal "2013-05-02"
@@ -35,33 +35,29 @@ describe Statement do
 
   it "handles relative URLs" do
     @feed_url = "http://www.gop.gov/republicans/news?offset=03/29/13"
-    stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "house_gop_releases.html")), :status => 200)
+    WebMock.stub_request(:any, @feed_url).to_return(:body => File.new(File.join(File.dirname(__FILE__), "house_gop_releases.html")), :status => 200)
     @results = Scraper.house_gop(@feed_url)
     @results.last[:url].must_equal "http://www.gop.gov/republicans/other/relative_url_test.html"
   end
   
   it "scrapes a senate cold fusion page" do
     @url = "http://www.billnelson.senate.gov/news/media.cfm?year=2013"
-    stub_request(:any, @url).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'bill_nelson_press.html')), :status => 200)
+    WebMock.stub_request(:any, @url).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'bill_nelson_press.html')), :status => 200)
     @results = Scraper.billnelson(year=2013)
     @results.last[:url].must_equal "http://www.billnelson.senate.gov/news/details.cfm?id=338190&"
   end
   
-  it "scrapes vitter and cowan pages for 2013" do
+  it "scrapes vitter pages for 2013" do
     @vitter = "http://www.vitter.senate.gov/newsroom/press?year=2013"
-    @cowan = "http://www.cowan.senate.gov/press?year=2013"
-    stub_request(:any, @vitter).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'vitter_press.html')), :status => 200)
-    stub_request(:any, @cowan).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'cowan_press.html')), :status => 200)
-    @results = Scraper.vitter_cowan(year=2013)
-    @results.map{|r| r[:domain]}.uniq.must_equal ["www.vitter.senate.gov", "www.cowan.senate.gov"]
+    WebMock.stub_request(:any, @vitter).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'vitter_press.html')), :status => 200)
+    @results = Scraper.vitter(year=2013)
+    @results.map{|r| r[:domain]}.uniq.must_equal ["www.vitter.senate.gov"]
   end
   
   it "only scrapes vitter page for 2012" do
     @vitter = "http://www.vitter.senate.gov/newsroom/press?year=2012"
-    @cowan = "http://www.cowan.senate.gov/press?year=2012"
-    stub_request(:any, @vitter).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'vitter_press.html')), :status => 200)
-    stub_request(:any, @cowan).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'cowan_press.html')), :status => 200)
-    @results = Scraper.vitter_cowan(year=2012)
+    WebMock.stub_request(:any, @vitter).to_return(:body => File.new(File.join(File.dirname(__FILE__), 'vitter_press.html')), :status => 200)
+    @results = Scraper.vitter(year=2012)
     @results.map{|r| r[:domain]}.uniq.must_equal ["www.vitter.senate.gov"]    
   end
   

data/statement.gemspec

@@ -19,9 +19,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency 'webmock'
-  spec.add_development_dependency 'minitest'
+  spec.add_dependency "rake"
+  spec.add_dependency 'webmock'
+  spec.add_dependency 'minitest'
   spec.add_dependency "american_date"
   if RUBY_VERSION > "1.8.7"
     spec.add_dependency "nokogiri"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 1.0.9
+  version: 1.0.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-24 00:00:00.000000000 Z
+date: 2013-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -35,7 +35,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -51,7 +51,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -67,7 +67,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -214,7 +214,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -946704021369605610
+      hash: -2139780170107598593
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -223,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -946704021369605610
+      hash: -2139780170107598593
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25