statement 1.8.6 → 1.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f777f657b5861f10eb1731a84ba45e85a7d2a401
-  data.tar.gz: 2c36d668c73097b0108ea0fa66d1a236c597f63c
+  metadata.gz: 5d24d337a4b4bf8bc8f8789940543cec80eee884
+  data.tar.gz: b718f210fc6e0c7a0d30b81304fc64d40029bf28
 SHA512:
-  metadata.gz: 5d58a345789e59a07f5aa4bab2f7c6ce369242b0fae9508df85565c0f0c0b67286eca88e4ba8119a64cf1227ebfcd295688c822c25af52ca8ed2b2022b5d0f74
-  data.tar.gz: 205ed57eb16bfab790ac6fa549e54efb79b76f49a777c3be5d05a4d673cff1245c81dd058f8190801110ff08c57897bb72876b5d61a5c1c558f81b2d01bd887c
+  metadata.gz: 474fc04b2a15b38ddbecd92ce6af3438a1be3fb0baa1ae56456315fbc10ae4358646f2f19bf37219ef25b630308219fdf85e04152728a2fb320af484be5a1171
+  data.tar.gz: 093756d65a5eba4b75bf21fb05784eb6bea4898fd72106cdbd50e9d94449c158f469dce3b9a4db3ab6fd8977792173eea9072ce0b3eb7aa8d252e3313f4d91e9

data/lib/statement/scraper.rb

@@ -29,7 +29,7 @@ module Statement
     end
     
     def self.member_methods
-      [:capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :vitter, :donnelly, :inhofe, :reid, :palazzo, :roe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton, :wolf_sherman_mccaul, :welch, :sessions, :gabbard]
+      [:capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :vitter, :donnelly, :inhofe, :palazzo, :roe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton, :wolf_sherman_mccaul, :welch, :sessions, :gabbard]
     end
     
     def self.committee_methods
@@ -38,9 +38,9 @@ module Statement
     
     def self.member_scrapers
       year = Date.today.year
-      results = [capuano, cold_fusion(year, 0), conaway, chabot, susandavis, klobuchar, palazzo(page=1), roe(page=1), billnelson(year=year), 
+      results = [capuano, cold_fusion(year, 0), conaway, chabot, susandavis, klobuchar(year), palazzo(page=1), roe(page=1), billnelson(year=year), 
         document_query(page=1), document_query(page=2), swalwell(page=1), donnelly(year=year), crapo, coburn, boxer(start=1),
-        vitter(year=year), inhofe(year=year), reid, fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, wolf_sherman_mccaul, welch,
+        vitter(year=year), inhofe(year=year), fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, wolf_sherman_mccaul, welch,
         sessions(year=year), gabbard, pryor].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
@@ -318,16 +318,16 @@ module Statement
       results
     end
     
-    def self.klobuchar
+    def self.klobuchar(year)
       results = []
       base_url = "http://www.klobuchar.senate.gov/"
-      [2012,2013].each do |year|
+      [year.to_i-1,year.to_i].each do |year|
         year_url = base_url + "public/news-releases?MonthDisplay=0&YearDisplay=#{year}"
         doc = open_html(year_url)
         return if doc.nil?
         doc.xpath("//tr")[1..-1].each do |row|
-          next if row.children[2].children[0].text.strip == 'Title'
-          results << { :source => year_url, :url => row.children[2].children[0]['href'], :title => row.children[2].children[0].text.strip, :date => Date.strptime(row.children[0].text, "%m/%d/%y"), :domain => "klobuchar.senate.gov" }
+          next if row.children[3].children[0].text.strip == 'Title'
+          results << { :source => year_url, :url => row.children[3].children[0]['href'], :title => row.children[3].children[0].text.strip, :date => Date.strptime(row.children[1].text, "%m/%d/%y"), :domain => "klobuchar.senate.gov" }
         end
       end
       results
@@ -377,7 +377,7 @@ module Statement
       doc = open_html(url)
       return if doc.nil?
       doc.xpath("//tr").each do |row|
-        results << { :source => url, :url => base_url + row.children[2].children[0]['href'], :title => row.children[2].text.strip, :date => Date.parse(row.children[0].text.strip.gsub('-','/')), :domain => "crapo.senate.gov" }
+        results << { :source => url, :url => base_url + row.children[3].children[0]['href'], :title => row.children[3].text.strip, :date => Date.parse(row.children[1].text.strip.gsub('-','/')), :domain => "crapo.senate.gov" }
       end
       results
     end
@@ -388,8 +388,8 @@ module Statement
       doc = open_html(url)
       return if doc.nil?
       doc.xpath("//tr")[2..-1].each do |row|
-        next if row.text[0..3] == "Date"
-        results << { :source => url, :url => row.children[2].children[0]['href'], :title => row.children[2].text.strip, :date => Date.strptime(row.children[0].text.strip, "%m/%d/%y"), :domain => "fischer.senate.gov" }
+        next if row.text.strip[0..3] == "Date"
+        results << { :source => url, :url => row.children[3].children[0]['href'], :title => row.children[3].text.strip, :date => Date.strptime(row.children[1].text.strip, "%m/%d/%y"), :domain => "fischer.senate.gov" }
       end
       results
     end
@@ -400,8 +400,8 @@ module Statement
       doc = open_html(url)
       return if doc.nil?
       doc.xpath("//tr")[2..-1].each do |row|
-        next if row.text[0..3] == "Date"
-        results << { :source => url, :url => row.children[2].children[0]['href'], :title => row.children[2].text.strip, :date => Date.strptime(row.children[0].text.strip, "%m/%d/%y"), :domain => "coburn.senate.gov" }
+        next if row.text.strip[0..3] == "Date"
+        results << { :source => url, :url => row.children[3].children[0]['href'], :title => row.children[3].text.strip, :date => Date.strptime(row.children[1].text.strip, "%m/%d/%y"), :domain => "coburn.senate.gov" }
       end
       results
     end
@@ -426,7 +426,7 @@ module Statement
       return if doc.nil?
       doc.xpath("//tr")[1..-1].each do |row|
         next if row.text.strip.size < 30
-        results << { :source => url, :url => row.children[2].children[0]['href'].strip, :title => row.children[2].text, :date => Date.strptime(row.children[0].text, "%m/%d/%y"), :domain => domain}
+        results << { :source => url, :url => row.children[3].children[0]['href'].strip, :title => row.children[3].text, :date => Date.strptime(row.children[1].text, "%m/%d/%y"), :domain => domain}
       end
       results
     end
@@ -439,7 +439,7 @@ module Statement
       return if doc.nil?
       doc.xpath("//tr")[1..-1].each do |row|
         next if row.text.strip.size < 30
-        results << { :source => url, :url => "http://www.donnelly.senate.gov"+row.children[2].children[1]['href'].strip, :title => row.children[2].text.strip, :date => Date.strptime(row.children[0].text, "%m/%d/%y"), :domain => domain}
+        results << { :source => url, :url => "http://www.donnelly.senate.gov"+row.children[3].children[1]['href'].strip, :title => row.children[3].text.strip, :date => Date.strptime(row.children[1].text, "%m/%d/%y"), :domain => domain}
       end
       results
     end
@@ -452,19 +452,7 @@ module Statement
       return if doc.nil?
       doc.xpath("//tr")[1..-1].each do |row|
         next if row.text.strip.size < 30
-        results << { :source => url, :url => row.children[2].children[0]['href'].strip, :title => row.children[2].text, :date => Date.strptime(row.children[0].text, "%m/%d/%y"), :domain => domain}
-      end
-      results
-    end
-    
-    def self.reid
-      results = []
-      url = "http://www.reid.senate.gov/newsroom/press_releases.cfm"
-      domain = "www.reid.senate.gov"
-      doc = open_html(url)
-      return if doc.nil?
-      doc.xpath("//table[@id='CS_PgIndex_21891_21893']//tr")[1..-1].each do |row|
-        results << { :source => url, :url => "http://www.reid.senate.gov"+row.children[0].children[0]['href'], :title => row.children[0].children[0].text, :date => Date.parse(row.children[0].children[2].text), :domain => domain}
+        results << { :source => url, :url => row.children[3].children[0]['href'].strip, :title => row.children[3].text, :date => Date.strptime(row.children[1].text, "%m/%d/%y"), :domain => domain}
       end
       results
     end
@@ -500,8 +488,8 @@ module Statement
       doc = open_html(url)
       return if doc.nil?
       (doc/:tr)[1..-1].each do |row|
-        next if row.children.first.text == 'Date'
-        results << { :source => url, :date => Date.parse(row.children.first.text), :title => row.children[2].children.text, :url => row.children[2].children[0]['href'], :domain => domain}
+        next if row.children[1].text.strip == 'Date'
+        results << { :source => url, :date => Date.parse(row.children[1].text.strip), :title => row.children[3].children.text, :url => row.children[3].children[0]['href'], :domain => domain}
       end
       results
     end
@@ -513,8 +501,8 @@ module Statement
       doc = open_html(url)
       return if doc.nil?
       (doc/:tr)[1..-1].each do |row|
-        next if row.children.first.text == 'Date'
-        results << { :source => url, :date => Date.parse(row.children.first.text), :title => row.children[2].children.text, :url => row.children[2].children[0]['href'], :domain => domain}
+        next if row.children[1].text.strip == 'Date'
+        results << { :source => url, :date => Date.parse(row.children[1].text), :title => row.children[3].children.text, :url => row.children[3].children[0]['href'], :domain => domain}
       end
       results
     end

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "1.8.6"
+  VERSION = "1.8.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 1.8.6
+  version: 1.8.7
 platform: ruby
 authors:
 - Derek Willis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-09 00:00:00.000000000 Z
+date: 2014-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler