statement 1.7.1 → 1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5118ff388b119a973d8c2c8151fd8fd0574c9752
-  data.tar.gz: 52f88ff3cae051044fa43674f4a08478c61ca624
+  metadata.gz: 57d601c1eeffddfe7479bc0a220b1bd8b0af5477
+  data.tar.gz: 9e117589cbc76f087ae42b0077575ace1055dd49
 SHA512:
-  metadata.gz: 1fefba7bae955d2c0b10100d1f48807bed2910ed68165e5f7ee3ac8cc2592d49d968c7d4779d828c452425aef9ba89932394630fb63e48920e8ee97fb31014dc
-  data.tar.gz: 74f9938ec9597affb48b7d2e1c807ed047641ac44c0252c079be3c0a0ef20ac804ceea42a20f0573ad9f39e4eea18f6cbf629d311af49fe53d8b11f7775a3e50
+  metadata.gz: 162ecc8fe267c4871a55bc594c4588dac40843558cc02ad301bf35d609f40f594fa6df6e08a74495942a0af362a200e5c4ede420b43d65442f5475414052930f
+  data.tar.gz: b1ae99e5b127c005358b20943d7b11f9742f83ced8242568137842c68f64a09379dca14a857e9768aa9eb20b62d37b91f67ecc2a420f8089b4fcc6b360b48e05

data/lib/statement/scraper.rb

@@ -29,7 +29,7 @@ module Statement
     end
     
     def self.member_methods
-      [:capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :lujan, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :vitter, :donnelly, :inhofe, :reid, :palazzo, :roe, :document_query, :swalwell, :fischer]
+      [:capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :lujan, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :vitter, :donnelly, :inhofe, :reid, :palazzo, :roe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton, :wolf_sherman_mccaul, :welch, :sessions, :gabbard]
     end
     
     def self.committee_methods
@@ -40,7 +40,8 @@ module Statement
       year = Date.today.year
       results = [capuano, cold_fusion(year, 0), conaway, chabot, susandavis, klobuchar, lujan, palazzo(page=1), roe(page=1), billnelson(year=year), 
         document_query(page=1), document_query(page=2), swalwell(page=1), donnelly(year=year), crapo, coburn, boxer(start=1),
-        vitter(year=year), inhofe(year=year), reid, fischer].flatten
+        vitter(year=year), inhofe(year=year), reid, fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, wolf_sherman_mccaul, welch,
+        sessions(year=year), gabbard].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
     end
@@ -48,8 +49,8 @@ module Statement
     def self.backfill_from_scrapers
       results = [cold_fusion(2012, 0), cold_fusion(2011, 0), cold_fusion(2010, 0), billnelson(year=2012), document_query(page=3), 
         document_query(page=4), coburn(year=2012), coburn(year=2011), coburn(year=2010), boxer(start=11), boxer(start=21), 
-        boxer(start=31), boxer(start=41), vitter(year=2012), vitter(year=2011), swalwell(page=2), swalwell(page=3)
-        ].flatten
+        boxer(start=31), boxer(start=41), vitter(year=2012), vitter(year=2011), swalwell(page=2), swalwell(page=3), clark(year=2013), culberson_chabot_grisham(page=2),
+        wolf_sherman_mccaul(page=1), sessions(year=2013)].flatten
       Utils.remove_generic_urls!(results)
     end
     
@@ -493,7 +494,114 @@ module Statement
       end
       results
     end
-    
+
+    def self.clark(year=Date.today.year)
+      results = []
+      domain = 'katherineclark.house.gov'
+      url = "http://katherineclark.house.gov/index.cfm/press-releases?MonthDisplay=0&YearDisplay=#{year}"
+      doc = open_html(url)
+      return if doc.nil?
+      (doc/:tr)[1..-1].each do |row|
+        next if row.children.first.text == 'Date'
+        results << { :source => url, :date => Date.parse(row.children.first.text), :title => row.children[2].children.text, :url => row.children[2].children[0]['href'], :domain => domain}
+      end
+      results
+    end
+
+    def self.sessions(year=Date.today.year)
+      results = []
+      domain = 'sessions.senate.gov'
+      url = "http://www.sessions.senate.gov/public/index.cfm/news-releases?YearDisplay=#{year}"
+      doc = open_html(url)
+      return if doc.nil?
+      (doc/:tr)[1..-1].each do |row|
+        next if row.children.first.text == 'Date'
+        results << { :source => url, :date => Date.parse(row.children.first.text), :title => row.children[2].children.text, :url => row.children[2].children[0]['href'], :domain => domain}
+      end
+      results
+    end
+
+    def self.edwards
+      results = []
+      domain = 'donnaedwards.house.gov'
+      url = "http://donnaedwards.house.gov/index.php?option=com_content&view=category&id=10&Itemid=18"
+      doc = open_html(url)
+      return if doc.nil?
+      table = (doc/:table)[4]
+      (table/:tr).each do |row|
+        results << { :source => url, :url => "http://donnaedwards.house.gov/"+row.children.children[1]['href'], :title => row.children.children[1].text.strip, :date => Date.parse(row.children.children[3].text.strip), :domain => domain}
+      end
+      results
+    end
+
+    def self.culberson_chabot_grisham(page=1)
+      results = []
+      domains = [{'culberson.house.gov' => 2573}, {'chabot.house.gov' => 2508}, {'lujangrisham.house.gov' => 2447}]
+      domains.each do |domain|
+        doc = open_html("http://"+domain.keys.first+"/news/documentquery.aspx?DocumentTypeID=#{domain.values.first}&Page=#{page}")
+        return if doc.nil?
+        doc.css('ul.UnorderedNewsList li').each do |row|
+          link = "http://"+domain.keys.first+"/news/" + row.children[1]['href']
+          title = row.children[1].text.strip
+          date = Date.parse(row.children[3].text.strip)
+          results << { :source => "http://"+domain.keys.first+"/news/"+"documentquery.aspx?DocumentTypeID=#{domain.values.first}&Page=#{page}", :title => title, :url => link, :date => date, :domain => domain }
+        end
+      end
+      results.flatten
+    end
+
+    def self.barton
+      results = []
+      domain = 'joebarton.house.gov'
+      url = "http://joebarton.house.gov/press-releasescolumns/"
+      doc = open_html(url)
+      return if doc.nil?
+      (doc/:h3)[0..-3].each do |row|
+        results << { :source => url, :url => "http://joebarton.house.gov/"+row.children[1]['href'], :title => row.children[1].text.strip, :date => Date.parse(row.next.next.text), :domain => domain}
+      end
+      results
+    end
+
+    def self.wolf_sherman_mccaul(page=0)
+      results = []
+      domains = ['wolf.house.gov', 'sherman.house.gov', 'mccaul.house.gov']
+      domains.each do |domain|
+        url = "http://#{domain}/media-center/press-releases?page=#{page}"
+        doc = open_html(url)
+        return if doc.nil?
+        dates = doc.xpath('//span[@class="field-content"]').map {|s| s.text if s.text.strip.include?("201")}.compact!
+        (doc/:h3).first(10).each_with_index do |row, i|
+          date = Date.parse(dates[i])
+          results << {:source => url, :url => "http://"+domain+row.children.first['href'], :title => row.children.first.text.strip, :date => date, :domain => domain}
+        end
+      end
+      results.flatten
+    end
+
+    def self.welch
+      results = []
+      domain = 'welch.house.gov'
+      url = "http://www.welch.house.gov/press-releases/"
+      doc = open_html(url)
+      return if doc.nil?
+      (doc/:h3).each do |row|
+        results << { :source => url, :url => "http://www.welch.house.gov/"+row.children[1]['href'], :title => row.children[1].text.strip, :date => Date.parse(row.next.next.text), :domain => domain}
+      end
+      results
+    end
+
+    def self.gabbard
+      results = []
+      domain = 'gabbard.house.gov'
+      url = "http://gabbard.house.gov/index.php/news/press-releases"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.css('ul.fc_leading li').each do |row|
+        results << {:source => url, :url => "http://gabbard.house.gov"+row.children[0].children[1]['href'], :title => row.children[0].children[1].text.strip, :date => Date.parse(row.children[2].text), :domain => domain}        
+      end
+      results
+    end
+
     def self.document_query(page=1)
       results = []
       domains = [{"thornberry.house.gov" => 1776}, {"wenstrup.house.gov" => 2491}]

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "1.7.1"
+  VERSION = "1.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: '1.8'
 platform: ruby
 authors:
 - Derek Willis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-10 00:00:00.000000000 Z
+date: 2014-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -198,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Given a url, Statement returns links to press releases and official statements.