statement 1.9.1 → 1.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc7a4069c3ee1c86d61f056a7139ce6b9db9100e
-  data.tar.gz: 171485c65c3713be3e695728d30bfd548c22f4f5
+  metadata.gz: 0be73a63b6661e11498742d7877c36ed2da5e182
+  data.tar.gz: 9368bd6373159d94b9463e57e657616c440d9c01
 SHA512:
-  metadata.gz: ce28008eb00aff77eef1c1e48425560504623ff02bb43b953f4b666e476fdf083f295b0479898bca8b5d334b8dada4e836820a6626486c7dbcd87b257928e268
-  data.tar.gz: 0e98a28b494b5d86f52ac8f2a34fb53bbb1c36a201d055e7e6310047d7c1159752edb2d01f4c8b95fdf4a793050ccc344b7a82126342b59039ec574f2bd03ef7
+  metadata.gz: 4ecf8b06def3327c5eafb8987b3037c793a08cf1ff85a54ceaf8823c3dce1a814289fecda74d359b49da75197ff57e104b60db42899cd2f2d7307c25bd930f0c
+  data.tar.gz: d70e2b77e900bbbd3122cce1665bc9d92293063c87bd4ce39b7f82b732e672152e8f4f1750160790942142f39cd4773beb8b8c082baf4714647fd4152c63aebf

data/lib/statement/feed.rb

@@ -7,7 +7,7 @@ require 'typhoeus'
 
 module Statement
   class Feed
-    
+
     def self.batch(urls)
       results = []
       failures = []
@@ -27,7 +27,7 @@ module Statement
       hydra.run
       [results.flatten, failures]
     end
-  
+
     def self.open_rss(url)
       begin
         Nokogiri::XML(open(url))
@@ -35,10 +35,10 @@ module Statement
         nil
       end
     end
-    
+
     def self.date_from_rss_item(link)
       if !link.xpath('pubDate').text.empty?
-        Date.parse(link.xpath('pubDate').text)
+         Date.parse(link.xpath('pubDate').text)
       elsif !link.xpath('pubdate').empty?
         Date.parse(link.xpath('pubdate').text)
       elsif link.xpath('link').text.include?("mikulski.senate.gov") and link.xpath('link').text.include?("-2014")
@@ -53,7 +53,7 @@ module Statement
       return unless doc
       parse_rss(doc, url)
     end
-    
+
     def self.parse_rss(doc, url)
       links = doc.xpath('//item')
       return if links.empty?

data/lib/statement/scraper.rb

@@ -29,7 +29,10 @@ module Statement
     end
 
     def self.member_methods
-      [:crenshaw, :capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :billnelson, :crapo, :boxer, :vitter, :inhofe, :palazzo, :roe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton, :sherman_mccaul, :welch, :sessions, :gabbard, :ellison, :costa, :farr, :mcclintock, :mcnerney, :olson]
+      [:crenshaw, :capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :billnelson, :crapo, :boxer,
+      :vitter, :inhofe, :palazzo, :roe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton,
+      :sherman_mccaul, :welch, :sessions, :gabbard, :ellison, :costa, :farr, :mcclintock, :mcnerney, :olson, :schumer, :lamborn, :walden,
+      :bennie_thompson, :speier, :poe]
     end
 
     def self.committee_methods
@@ -40,8 +43,9 @@ module Statement
       year = Date.today.year
       results = [crenshaw, capuano, cold_fusion(year, nil), conaway, chabot, susandavis, klobuchar(year), palazzo(page=1), roe(page=1), billnelson(year=year),
         document_query(page=1), document_query(page=2), swalwell(page=1), crapo, coburn, boxer(start=1),
-        vitter(year=year), inhofe(year=2014), fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, sherman_mccaul, welch,
-        sessions(year=year), gabbard, ellison(page=0), costa, farr, olson, mcnerney].flatten
+        vitter(year=year), inhofe(year=year), fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, sherman_mccaul, welch,
+        sessions(year=year), gabbard, ellison(page=0), costa, farr, olson, mcnerney, schumer, lamborn(limit=10), walden, bennie_thompson, speier,
+        poe(year=year, month=0)].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
     end
@@ -51,7 +55,8 @@ module Statement
         document_query(page=4), coburn(year=2012), coburn(year=2011), coburn(year=2010), boxer(start=11), boxer(start=21),
         boxer(start=31), boxer(start=41), vitter(year=2012), vitter(year=2011), swalwell(page=2), swalwell(page=3), clark(year=2013), culberson_chabot_grisham(page=2),
         sherman_mccaul(page=1), sessions(year=2013), pryor(page=1), ellison(page=1), ellison(page=2), ellison(page=3), farr(year=2013), farr(year=2012), farr(year=2011),
-        mcnerney(page=2), mcnerney(page=3), mcnerney(page=4), mcnerney(page=5), mcnerney(page=6), olson(year=2013)].flatten
+        mcnerney(page=2), mcnerney(page=3), mcnerney(page=4), mcnerney(page=5), mcnerney(page=6), olson(year=2013), schumer(page=2), schumer(page=3), poe(year=2015, month=2),
+        poe(year=2015, month=1)].flatten
       Utils.remove_generic_urls!(results)
     end
 
@@ -334,18 +339,6 @@ module Statement
       results
     end
 
-    def self.susandavis
-      results = []
-      base_url = "http://www.house.gov/susandavis/"
-      doc = open_html(base_url+'news.shtml')
-      return if doc.nil?
-      doc.search("ul")[6].children.each do |row|
-        next if row.text.strip == ''
-        results << { :source => base_url+'news.shtml', :url => base_url + row.children[1]['href'], :title => row.children[1].text.split.join(' '), :date => Date.parse(row.children.first.text), :domain => "house.gov/susandavis" }
-      end
-      results
-    end
-
     def self.klobuchar(year)
       results = []
       base_url = "http://www.klobuchar.senate.gov/"
@@ -361,6 +354,20 @@ module Statement
       results
     end
 
+    def self.poe(year, month=0)
+      results = []
+      base_url = "http://poe.house.gov"
+      month_url = base_url + "/press-releases?MonthDisplay=#{month}&YearDisplay=#{year}"
+        doc = open_html(year_url)
+        return if doc.nil?
+        doc.xpath("//tr")[1..-1].each do |row|
+          next if row.children[3].children[0].text.strip == 'Title'
+          results << { :source => month_url, :url => base_url + row.children[3].children[0]['href'], :title => row.children[3].children[0].text.strip, :date => Date.strptime(row.children[1].text, "%m/%d/%y"), :domain => "poe.house.gov" }
+        end
+      end
+      results
+    end
+
     def self.lujan
       results = []
       base_url = 'http://lujan.house.gov/'
@@ -709,6 +716,72 @@ module Statement
       results.flatten
     end
 
+    def self.schumer(page=1)
+      results = []
+      domain = 'www.schumer.senate.gov'
+      url = "http://www.schumer.senate.gov/newsroom/press-releases/table?PageNum_rs=#{page}"
+      doc = open_html(url)
+      return if doc.nil?
+      rows = (doc/:table/:tr).select{|r| !r.children[3].nil?}
+      rows.each do |row|
+        results << {:source => url, :url => row.children[3].children[1]['href'].strip, :title => row.children[3].text.strip, :date => Date.parse(row.children[1].text.strip), :domain => domain }
+      end
+      results
+    end
+
+    def self.lamborn(limit=nil)
+      results = []
+      domain = 'lamborn.house.gov'
+      url = "http://lamborn.house.gov/2015-press-releases/"
+      doc = open_html(url)
+      return if doc.nil?
+      links = (doc/:h3).map{|h| { "http://lamborn.house.gov"+h.children[1]['href'] => h.text.strip} }
+      links = links.first(limit) if limit
+      links.each do |link|
+        page = open_html(link.keys.first)
+        print_path = page.search("a").detect{|a| a['onclick'] && a['onclick'].include?('popup')}['onclick'].split("'")[1]
+        print_page = open_html("http://lamborn.house.gov"+print_path)
+        results << {:source => url, :url => link.keys.first, :title => link.values.first, :date => Date.parse(print_page.xpath('//*[@class="PopupNewsDetailsDate"]').text), :domain => domain }
+      end
+      results
+    end
+
+    def self.walden
+      results = []
+      domain = 'walden.house.gov'
+      url = "http://walden.house.gov/s2015/"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.xpath('//*[@id="centerbox"]/div[1]/ul/li').each do |row|
+        results << {:source => url, :url => 'http://walden.house.gov' + row.children[3].children[1]['href'], :title => row.children[3].text.strip, :date => Date.parse(row.children[5].text), :domain => domain }
+      end
+      results
+    end
+
+    def self.bennie_thompson
+      results = []
+      domain = "benniethompson.house.gov"
+      url = "http://benniethompson.house.gov/index.php?option=com_content&view=category&id=41&Itemid=148"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.xpath('//*[@id="adminForm"]/table/tbody/tr').each do |row|
+        results << {:source => url, :url => 'http://benniethompson.house.gov' + row.children[1].children[1]['href'], :title => row.children[1].children[1].text.strip, :date => Date.parse(row.children[3].text.strip), :domain => domain }
+      end
+      results
+    end
+
+    def self.speier
+      results = []
+      domain = "speier.house.gov"
+      url = "http://speier.house.gov/index.php?option=com_content&view=category&id=20&Itemid=14"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.xpath('//*[@id="adminForm"]/table/tbody/tr').each do |row|
+        results << {:source => url, :url => 'http://speier.house.gov' + row.children[1].children[1]['href'], :title => row.children[1].children[1].text.strip, :date => Date.parse(row.children[3].text.strip), :domain => domain }
+      end
+      results
+    end
+
     def self.backfill_bilirakis
       results = []
       domain = 'bilirakis.house.gov'

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "1.9.1"
+  VERSION = "1.9.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 1.9.1
+  version: 1.9.2
 platform: ruby
 authors:
 - Derek Willis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-23 00:00:00.000000000 Z
+date: 2015-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler