statement 2.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11dcba16755ef54dff1c0c48db50aca841485abd
-  data.tar.gz: dda7e3b05004b1d7bf59411c902ed2a670436914
+  metadata.gz: 74eeb1be0ee4ada11df34f5b8ddce00ffd5649cf
+  data.tar.gz: dd7fa7d9e6b0ccf0ae0bd3cdfa3f61f9306f4ef4
 SHA512:
-  metadata.gz: 20f0513c7aa7a3d70b9e3c3f8d8eb6c30e66b2b88a3dec326d0516e031d3450b201e7a2db8f98b90d29a1d2fb2a384ed096d9ea9f9940cfa41abf9ef79a84590
-  data.tar.gz: 358a8cfa517caf462ebe20119a81a898ac46b9806b1614e2ed6f34ceabc89bcd5dd40ecf43dc38cd989f8fd17449af2a32e63c2d636cd54f92830919bf569533
+  metadata.gz: 392340713cdf259bff3e60816d3ea12966a09127b0cad75c65f71307b18a692812bb7dc594fc332bbe9c49df15d52f6ab9d396d27795890d736de65ff10330a7
+  data.tar.gz: a80659fe9a3a651f8db4585112dd193fe413762d1409afd4703ff1d63454599298c22416e4f1f8735897a04169e4e09d5c3436a3e8699c67d7f124654423033f

data/lib/statement/feed.rb

@@ -17,6 +17,7 @@ module Statement
         req.on_complete do |response|
           if response.success?
             doc = Nokogiri::XML(response.body)
+            results << parse_atom(doc, url) if url == "http://larson.house.gov/index.php?option=com_ninjarsssyndicator&feed_id=1&format=raw"
             results << parse_rss(doc, url)
           else
             failures << url
@@ -51,7 +52,11 @@ module Statement
     def self.from_rss(url)
       doc = open_rss(url)
       return unless doc
-      parse_rss(doc, url)
+      if url == "http://larson.house.gov/index.php?option=com_ninjarsssyndicator&feed_id=1&format=raw"
+        parse_atom(doc, url)
+      else
+        parse_rss(doc, url)
+      end
     end
 
     def self.parse_rss(doc, url)
@@ -65,5 +70,14 @@ module Statement
       end
       Utils.remove_generic_urls!(results)
     end
+
+    def self.parse_atom(doc, url)
+      links = (doc/:entry)
+      return if links.empty?
+      results = links.map do |link|
+        { :source => url, :url => link.children[3]['href'], :title => link.children[1].text, :date => Date.parse(link.children[5].text), :domain => URI.parse(url).host }
+      end
+    end
+
   end
 end

data/lib/statement/scraper.rb

@@ -32,7 +32,7 @@ module Statement
       [:crenshaw, :capuano, :cold_fusion, :conaway, :chabot, :freshman_senators, :klobuchar, :billnelson, :crapo, :boxer,
       :vitter, :inhofe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton,
       :welch, :sessions, :gabbard, :costa, :farr, :mcclintock, :olson, :schumer, :lamborn, :walden,
-      :bennie_thompson, :speier, :poe, :grassley, :bennet, :shaheen, :keating, :drupal, :jenkins]
+      :bennie_thompson, :speier, :poe, :grassley, :bennet, :shaheen, :keating, :drupal, :jenkins, :durbin_burr]
     end
 
     def self.committee_methods
@@ -45,7 +45,7 @@ module Statement
         document_query(page=1), document_query(page=2), swalwell(page=1), crapo, boxer, grassley(page=0),
         vitter(year=year), inhofe(year=year), fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, welch,
         sessions(year=year), gabbard, costa, farr, olson, schumer, lamborn(limit=10), walden, bennie_thompson, speier,
-        poe(year=year, month=0), bennet(page=1), shaheen(page=1), perlmutter, keating, drupal, jenkins].flatten
+        poe(year=year, month=0), bennet(page=1), shaheen(page=1), perlmutter, keating, drupal, jenkins, durbin_burr(page=1)].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
     end
@@ -491,6 +491,20 @@ module Statement
       results
     end
 
+    def self.durbin_burr(page=1)
+      results = []
+      domains = ["www.durbin.senate.gov", "www.burr.senate.gov"]
+      domains.each do |domain|
+        url = "http://#{domain}/newsroom/press-releases?PageNum_rs=#{page}&"
+        doc = open_html(url)
+        return if doc.nil?
+        doc.xpath("//div[@id='press']//h2").each do |row|
+          results << { :source => url, :url => "http://#{domain}"+row.children[0]['href'], :title => row.children[0].text.strip, :date => Date.parse(row.previous.previous.text.gsub(".","/")), :domain => domain}
+        end
+      end
+      results
+    end
+
     def self.inhofe(year=Date.today.year)
       results = []
       url = "http://www.inhofe.senate.gov/newsroom/press-releases?year=#{year}"
@@ -651,7 +665,25 @@ module Statement
 
     def self.document_query(page=1)
       results = []
-      domains = [{"thornberry.house.gov" => 1776}, {"wenstrup.house.gov" => 2491}, {"clawson.house.gov" => 2641}, {"palazzo.house.gov" => 2519}, {"roe.house.gov" => 1532}, {"perry.house.gov" => 2608}, {"rodneydavis.house.gov" => 2427}, {"kevinbrady.house.gov" => 2657}]
+      domains = [
+        {"thornberry.house.gov" => 1776},
+        {"wenstrup.house.gov" => 2491},
+        {"clawson.house.gov" => 2641},
+        {"palazzo.house.gov" => 2519},
+        {"roe.house.gov" => 1532},
+        {"perry.house.gov" => 2608},
+        {"rodneydavis.house.gov" => 2427},
+        {"kevinbrady.house.gov" => 2657},
+        {"loudermilk.house.gov" => 27},
+        {"babin.house.gov" => 27},
+        {"bridenstine.house.gov" => 2412},
+        {"allen.house.gov" => 27},
+        {"davidscott.house.gov" => 377},
+        {"buddycarter.house.gov" => 27},
+        {"grothman.house.gov" => 27},
+        {"beyer.house.gov" => 27},
+        {"kathleenrice.house.gov" => 27}
+      ]
       domains.each do |domain|
         doc = open_html("http://"+domain.keys.first+"/news/documentquery.aspx?DocumentTypeID=#{domain.values.first}&Page=#{page}")
         return if doc.nil?
@@ -827,7 +859,9 @@ module Statement
             "http://sarbanes.house.gov/media-center/press-releases",
             "http://wilson.house.gov/media-center/press-releases",
             "https://bilirakis.house.gov/press-releases",
-            "http://quigley.house.gov/media-center/press-releases"
+            "http://quigley.house.gov/media-center/press-releases",
+            "https://denham.house.gov/media-center/press-releases",
+            "https://sewell.house.gov/media-center/press-releases"
         ]
       end
 

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "2.0"
+  VERSION = "2.0.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: '2.0'
+  version: 2.0.1
 platform: ruby
 authors:
 - Derek Willis