statement 2.0.4 → 2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c771519d9fc8c1d3b906d0b09695205a92f893f2
-  data.tar.gz: 737c1f6530c5d7d9424be1f5629df0042c6abc8c
+  metadata.gz: 92cb454019d854f249abc59ab4a0568b7bf98c0e
+  data.tar.gz: bc6b887f904c4e7bca2990bf897ad67e694cd4ef
 SHA512:
-  metadata.gz: c1c53d467e8b878ff6995b6aacbc1618904a263bf55fe2d50e702334afbfbd0d323979d26be9e1d39abf83c2e21dc9258dcce98330cfc8a962620714bebd5707
-  data.tar.gz: e2ef4d6db0d2276a089c8e4bace60d859bc60d584177c4ec862bc55ddf2973ea3aa1b10aecf19769b4e86a5cc6143ae7206eef7688fe81c0a4f3ea57a1505827
+  metadata.gz: 7ea5f3994b277210e343c59c2fdbeb10b36194036ec6c0e3bd78de9d528542c7105674e440118482e63317f17f24906fc36784cf15d19506b7a6afe95ba1aa31
+  data.tar.gz: 651466b0481dcf50682bae862f3f8f5d346636db21ede45c96e8cfe7b08644b9ce9efe77bcf1286a5121a619401a2b83151a5dd3c242fda5ecded34370f029bb

data/lib/statement/scraper.rb

@@ -29,10 +29,10 @@ module Statement
     end
 
     def self.member_methods
-      [:crenshaw, :capuano, :cold_fusion, :conaway, :chabot, :freshman_senators, :klobuchar, :billnelson, :crapo, :boxer,
-      :vitter, :inhofe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton,
-      :welch, :sessions, :gabbard, :costa, :farr, :mcclintock, :olson, :schumer, :lamborn, :walden, :boehner,
-      :bennie_thompson, :speier, :poe, :grassley, :bennet, :shaheen, :keating, :drupal, :jenkins, :durbin_burr, :rand_paul]
+      [:crenshaw, :capuano, :cold_fusion, :conaway, :chabot, :klobuchar, :billnelson, :crapo, :boxer, :burr, :ellison,
+      :vitter, :inhofe, :document_query, :swalwell, :fischer, :clark, :edwards, :culberson_chabot_grisham, :barton, :schiff,
+      :welch, :sessions, :gabbard, :costa, :farr, :mcclintock, :olson, :schumer, :cassidy, :lowey, :mcmorris, :takano,
+      :bennie_thompson, :speier, :poe, :grassley, :bennet, :shaheen, :keating, :drupal, :jenkins, :durbin, :rand_paul]
     end
 
     def self.committee_methods
@@ -41,11 +41,11 @@ module Statement
 
     def self.member_scrapers
       year = Date.today.year
-      results = [crenshaw, capuano, cold_fusion(year, nil), conaway, chabot, klobuchar(year), billnelson(page=0),
-        document_query(page=1), document_query(page=2), swalwell(page=1), crapo, boxer, grassley(page=0),
+      results = [crenshaw, capuano, cold_fusion(year, nil), conaway, chabot, klobuchar(year), billnelson(page=0), ellison,
+        document_query(page=1), document_query(page=2), swalwell(page=1), crapo, boxer, grassley(page=0), burr, cassidy,
         vitter(year=year), inhofe(year=year), fischer, clark(year=year), edwards, culberson_chabot_grisham(page=1), barton, welch,
-        sessions(year=year), gabbard, costa, farr, olson, schumer, lamborn(limit=10), walden, bennie_thompson, speier,
-        poe(year=year, month=0), bennet(page=1), shaheen(page=1), perlmutter, keating, drupal, jenkins, durbin_burr(page=1),
+        sessions(year=year), gabbard, costa, farr, olson, schumer, bennie_thompson, speier, lowey, mcmorris, schiff, takano,
+        poe(year=year, month=0), bennet(page=1), shaheen(page=1), perlmutter, keating, drupal, jenkins, durbin(page=1),
         rand_paul(page = 1)].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
@@ -53,11 +53,12 @@ module Statement
 
     def self.backfill_from_scrapers
       results = [cold_fusion(2012, 0), cold_fusion(2011, 0), cold_fusion(2010, 0), billnelson(year=2012), document_query(page=3),
-        document_query(page=4), grassley(page=1), grassley(page=2), grassley(page=3),
+        document_query(page=4), grassley(page=1), grassley(page=2), grassley(page=3), burr(page=2), burr(page=3), burr(page=4),
         vitter(year=2012), vitter(year=2011), swalwell(page=2), swalwell(page=3), clark(year=2013), culberson_chabot_grisham(page=2),
-        sessions(year=2013), pryor(page=1), farr(year=2013), farr(year=2012), farr(year=2011),
-        olson(year=2013), schumer(page=2), schumer(page=3), poe(year=2015, month=2),
-        poe(year=2015, month=1)].flatten
+        sessions(year=2013), pryor(page=1), farr(year=2013), farr(year=2012), farr(year=2011), cassidy(page=2), cassidy(page=3),
+        olson(year=2013), schumer(page=2), schumer(page=3), poe(year=2015, month=2), ellison(page=1), ellison(page=2), lowey(page=1),
+        lowey(page=2), lowey(page=3), poe(year=2015, month=1), mcmorris(page=2), mcmorris(page=3), schiff(page=2), schiff(page=3),
+        takano(page=2), takano(page=3)].flatten
       Utils.remove_generic_urls!(results)
     end
 
@@ -247,19 +248,6 @@ module Statement
       results
     end
 
-    def self.boehner(page=1, year=Date.today.year)
-      results = []
-      url = "http://boehner.house.gov/category/press-releases/page/#{page}/"
-      doc = open_html(url)
-      return if doc.nil?
-      (doc/:article).each do |row|
-        month = row.children[1].children[1].children[1].children[0].text
-        day = row.children[1].children[1].children[1].children[1].text
-        results << { :source => url, :url => "http://boehner.house.gov"+row.children[12].children[1].children[0]['href'], :title => row.children[1].children[1].children[3].text, :date => Date.parse(month+" "+day+" "+year.to_s), :domain => 'boehner.house.gov'}
-      end
-      results
-    end
-
     def self.capuano
       results = []
       base_url = "http://www.house.gov/capuano/news/"
@@ -365,6 +353,17 @@ module Statement
       results
     end
 
+    def self.mcmorris(page=1)
+      results = []
+      url = "http://mcmorris.house.gov/issues/page/#{page}/?tax=types&term=news_releases"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.css(".feed-result").each do |row|
+        results << { :source => url, :url => row.children[3].children[3].children.first['href'], :title => row.children[3].children[3].children.first.text.strip, :date => Date.parse(row.children[3].children[1].text), :domain => "mcmorris.house.gov" }
+      end
+      results
+    end
+
     def self.klobuchar(year)
       results = []
       base_url = "http://www.klobuchar.senate.gov/"
@@ -461,16 +460,62 @@ module Statement
       results
     end
 
+    def self.schiff(page=1)
+      results = []
+      url = "http://schiff.house.gov/news/press-releases?PageNum_rs=#{page}&"
+      doc = open_html(url)
+      return if doc.nil?
+      rows = doc.css("#press").first.css('h2')
+      rows.each do |row|
+        results << { :source => url, :url => "http://schiff.house.gov" + row.children.first['href'], :title => row.children.last.text.strip, :date => Date.strptime(row.previous.previous.text, "%m.%d.%y"), :domain => "schiff.house.gov" }
+      end
+      results
+    end
 
-    # fetches the latest 1000 releases, can be altered
-    def self.lautenberg(rows=1000)
+    def self.takano(page=1)
       results = []
-      base_url = 'http://www.lautenberg.senate.gov/newsroom/'
-      url = base_url + "releases.cfm?maxrows=#{rows}&startrow=1&&type=1"
+      url = "http://takano.house.gov/newsroom/press-releases?PageNum_rs=#{page}"
       doc = open_html(url)
       return if doc.nil?
-      doc.xpath("//tr")[4..-2].each do |row|
-        results << { :source => url, :url => base_url + row.children[2].children[0]['href'], :title => row.children[2].text.strip, :date => Date.strptime(row.children[0].text.strip, "%m/%d/%y"), :domain => "lautenberg.senate.gov" }
+      rows = doc.css("#press").first.css('h2')
+      rows.each do |row|
+        results << { :source => url, :url => "http://takano.house.gov" + row.children.first['href'], :title => row.children.last.text.strip, :date => Date.strptime(row.previous.previous.text, "%m.%d.%y"), :domain => "takano.house.gov" }
+      end
+      results
+    end
+
+    def self.speier
+      results = []
+      url = "http://speier.house.gov/index.php?option=com_content&view=category&id=20&Itemid=14"
+      doc = open_html(url)
+      return if doc.nil?
+      rows = doc.css("table.category tr")
+      rows.each do |row|
+        results << { :source => url, :url => "http://speier.house.gov" + row.children[1].children[1]['href'], :title => row.children[1].children[1].text.strip, :date => Date.parse(row.children[3].text.strip), :domain => "speier.house.gov" }
+      end
+      results
+    end
+
+    def self.burr(page=1)
+      results = []
+      url = "http://www.burr.senate.gov/press/releases?PageNum_rs=#{page}&"
+      doc = open_html(url)
+      return if doc.nil?
+      rows = doc.css("#press").first.css('h2')
+      rows.each do |row|
+        results << { :source => url, :url => "http://www.burr.senate.gov" + row.children.first['href'], :title => row.children.last.text.strip, :date => Date.strptime(row.previous.previous.text, "%m.%d.%y"), :domain => "burr.senate.gov" }
+      end
+      results
+    end
+
+    def self.cassidy(page=1)
+      results = []
+      url = "http://www.cassidy.senate.gov/newsroom/press-releases?PageNum_rs=#{page}&"
+      doc = open_html(url)
+      return if doc.nil?
+      rows = doc.css("#press").first.css('h2')
+      rows.each do |row|
+        results << { :source => url, :url => "http://www.cassidy.senate.gov" + row.children.first['href'], :title => row.children.last.text.strip, :date => Date.strptime(row.previous.previous.text, "%m.%d.%y"), :domain => "cassidy.senate.gov" }
       end
       results
     end
@@ -510,6 +555,17 @@ module Statement
       results
     end
 
+    def self.ellison(page=0)
+      results = []
+      url = "http://ellison.house.gov/media-center/press-releases?page=#{page}"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.xpath("//div[@class='views-field views-field-created datebar']").each do |row|
+        results << { :source => url, :url => "http://ellison.house.gov" + row.next.next.children[1].children[0]['href'], :title => row.next.next.text.strip, :date => Date.parse(row.text.strip), :domain => "ellison.house.gov" }
+      end
+      results
+    end
+
     def self.boxer
       results = []
       url = "http://www.boxer.senate.gov/press/release"
@@ -550,16 +606,13 @@ module Statement
       results
     end
 
-    def self.durbin_burr(page=1)
+    def self.durbin(page=1)
       results = []
-      domains = ["www.durbin.senate.gov", "www.burr.senate.gov"]
-      domains.each do |domain|
-        url = "http://#{domain}/newsroom/press-releases?PageNum_rs=#{page}&"
-        doc = open_html(url)
-        return if doc.nil?
-        doc.xpath("//div[@id='press']//h2").each do |row|
-          results << { :source => url, :url => "http://#{domain}"+row.children[0]['href'], :title => row.children[0].text.strip, :date => Date.parse(row.previous.previous.text.gsub(".","/")), :domain => domain}
-        end
+      url = "http://www.durbin.senate.gov/newsroom/press-releases?PageNum_rs=#{page}&"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.xpath("//div[@id='press']//h2").each do |row|
+        results << { :source => url, :url => "http://www.durbin.senate.gov"+row.children[0]['href'], :title => row.children[0].text.strip, :date => Date.parse(row.previous.previous.text.gsub(".","/")), :domain => 'www.durbin.senate.gov'}
       end
       results
     end
@@ -730,7 +783,7 @@ module Statement
         {"clawson.house.gov" => 2641},
         {"palazzo.house.gov" => 2519},
         {"roe.house.gov" => 1532},
-        {"perry.house.gov" => 2608},
+        {"perry.house.gov" => 2607},
         {"rodneydavis.house.gov" => 2427},
         {"kevinbrady.house.gov" => 2657},
         {"loudermilk.house.gov" => 27},
@@ -742,7 +795,13 @@ module Statement
         {"grothman.house.gov" => 27},
         {"beyer.house.gov" => 27},
         {"kathleenrice.house.gov" => 27},
-        {"hanna.house.gov" => 27}
+        {"hanna.house.gov" => 27},
+        {"trentkelly.house.gov" => 27},
+        {"lamborn.house.gov" => 27},
+        {"wittman.house.gov" => 2670},
+        {"kinzinger.house.gov" => 2665},
+        {"ellmers.house.gov" => 27},
+        {"frankel.house.gov" => 27}
       ]
       domains.each do |domain|
         doc = open_html("http://"+domain.keys.first+"/news/documentquery.aspx?DocumentTypeID=#{domain.values.first}&Page=#{page}")
@@ -792,23 +851,6 @@ module Statement
       results
     end
 
-    def self.lamborn(limit=nil)
-      results = []
-      domain = 'lamborn.house.gov'
-      url = "http://lamborn.house.gov/2015-press-releases/"
-      doc = open_html(url)
-      return if doc.nil?
-      links = (doc/:h3).map{|h| { "http://lamborn.house.gov"+h.children[1]['href'] => h.text.strip} }
-      links = links.first(limit) if limit
-      links.each do |link|
-        page = open_html(link.keys.first)
-        print_path = page.search("a").detect{|a| a['onclick'] && a['onclick'].include?('popup')}['onclick'].split("'")[1]
-        print_page = open_html("http://lamborn.house.gov"+print_path)
-        results << {:source => url, :url => link.keys.first, :title => link.values.first, :date => Date.parse(print_page.xpath('//*[@class="PopupNewsDetailsDate"]').text), :domain => domain }
-      end
-      results
-    end
-
     def self.jenkins
       results = []
       domain = 'lynnjenkins.house.gov/'
@@ -821,18 +863,6 @@ module Statement
       results
     end
 
-    def self.walden
-      results = []
-      domain = 'walden.house.gov'
-      url = "http://walden.house.gov/s2015/"
-      doc = open_html(url)
-      return if doc.nil?
-      doc.xpath('//*[@id="centerbox"]/div[1]/ul/li').each do |row|
-        results << {:source => url, :url => 'http://walden.house.gov' + row.children[3].children[1]['href'], :title => row.children[3].text.strip, :date => Date.parse(row.children[5].text), :domain => domain }
-      end
-      results
-    end
-
     def self.bennie_thompson
       results = []
       domain = "benniethompson.house.gov"
@@ -845,14 +875,14 @@ module Statement
       results
     end
 
-    def self.speier
+    def self.lowey(page=0)
       results = []
-      domain = "speier.house.gov"
-      url = "http://speier.house.gov/index.php?option=com_content&view=category&id=20&Itemid=14"
+      domain = "lowey.house.gov"
+      url = "https://lowey.house.gov/media-center/press-releases?page=#{page}"
       doc = open_html(url)
       return if doc.nil?
-      doc.xpath('//*[@id="adminForm"]/table/tbody/tr').each do |row|
-        results << {:source => url, :url => 'http://speier.house.gov' + row.children[1].children[1]['href'], :title => row.children[1].children[1].text.strip, :date => Date.parse(row.children[3].text.strip), :domain => domain }
+      doc.css(".view-content .views-row").first(10).each do |row|
+        results << {:source => url, :url => 'http://lowey.house.gov' + row.css('h3').first.children.first['href'], :title => row.css('h3').first.children.first.text.strip, :date => Date.parse(row.css(".views-field .field-content")[1].text), :domain => domain }
       end
       results
     end
@@ -928,14 +958,19 @@ module Statement
             "http://butterfield.house.gov/media-center/press-releases",
             "http://walz.house.gov/media-center/press-releases",
             "https://pingree.house.gov/media-center/press-releases",
-            "http://sarbanes.house.gov/media-center/press-releases",
             "http://wilson.house.gov/media-center/press-releases",
             "https://bilirakis.house.gov/press-releases",
             "http://quigley.house.gov/media-center/press-releases",
             "https://denham.house.gov/media-center/press-releases",
             "https://sewell.house.gov/media-center/press-releases",
             "https://buchanan.house.gov/media-center/press-releases",
-            "https://meehan.house.gov/media-center/press-releases"
+            "https://meehan.house.gov/media-center/press-releases",
+            "https://olson.house.gov/media-center/press-releases",
+            "https://louise.house.gov/media-center/press-releases",
+            "https://waters.house.gov/media-center/press-releases",
+            "https://walden.house.gov/media-center/press-releases",
+            "https://brooks.house.gov/media-center/news-releases",
+            "https://swalwell.house.gov/media-center/press-releases"
         ]
       end
 

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "2.0.4"
+  VERSION = "2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: '2.1'
 platform: ruby
 authors:
 - Derek Willis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-29 00:00:00.000000000 Z
+date: 2015-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -205,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Given a url, Statement returns links to press releases and official statements.