statement 0.3 → 0.4

data/.gitignore

@@ -8,6 +8,8 @@ InstalledFiles
 _yardoc
 coverage
 doc/
+coverage
+InstalledFiles
 lib/bundler/man
 pkg
 rdoc

data/README.md

@@ -6,30 +6,40 @@ Statement parses RSS feeds and HTML pages containing press releases and other of
 
 Add this line to your application's Gemfile:
 
-    gem 'statement'
+```ruby
+gem 'statement'
+```
 
 And then execute:
 
-    $ bundle
+```sh
+$ bundle
+```
 
 Or install it yourself as:
 
-    $ gem install statement
+```sh
+$ gem install statement
+```
 
 ## Usage
 
-    require 'rubygems'
-    require 'statement'
-    
-    results = Statement::Link.house_gop('http://www.gop.gov/republicans/news?offset=03/29/11')
-    puts results.first
-    {:source=>"http://www.gop.gov/republicans/news?offset=03/29/11", :url=>"http://poe.house.gov/News/DocumentSingle.aspx?DocumentID=233004", :title=>"Poe: War in the Name of Humanity", :date=> <Date: 2011-03-29 ((2455650j,0s,0n),+0s,2299161j)>, :domain=>"poe.house.gov"}
+```ruby
+require 'rubygems'
+require 'statement'
     
+results = Statement::Link.house_gop('http://www.gop.gov/republicans/news?offset=03/29/11')
+puts results.first
+{:source=>"http://www.gop.gov/republicans/news?offset=03/29/11", :url=>"http://poe.house.gov/News/DocumentSingle.aspx?DocumentID=233004", :title=>"Poe: War in the Name of Humanity", :date=> <Date: 2011-03-29 ((2455650j,0s,0n),+0s,2299161j)>, :domain=>"poe.house.gov"}
+```
+
 ## Tests
 
 Statement uses MiniTest, to run tests:
 
-    rake test
+```sh
+$ rake test
+```
 
 ## Contributing
 
@@ -42,4 +52,5 @@ Statement uses MiniTest, to run tests:
 ## Authors
 
 * Derek Willis
-* Jacob Harris
+* Jacob Harris
+

data/lib/statement.rb

@@ -34,6 +34,145 @@ module Statement
       end
     end
     
+    def self.from_scrapers
+      [freshman_senators, capuano, crenshaw(2013, 0), conaway, susandavis, faleomavaega, klobuchar, lujan, billnelson(year=2013), 
+        billnelson(year=2012), roe(page=1), roe(page=2), roe(page=3), thornberry(page=1), thornberry(page=2), thornberry(page=3)].flatten
+    end
+    
+    ## special cases for members without RSS feeds
+    
+    def self.capuano
+      results = []
+      base_url = "http://www.house.gov/capuano/news/"
+      list_url = base_url + 'date.shtml'
+      doc = Nokogiri::HTML(open(list_url).read)
+      doc.xpath("//a").each do |link|
+        if link['href'] and link['href'].include?('/pr')
+          begin 
+            date = Date.parse(link.text) 
+          rescue 
+            date = nil
+          end
+          results << { :source => list_url, :url => base_url + link['href'], :title => link.text.split(' ',2).last, :date => date, :domain => "www.house.gov/capuano/" }
+        end
+      end
+      return results[0..-5]
+    end
+    
+    def self.crenshaw(year, month)
+      results = []
+      year = Date.today.year if not year
+      month = 0 if not month
+      url = "http://crenshaw.house.gov/index.cfm/pressreleases?YearDisplay=#{year}&MonthDisplay=#{month}&page=1"
+      doc = Nokogiri::HTML(open(url).read)
+      doc.xpath("//tr")[2..-1].each do |row|
+        date_text, title = row.children.map{|c| c.text.strip}.reject{|c| c.empty?}
+        next if date_text == 'Date'
+        date = Date.parse(date_text)
+        results << { :source => url, :url => row.children[2].children.first['href'], :title => title, :date => date, :domain => "crenshaw.house.gov" }
+      end
+      results
+    end
+    
+    def self.conaway(page=1)
+      results = []
+      base_url = "http://conaway.house.gov/news/"
+      page_url = base_url + "documentquery.aspx?DocumentTypeID=1279&Page=#{page}"
+      doc = Nokogiri::HTML(open(page_url).read)
+      doc.xpath("//tr")[1..-1].each do |row|
+        results << { :source => page_url, :url => base_url + row.children.children[1]['href'], :title => row.children.children[1].text.strip, :date => Date.parse(row.children.children[4].text), :domain => "conaway.house.gov" }
+      end
+      results
+    end
+    
+    def self.susandavis
+      results = []
+      base_url = "http://www.house.gov/susandavis/"
+      doc = Nokogiri::HTML(open(base_url+'news.shtml').read)
+      doc.search("ul")[6].children.each do |row|
+        next if row.text.strip == ''
+        results << { :source => base_url+'news.shtml', :url => base_url + row.children[1]['href'], :title => row.children[1].text.split.join(' '), :date => Date.parse(row.children.first.text), :domain => "house.gov/susandavis" }
+      end
+      results
+    end
+    
+    def self.faleomavaega
+      results = []
+      base_url = "http://www.house.gov/faleomavaega/news-press.shtml"
+      doc = Nokogiri::HTML(open(base_url).read)
+      doc.xpath("//li[@type='disc']").each do |row|
+        results << { :source => base_url, :url => "http://www.house.gov/" + row.children[0]['href'], :title => row.children[0].text.gsub(/[\x80-\xff]/,'').split('Washington, D.C.').last, :date => Date.parse(row.children[1].text.gsub(/[\x80-\xff]/,' ')), :domain => "house.gov/faleomavaega" }
+      end
+      results
+    end
+    
+    def self.freshman_senators
+      results = []
+      ['baldwin', 'donnelly', 'flake', 'hirono','heinrich','murphy','scott','king','heitkamp','cruz','kaine'].each do |senator|
+        base_url = "http://www.#{senator}.senate.gov/"
+        doc = Nokogiri::HTML(open(base_url+'press.cfm?maxrows=200&startrow=1&&type=1').read)
+        doc.xpath("//tr")[3..-1].each do |row|
+          next if row.text.strip == ''
+          results << { :source => base_url+'press.cfm?maxrows=200&startrow=1&&type=1', :url => base_url + row.children.children[1]['href'], :title => row.children.children[1].text.strip.split.join(' '), :date => Date.parse(row.children.children[0].text.gsub(/[\x80-\xff]/,'')), :domain => "#{senator}.senate.gov" }
+        end
+      end
+      results.flatten
+    end
+    
+    def self.klobuchar
+      results = []
+      base_url = "http://www.klobuchar.senate.gov/"
+      [2012,2013].each do |year|
+        year_url = base_url + "newsreleases.cfm?year=#{year}"
+        doc = Nokogiri::HTML(open(year_url).read)
+        doc.xpath("//dt").each do |row|
+          results << { :source => year_url, :url => base_url + row.next.children[0]['href'], :title => row.next.text.strip.gsub(/[\x80-\xff]/,'').split.join(' '), :date => Date.parse(row.text), :domain => "klobuchar.senate.gov" }
+        end
+      end
+      results
+    end
+    
+    def self.lujan
+      results = []
+      base_url = 'http://lujan.house.gov/'
+      doc = Nokogiri::HTML(open(base_url+'index.php?option=com_content&view=article&id=981&Itemid=78').read)
+      doc.xpath('//ul')[1].children.each do |row|
+        next if row.text.strip == ''
+        results << { :source => base_url+'index.php?option=com_content&view=article&id=981&Itemid=78', :url => base_url + row.children[0]['href'], :title => row.children[0].text.gsub(/[\x80-\xff]/,'').gsub('Lujn','Lujan'), :date => nil, :domain => "lujan.house.gov" }
+      end
+      results
+    end
+    
+    def self.billnelson(year=2013)
+      results = []
+      base_url = "http://www.billnelson.senate.gov/news/"
+      year_url = base_url + "media.cfm?year=#{year}"
+      doc = Nokogiri::HTML(open(year_url).read)
+      doc.xpath('//li').each do |row|
+        results << { :source => year_url, :url => base_url + row.children[0]['href'], :title => row.children[0].text.strip, :date => Date.parse(row.children.last.text).to_s, :domain => "billnelson.senate.gov" }
+      end
+      results
+    end
+    
+    def self.roe(page=1)
+      results = []
+      base_url = "http://roe.house.gov/news/"
+      doc = Nokogiri::HTML(open(base_url+"documentquery.aspx?DocumentTypeID=1532&Page=#{page}").read)
+      doc.xpath("//span[@class='middlecopy']").each do |row|
+        results << { :source => base_url+"documentquery.aspx?DocumentTypeID=1532&Page=#{page}", :url => base_url + row.children[6]['href'], :title => row.children[1].text.strip.gsub(/[\x80-\xff]/,''), :date => Date.parse(row.children[4].text.gsub(/[\x80-\xff]/,'').strip), :domain => "roe.house.gov" }
+      end
+      results
+    end
+    
+    def self.thornberry(page=1)
+      results = []
+      base_url = "http://thornberry.house.gov/news/"
+      doc = Nokogiri::HTML(open(base_url+"documentquery.aspx?DocumentTypeID=1776&Page=#{page}").read)
+      doc.xpath("//span[@class='middlecopy']").each do |row|
+        results << { :source => base_url+"documentquery.aspx?DocumentTypeID=1776&Page=#{page}", :url => base_url + row.children[6]['href'], :title => row.children[1].text.strip.gsub(/[\x80-\xff]/,''), :date => Date.parse(row.children[4].text.gsub(/[\x80-\xff]/,'').strip), :domain => "thornberry.house.gov" }
+      end
+      results
+    end
+    
   end
-  
 end

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "0.3"
+  VERSION = "0.4"
 end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: statement
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 3
-  version: "0.3"
+  - 4
+  version: "0.4"
 platform: ruby
 authors: 
 - Derek Willis
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-04-22 00:00:00 Z
+date: 2013-04-25 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: bundler