statement 1.0.15 → 1.1

data/lib/statement/feed.rb

@@ -49,12 +49,12 @@ module Statement
     def self.from_rss(url)
       doc = open_rss(url)
       return unless doc
-      return if Mime::Type.new(doc).html?
       parse_rss(doc, url)
     end
     
     def self.parse_rss(doc, url)
       links = doc.xpath('//item')
+      return if links.empty?
       results = links.map do |link|
         abs_link = Utils.absolute_link(url, link.xpath('link').text)
         abs_link = "http://www.burr.senate.gov/public/"+ link.xpath('link').text if url == 'http://www.burr.senate.gov/public/index.cfm?FuseAction=RSS.Feed'

data/lib/statement/scraper.rb

@@ -29,7 +29,7 @@ module Statement
     end
     
     def self.member_methods
-      [:capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :lujan, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :mccain, :vitter, :donnelly, :inhofe, :levin, :reid, :palazzo, :document_query, :farenthold, :swalwell, :fischer]
+      [:capuano, :cold_fusion, :conaway, :chabot, :susandavis, :freshman_senators, :klobuchar, :lujan, :billnelson, :lautenberg, :crapo, :coburn, :boxer, :mccain, :vitter, :donnelly, :inhofe, :levin, :reid, :palazzo, :roe, :document_query, :swalwell, :fischer]
     end
     
     def self.committee_methods
@@ -38,8 +38,8 @@ module Statement
     
     def self.member_scrapers
       year = Date.today.year
-      results = [freshman_senators, capuano, cold_fusion(year, 0), conaway, chabot, susandavis, klobuchar, lujan, palazzo(page=1), billnelson(year=year), 
-        document_query(page=1), document_query(page=2), farenthold(year), swalwell(page=1), donnelly(year=year), crapo, coburn, boxer(start=1), mccain(year=year), 
+      results = [freshman_senators, capuano, cold_fusion(year, 0), conaway, chabot, susandavis, klobuchar, lujan, palazzo(page=1), roe(page=1), billnelson(year=year), 
+        document_query(page=1), document_query(page=2), swalwell(page=1), donnelly(year=year), crapo, coburn, boxer(start=1), mccain(year=year), 
         vitter(year=year), inhofe(year=year), reid, fischer].flatten
       results = results.compact
       Utils.remove_generic_urls!(results)
@@ -519,10 +519,22 @@ module Statement
       end
       results
     end
+
+    def self.roe(page=1)
+      results = []
+      domain = 'roe.house.gov'
+      url = "http://roe.house.gov/news/documentquery.aspx?DocumentTypeID=1532&Page=#{page}"
+      doc = open_html(url)
+      return if doc.nil?
+      doc.xpath("//div[@class='middlecopy']//li").each do |row|
+        results << { :source => url, :url => "http://roe.house.gov/news/" + row.children[1]['href'], :title => row.children[1].text.strip, :date => Date.parse(row.children[3].text.strip), :domain => domain }
+      end
+      results
+    end
     
     def self.document_query(page=1)
       results = []
-      domains = [{"roe.house.gov" => 1532}, {"thornberry.house.gov" => 1776}, {"wenstrup.house.gov" => 2491}]
+      domains = [{"thornberry.house.gov" => 1776}, {"wenstrup.house.gov" => 2491}]
       domains.each do |domain|
         doc = open_html("http://"+domain.keys.first+"/news/documentquery.aspx?DocumentTypeID=#{domain.values.first}&Page=#{page}")
         return if doc.nil?
@@ -533,16 +545,5 @@ module Statement
       results.flatten
     end
     
-    def self.farenthold(year=2013)
-      results = []
-      url = "http://farenthold.house.gov/index.php?flt_m=&flt_y=#{year}&option=com_content&view=article&id=1181&Itemid=100059&layout=default"
-      doc = open_html(url)
-      return if doc.nil?
-      doc.xpath("//div[@id='idGtReportDisplay']//li").each do |row|
-        results << { :source => url, :url => 'http://farenthold.house.gov'+row.children[0]['href'], :title => row.children[0].text.strip, :date => nil, :domain => "farenthold.house.gov"}
-      end
-      results
-    end
-    
   end
 end

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "1.0.15"
+  VERSION = "1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 1.0.15
+  version: '1.1'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-17 00:00:00.000000000 Z
+date: 2013-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -214,7 +214,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4450777098153523986
+      hash: 3854190129938227300
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -223,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4450777098153523986
+      hash: 3854190129938227300
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25