statement 0.9.9 → 1.0

data/README.md

@@ -44,6 +44,13 @@ puts results.first
 {:source=>"http://blumenauer.house.gov/index.php?option=com_bca-rss-syndicator&feed_id=1", :url=>"http://blumenauer.house.gov/index.php?option=com_content&amp;view=article&amp;id=2203:blumenauer-qwe-need-a-national-system-that-speaks-to-the-transportation-challenges-of-todayq&amp;catid=66:2013-press-releases", :title=>"Blumenauer: &quot;We need a national system that speaks to the transportation challenges of ...", :date=>#<Date: 2013-04-24 ((2456407j,0s,0n),+0s,2299161j)>, :domain=>"blumenauer.house.gov"}
 ```
 
+If you have a batch of RSS URLs, you can pass them to Feed's `batch` class method, which will use Typhoeus to fetch them in parallel:
+
+```ruby
+urls = ['http://aderholt.house.gov/common/rss//index.cfm?rss=20', 'http://andrews.house.gov/rss.xml', "http://alexander.house.gov/common/rss/?rss=24", "http://amash.house.gov/rss.xml"]
+results = Statement::Feed.batch(urls)
+```
+
 The sites that require HTML scraping are detailed in individual methods, and can be called individually or in bulk:
 
 ```ruby

data/lib/statement/feed.rb

@@ -3,9 +3,25 @@ require 'uri'
 require 'open-uri'
 require 'american_date'
 require 'nokogiri'
+require 'typhoeus'
 
 module Statement
   class Feed
+    
+    def self.batch(urls)
+      results = []
+      hydra = Typhoeus::Hydra.new
+      urls.each do |url|
+        req = Typhoeus::Request.new(url)
+        req.on_complete do |response|
+          doc = Nokogiri::XML(response.body)
+          results << parse_rss(doc, url)
+        end
+        hydra.queue(req)
+      end
+      hydra.run
+      results.flatten
+    end
   
     def self.open_rss(url)
       begin
@@ -28,6 +44,10 @@ module Statement
     def self.from_rss(url)
       doc = open_rss(url)
       return unless doc
+      parse_rss(doc, url)
+    end
+    
+    def self.parse_rss(doc, url)
       links = doc.xpath('//item')
       results = links.map do |link|
         abs_link = Utils.absolute_link(url, link.xpath('link').text)

data/lib/statement/version.rb

@@ -1,3 +1,3 @@
 module Statement
-  VERSION = "0.9.9"
+  VERSION = "1.0"
 end

data/statement.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency "koala"
   spec.add_dependency "oj"
   spec.add_dependency "twitter"
+  spec.add_dependency "typhoeus"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statement
 version: !ruby/object:Gem::Version
-  version: 0.9.9
+  version: '1.0'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-10 00:00:00.000000000 Z
+date: 2013-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2160388100 !ruby/object:Gem::Requirement
+  requirement: &2152391660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '1.3'
   type: :development
   prerelease: false
-  version_requirements: *2160388100
+  version_requirements: *2152391660
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2160387540 !ruby/object:Gem::Requirement
+  requirement: &2152391200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2160387540
+  version_requirements: *2152391200
 - !ruby/object:Gem::Dependency
   name: webmock
-  requirement: &2160386960 !ruby/object:Gem::Requirement
+  requirement: &2152407040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2160386960
+  version_requirements: *2152407040
 - !ruby/object:Gem::Dependency
   name: american_date
-  requirement: &2160386260 !ruby/object:Gem::Requirement
+  requirement: &2152406620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160386260
+  version_requirements: *2152406620
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2160385680 !ruby/object:Gem::Requirement
+  requirement: &2152406180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160385680
+  version_requirements: *2152406180
 - !ruby/object:Gem::Dependency
   name: koala
-  requirement: &2160385220 !ruby/object:Gem::Requirement
+  requirement: &2152405700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160385220
+  version_requirements: *2152405700
 - !ruby/object:Gem::Dependency
   name: oj
-  requirement: &2160384700 !ruby/object:Gem::Requirement
+  requirement: &2152405200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160384700
+  version_requirements: *2152405200
 - !ruby/object:Gem::Dependency
   name: twitter
-  requirement: &2160384260 !ruby/object:Gem::Requirement
+  requirement: &2152404700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,7 +98,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2160384260
+  version_requirements: *2152404700
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: &2152403760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152403760
 description: Crawls congressional websites for press releases.
 email:
 - dwillis@gmail.com