vnews 0.3.5 → 0.3.6

data/lib/vnews/config.rb

@@ -1,6 +1,8 @@
 require 'vnews/sql'
 require 'yaml'
 require 'thread_pool'
+require 'vnews/constants'
+require 'timeout'
 
 class Vnews
   def self.sql_client
@@ -86,12 +88,18 @@ class Vnews
       puts "Adding feeds: #{(new_feeds - old_feeds).inspect}"
       puts "Adding folder-feed associations: #{(ff - old_ff).inspect}"
       feeds2 = []
-      pool = ThreadPool.new(10)
+      pool = ThreadPool.new Vnews::POOLSIZE
       puts "Using thread pool size of 10"
       # TODO sometimes feeds are downloaded twice;
       ff.each do |feed_url, folder|
         pool.process do 
-          feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
+          begin
+            Timeout::timeout(Vnews::TIMEOUT) do
+              feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
+            end
+          rescue Timeout::Error
+            puts "TIMEOUT ERROR: #{feed_url}"
+          end
         end
       end
       pool.join

data/lib/vnews/folder.rb

@@ -1,5 +1,6 @@
 require 'vnews/feed'
 require 'thread_pool'
+require 'vnews/constants'
 
 class Vnews
   class Folder
@@ -12,12 +13,17 @@ class Vnews
         folder = Vnews::Display.strip_item_count(folder)
         puts "Updating folder: #{folder.inspect}"
         feeds = []
-        pool = ThreadPool.new(10)
-        puts "Using thread pool size of 10"
+        pool = ThreadPool.new Vnews::POOLSIZE
+        puts "Using thread pool size of #{Vnews::POOLSIZE}"
         Vnews.sql_client.feeds_in_folder(folder.strip).each do |feed|
           pool.process do 
-            sleep(rand(10))
-            feeds << Vnews::Feed.fetch_feed(feed, folder)
+            begin
+              Timeout::timeout(Vnews::TIMEOUT) do 
+                feeds << Vnews::Feed.fetch_feed(feed, folder)
+              end
+            rescue Timeout::Error
+              puts "TIMEOUT ERROR: #{feed_url}"
+            end
           end
         end
         pool.join

data/lib/vnews/opml.rb

@@ -1,5 +1,6 @@
 require 'nokogiri'
 require 'vnews/feed'
+require 'vnews/constants'
 
 class Vnews
   class Opml
@@ -11,18 +12,24 @@ class Vnews
       doc = Nokogiri::XML.parse(opml) 
       feeds = []
       doc.xpath('/opml/body/outline').each_slice(CONCURRENCY) do |xs|
-        pool = ThreadPool.new(10)
-        puts "Using thread pool size of 10"
+        pool = ThreadPool.new Vnews::POOLSIZE
+        puts "Using thread pool size of #{Vnews::POOLSIZE}"
         xs.each do |n|
           pool.process do 
-            if n.attributes['xmlUrl']
-              feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
-            else
-              folder = n.attributes["title"].to_s
-              $stderr.print "Found folder: #{folder}\n"
-              n.xpath("outline[@xmlUrl]").each do |m|
-                feeds << Vnews::Feed.fetch_feed(m.attributes['xmlUrl'].to_s, folder)
+            begin
+              Timeout::timeout(Vnews::TIMEOUT) do 
+                if n.attributes['xmlUrl']
+                  feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
+                else
+                  folder = n.attributes["title"].to_s
+                  $stderr.print "Found folder: #{folder}\n"
+                  n.xpath("outline[@xmlUrl]").each do |m|
+                    feeds << Vnews::Feed.fetch_feed(m.attributes['xmlUrl'].to_s, folder)
+                  end
+                end
               end
+            rescue Timeout::Error
+              puts "TIMEOUT ERROR: #{feed_url}"
             end
           end
         end

data/lib/vnews/version.rb

@@ -1,4 +1,4 @@
 class Vnews
-  VERSION = '0.3.5'
+  VERSION = '0.3.6'
 end
 

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 5
-  version: 0.3.5
+  - 6
+  version: 0.3.6
 platform: ruby
 authors: 
 - Daniel Choi