vnews 0.2.3 → 0.2.4

data/README.markdown

@@ -20,7 +20,7 @@ Vnews is free and open-source.
 ## Prerequisites
 
 * a recent version of Vim (Vnews is developed against Vim 7.2)
-* a recent version of Ruby: Ruby 1.9.2 is recommended
+* Ruby 1.9: Ruby 1.9.2 is recommended
 * MySQL
 * the Unix program `tidy` 
 * the Unix program `fmt` 

data/lib/thread_pool.rb

@@ -0,0 +1,87 @@
+# taken from  http://snippets.dzone.com/posts/show/3276
+require 'thread'
+
+class ThreadPool
+  class Worker
+    def initialize
+      @mutex = Mutex.new
+      @thread = Thread.new do
+        while true
+          sleep 0.001
+          block = get_block
+          if block
+            block.call
+            reset_block
+          end
+        end
+      end
+    end
+    
+    def get_block
+      @mutex.synchronize {@block}
+    end
+    
+    def set_block(block)
+      @mutex.synchronize do
+        raise RuntimeError, "Thread already busy." if @block
+        @block = block
+      end
+    end
+    
+    def reset_block
+      @mutex.synchronize {@block = nil}
+    end
+    
+    def busy?
+      @mutex.synchronize {!@block.nil?}
+    end
+  end
+  
+  attr_accessor :max_size
+  attr_reader :workers
+
+  def initialize(max_size = 10)
+    @max_size = max_size
+    @workers = []
+    @mutex = Mutex.new
+  end
+  
+  def size
+    @mutex.synchronize {@workers.size}
+  end
+  
+  def busy?
+    @mutex.synchronize {@workers.any? {|w| w.busy?}}
+  end
+  
+  def join
+    sleep 0.01 while busy?
+  end
+  
+  def process(&block)
+    wait_for_worker.set_block(block)
+  end
+  
+  def wait_for_worker
+    while true
+      worker = find_available_worker
+      return worker if worker
+      sleep 0.01
+    end
+  end
+  
+  def find_available_worker
+    @mutex.synchronize {free_worker || create_worker}
+  end
+  
+  def free_worker
+    @workers.each {|w| return w unless w.busy?}; nil
+  end
+  
+  def create_worker
+    return nil if @workers.size >= @max_size
+    worker = Worker.new
+    @workers << worker
+    worker
+  end
+end

data/lib/vnews/config.rb

@@ -1,5 +1,6 @@
 require 'vnews/sql'
 require 'yaml'
+require 'thread_pool'
 
 class Vnews
   def self.sql_client
@@ -85,13 +86,14 @@ class Vnews
       puts "Adding feeds: #{(new_feeds - old_feeds).inspect}"
       puts "Adding folder-feed associations: #{(ff - old_ff).inspect}"
       feeds2 = []
-      threads = []
+      pool = ThreadPool.new(10)
+      puts "Using thread pool size of 10"
       ff.each do |feed_url, folder|
-        threads << Thread.new do 
+        pool.process do 
           feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
         end
       end
-      threads.each {|t| t.join}
+      pool.join
       feeds2.each do |x|
         feed_url, f, folder = *x
         folder ||= "Misc"

data/lib/vnews/folder.rb

@@ -1,4 +1,5 @@
 require 'vnews/feed'
+require 'thread_pool'
 
 class Vnews
   class Folder
@@ -10,14 +11,16 @@ class Vnews
         require 'vnews/display'
         folder = Vnews::Display.strip_item_count(folder)
         puts "Updating folder: #{folder.inspect}"
-        threads = []
         feeds = []
+        pool = ThreadPool.new(10)
+        puts "Using thread pool size of 10"
         Vnews.sql_client.feeds_in_folder(folder.strip).each do |feed|
-          threads << Thread.new do 
+          pool.process do 
+            sleep(rand(10))
             feeds << Vnews::Feed.fetch_feed(feed, folder)
           end
         end
-        threads.each {|t| t.join}
+        pool.join
         puts "Saving data to database"
         feeds.select {|x| x[1]}.compact.each do |feed_url, f, folder|
           Vnews::Feed.save_feed feed_url, f, folder

data/lib/vnews/opml.rb

@@ -11,9 +11,10 @@ class Vnews
       doc = Nokogiri::XML.parse(opml) 
       feeds = []
       doc.xpath('/opml/body/outline').each_slice(CONCURRENCY) do |xs|
-        threads = []
+        pool = ThreadPool.new(10)
+        puts "Using thread pool size of 10"
         xs.each do |n|
-          threads << Thread.new do 
+          pool.process do 
             if n.attributes['xmlUrl']
               feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
             else
@@ -25,7 +26,7 @@ class Vnews
             end
           end
         end
-        threads.each {|t| t.join}
+        pool.join
       end
 
       $stderr.puts "Making database records"

data/lib/vnews/version.rb

@@ -1,4 +1,4 @@
 class Vnews
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end
 

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Daniel Choi
@@ -78,6 +78,7 @@ files:
 - bin/vnews
 - bin/vnews-client
 - lib/create.sql
+- lib/thread_pool.rb
 - lib/vnews.rb
 - lib/vnews.vim
 - lib/vnews/autodiscoverer.rb