workers 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e3fff7e649a2c1b3cb5c79855466ffc591a1e89e
-  data.tar.gz: 65bbb33f63f9427a8c6b58961834e44baeac1334
+  metadata.gz: 50e412eae6d1bdf77fe6588ffc6a8d2412f739f7
+  data.tar.gz: ab20ac26b42dd524faa2d0d1171e6887de2e3bbf
 SHA512:
-  metadata.gz: 3324a9047a862ce798de1b78966ca4526b690fdee1ea17933d2cfa4f3c74f9abf9e63eaec91181aa48954b3da213fc94123e4dab52275029bd7859d7d62aaad7
-  data.tar.gz: e8b7a6b839014c1771dbc23c6fba4488c74338fef156bb2d272cb0718e0a51bf456553a4e98b880f89913a2607cea4411a4e197b2cce3ac146fe68ee0501a373
+  metadata.gz: d22b5c5fb81b2b5b7105c1d237b2d9b779af7b2ef9cad993cd80c9b012a851390d851ec976497f51d87a7cbc2265d31e3712c4d2b2d2572da65c7ed8fa9fbec7
+  data.tar.gz: 3c44a7b8cc2e9943e919cb02e231f2d25511b5b24a4df1be847ef7e9c3b7ae083d07d35b3f1f99c25d5e8784745953300301ebecce7b67c155de3182ca6ecd94

data/README.md

@@ -42,7 +42,7 @@ Any exceptions while mapping with cause the entire map method to fail.
 If your block is prone to temporary failures (exceptions), you can retry it.
 
     Workers.map([1, 2, 3, 4, 5], :max_tries => 100) do |i|
-      if rand <= 0.5
+      if rand <= 0.8
         puts "Sometimes I like to fail while computing #{i} * #{i}."
         raise 'sad face'
       end
@@ -53,7 +53,7 @@ If your block is prone to temporary failures (exceptions), you can retry it.
 ## Tasks
 
 Tasks and task groups provide more flexibility than parallel map.
-The main benefit is that you get to decide how you want to handle exceptions.
+For example, you get to decide how you want to handle exceptions.
 
     # Create a task group (it contains a pool of worker threads).
     group = Workers::TaskGroup.new
@@ -61,8 +61,12 @@ The main benefit is that you get to decide how you want to handle exceptions.
     # Add tasks to the group.
     10.times do |i|
       10.times do |j|
-        group.add do
+        group.add(:max_tries => 10) do
           group.synchronize { puts "Computing #{i} * #{j}..." }
+          if rand <= 0.9
+            group.synchronize { puts "Sometimes I like to fail while computing #{i} * #{i}." }
+            raise 'sad face'
+          end
           i * j # Last statement executed is the result of the task.
         end
       end
@@ -104,9 +108,9 @@ This method uses a mutex so you can serialize portions of your tasks that aren't
 
     task = Workers::Task.new(
       :logger => nil,                   # Ruby logger instance.
-      :perform => proc {},              # Required option.  Block of code to run.
+      :on_perform => proc {},           # Required option.  Block of code to run.
       :args => [],                      # Array of arguments passed to the 'perform' block.
-      :finished => nil,                 # Callback to execute after attempting to run the task.
+      :on_finished => nil,              # Callback to execute after attempting to run the task.
       :max_tries => 1,                  # Number of times to try completing the task (without an exception).
     )
 
@@ -116,32 +120,27 @@ This method uses a mutex so you can serialize portions of your tasks that aren't
 
 The main purpose of the Worker class is to add an event system on top of Ruby's built-in Thread class.
 This greatly simplifies inter-thread communication.
-Workers are fairly low level and don't handle exceptions for you.
-Failing to handle exceptions will result in dead workers so you must rescue them in your application code.
+You must manually dispose of pools and workers so they get garbage collected.
 
     # Initialize a worker pool.
-    pool = Workers::Pool.new
+    pool = Workers::Pool.new(:on_exception => proc { |e|
+      puts "A worker encountered an exception: #{e.class}: #{e.message}"
+    })
 
     # Perform some work in the background.
     100.times do
       pool.perform do
-        begin
-          sleep(rand(3))
-          puts "Hello world from thread #{Thread.current.object_id}"
-        rescue Exception => e
-          puts "Oh no, my hello world failed!"
-        end
+        sleep(rand(3))
+        raise 'sad face' if rand < 0.5
+        puts "Hello world from thread #{Thread.current.object_id}"
       end
     end
 
-    # Tell the workers to shutdown.
-    pool.shutdown do
+    # Wait up to 30 seconds for the workers to cleanly shutdown (or forcefully kill them).
+    pool.dispose(30) do
       puts "Worker thread #{Thread.current.object_id} is shutting down."
     end
 
-    # Wait for the workers to shutdown.
-    pool.join
-
 #### Advanced
 
 The Worker class is designed to be customized through inheritence and its event system:
@@ -153,12 +152,8 @@ The Worker class is designed to be customized through inheritence and its event
       def event_handler(event)
         case event.command
         when :my_custom
-          begin
-            puts "Worker received custom event: #{event.data}"
-            sleep(1)
-          rescue Exception => e
-            puts "This is a very sad program."
-          end
+          puts "Worker received custom event: #{event.data}"
+          sleep(1)
         else
           super(event)
         end
@@ -166,27 +161,25 @@ The Worker class is designed to be customized through inheritence and its event
     end
 
     # Create a pool that uses your custom worker class.
-    pool = Workers::Pool.new(:worker_class => CustomWorker)
+    pool = Workers::Pool.new(:worker_class => CustomWorker, :on_exception => proc { |e|
+      puts "A worker encountered an exception: #{e.class}: e.message}"
+    })
 
     # Tell the workers to do some work using custom events.
     100.times do |i|
       pool.enqueue(:my_custom, i)
     end
 
-    # Tell the workers to shutdown.
-    pool.shutdown do
+    # Wait up to 30 seconds for the workers to cleanly shutdown (or forcefully kill them).
+    pool.dispose(30) do
       puts "Worker thread #{Thread.current.object_id} is shutting down."
     end
 
-    # Wait for the workers to shutdown.
-    pool.join
-
 #### Without pools
 
 In most cases you will be using a group of workers (a pool) as demonstrated above.
 In certain cases, you may want to use a worker directly without the pool.
-This effectively gives you direct access to a single event-driven thread.
-Note that you must handle exceptions yourself since you are working directly with the worker class:
+This gives you direct access to a single event-driven thread that won't die on an exception.
 
     # Create a single worker.
     worker = Workers::Worker.new
@@ -194,23 +187,20 @@ Note that you must handle exceptions yourself since you are working directly wit
     # Perform some work in the background.
     25.times do |i|
       worker.perform do
-        begin
-          sleep(0.1)
-          puts "Hello world from thread #{Thread.current.object_id}"
-        rescue Exception => e
-          puts "Oh no, my hello world failed!"
-        end
+        sleep(0.1)
+        puts "Hello world from thread #{Thread.current.object_id}"
       end
     end
 
-    # Tell the worker to shutdown.
-    worker.shutdown
+    # Wait up to 30 seconds for the worker to cleanly shutdown (or forcefully kill it).
+    worker.dispose(30)
 
 #### Options
 
     worker = Workers::Worker.new(
       :logger => nil,                   # Ruby Logger instance.
-      :input_queue => nil               # Ruby Queue used for input events.
+      :input_queue => nil,              # Ruby Queue used for input events.
+      :on_exception => nil              # Callback to execute on exception (exception passed as only argument).
     )
 
 ## Pools
@@ -240,6 +230,7 @@ Pools can be adjusted using the below methods:
       :size => 20,                      # Number of threads to create.
       :logger => nil,                   # Ruby Logger instance.
       :worker_class => Workers::Worker  # Class of worker to use for this pool.
+      :on_exception => nil              # Callback to execute on exception (exception passed as only argument).
     )
 
 ## Timers
@@ -346,7 +337,7 @@ Threads performing such IO will temporarily release the GIL and thus let another
 
 #### MRI 1.8.x or older (not supported)
 
-These old versions of Ruby used green threads (application layer threads) instead of operating system level threads.
+These old versions of Ruby use green threads (application layer threads) instead of operating system level threads.
 I recommend you upgrade to a newer version as I haven't tested Workers with them.
 They also aren't officially supported by the Ruby community at this point.
 

data/lib/workers/pool.rb

@@ -4,6 +4,8 @@ module Workers
 
     DEFAULT_POOL_SIZE = 20
 
+    attr_accessor :on_exception
+
     def initialize(options = {})
       @logger = Workers::LogProxy.new(options[:logger])
       @worker_class = options[:worker_class] || Workers::Worker
@@ -11,7 +13,7 @@ module Workers
       @lock = Monitor.new
       @workers = Set.new
       @size = 0
-      @exception_callback = options[:on_exception]
+      @on_exception = options[:on_exception]
 
       expand(options[:size] || Workers::Pool::DEFAULT_POOL_SIZE)
 
@@ -48,8 +50,11 @@ module Workers
       results
     end
 
-    def dispose(max_wait = nil)
-      shutdown  
+    def dispose(max_wait = nil, &block)
+      shutdown do
+        block.call if block
+      end
+
       join(max_wait)
     end
 
@@ -66,8 +71,7 @@ module Workers
     def expand(count)
       @lock.synchronize do
         count.times do
-          worker = @worker_class.new(:input_queue => @input_queue, :die_on_exception => false,
-                                     :on_exception => @exception_callback, :logger => @logger)
+          worker = @worker_class.new(:input_queue => @input_queue, :on_exception => @on_exception, :logger => @logger)
           @workers << worker
           @size += 1
         end

data/lib/workers/task.rb

@@ -12,8 +12,8 @@ module Workers
     def initialize(options = {})
       @logger = Workers::LogProxy.new(options[:logger])
       @input = options[:input] || []
-      @perform = options[:perform] || raise(Workers::MissingCallbackError, 'Perform callback is required.')
-      @finished = options[:finished]
+      @on_perform = options[:on_perform] || raise(Workers::MissingCallbackError, 'on_perform callback is required.')
+      @on_finished = options[:on_finished]
       @max_tries = options[:max_tries] || 1
       @state = :initialized
       @tries = 0
@@ -32,7 +32,7 @@ module Workers
         @tries += 1
 
         begin
-          @result = @perform.call(@input)
+          @result = @on_perform.call(@input)
           @state = :succeeded
           @exception = nil
         rescue Exception => e
@@ -41,7 +41,7 @@ module Workers
         end
       end
 
-      @finished.call(self)
+      @on_finished.call(self)
 
       nil
     end

data/lib/workers/task_group.rb

@@ -21,8 +21,8 @@ module Workers
     def add(options = {}, &block)
       state!(:initialized)
 
-      options[:finished] = method(:finished)
-      options[:perform] ||= block
+      options[:on_finished] = method(:finished)
+      options[:on_perform] ||= block
 
       @tasks << Workers::Task.new(options)
 
@@ -67,10 +67,11 @@ module Workers
 
       if (failure = failures[0])
         a = failure.input.inspect
+        c = failure.exception.class.to_s
         m = failure.exception.message
         b = failure.exception.backtrace.join("\n")
 
-        raise Workers::FailedTaskError, "At least one task failed. ARGS=#{a}, TRACE=#{m}\n#{b}\n----------\n"
+        raise Workers::FailedTaskError, "#{failures.count} task(s) failed (Only the first failure is shown).\nARGS=#{a}, EXCEPTION=#{c}: #{m}\n#{b}\n----------\n"
       end
 
       tasks.map { |t| t.result }

data/lib/workers/version.rb

@@ -1,3 +1,3 @@
 module Workers
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

data/lib/workers/worker.rb

@@ -3,13 +3,13 @@ module Workers
     include Workers::Helpers
 
     attr_accessor :exception
+    attr_accessor :on_exception
 
     def initialize(options = {})
       @logger = Workers::LogProxy.new(options[:logger])
       @input_queue = options[:input_queue] || Queue.new
       @thread = Thread.new { start_event_loop }
-      @exception_callback = options[:on_exception]
-      @die_on_exception = options.include?(:die_on_exception) ? options[:die_on_exception] : true
+      @on_exception = options[:on_exception]
       @run = true
 
       nil
@@ -105,8 +105,12 @@ module Workers
 
     def exception_handler(e)
       @exception = e
-      @exception_callback.call(e) if @exception_callback
-      raise(e) if @die_on_exception
+
+      begin
+        @on_exception.call(e) if @on_exception
+      rescue Exception => e2
+        STDERR.puts "Your #{self.class.to_s} exception handler raised an error. Fix your handler!\n#{e2.class.to_s}: #{e2.message}\n#{e2.backtrace.join("\n")}"
+      end
 
       nil
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workers
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Chad Remesch
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler