workers 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b66f1edc747b5df57e671c9efe7abbd4f36a863
-  data.tar.gz: 16b9fcb187bfda5a629160de3b0c742c66f5f1b4
+  metadata.gz: 07932a16f900199549ba88c4eb68a9aee5bf79f3
+  data.tar.gz: 697e7518ffaab1a98fa034aa309b2076392b47d9
 SHA512:
-  metadata.gz: 1c4db47993b135c9dfbb6d8feb703ee82946bd67f98a953ef00673e5b83d432426764ab84e2604650b46628c55e11172b76940960024b26a1a6b9cf5a924cb06
-  data.tar.gz: 4d1a0306500a6e26741d0516ae0906b7ad9c9f5fc2e6ab76561174ab73c8dada2a88d683a4d3c2108d90b62b79c0a96e72f7f95716023c1238cea1cf36fd75bd
+  metadata.gz: 3b7ef7a498830b6d6eff47fe5068abd2180b317ddc80cd3f6d4073dd0c7defcb6d74db05664f08e757a314674c3ee750d34c2327189408d92c5ba429b7796109
+  data.tar.gz: 5559d5cd379dd854e4f1df50ecf8e3fdb8538af9a215f972dc65dd4434eb6caeeae803115687acd8a66c08fddb377e4efd018f45642e885f00459f53f700dc45

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.10.5

data/Gemfile

@@ -1,4 +1,5 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in workers.gemspec
+gem 'coveralls', :require => false
+
 gemspec

data/README.md

@@ -1,10 +1,22 @@
-# Workers
+# Workers [![Build Status](https://travis-ci.org/chadrem/workers.svg)](https://travis-ci.org/chadrem/workers) [![Coverage Status](https://coveralls.io/repos/chadrem/workers/badge.svg?branch=master&service=github)](https://coveralls.io/github/chadrem/workers?branch=master)
 
 Workers is a Ruby gem for performing work in background threads.
 Design goals include high performance, low latency, simple API, customizability, and multi-layered architecture.
 It provides a number of simple to use classes that solve a wide range of concurrency problems.
 It is used by [Tribe](https://github.com/chadrem/tribe "Tribe") to implement event-driven actors.
 
+## Contents
+
+- [Installation](#installation)
+- [Parallel Map](#parallel-map)
+- [Tasks](#tasks)
+- [Workers](#workers)
+- [Pool](#pools)
+- [Timers](#timers)
+- [Schedulers](#schedulers)
+- [Concurrency and performance](#concurrency-and-performance)
+- [Contributing](#contributing)
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -175,7 +187,7 @@ Note that you must handle exceptions yourself since you are working directly wit
 
     # Create a single worker.
     worker = Workers::Worker.new
-    
+
     # Perform some work in the background.
     25.times do |i|
       worker.perform do
@@ -187,7 +199,7 @@ Note that you must handle exceptions yourself since you are working directly wit
         end
       end
     end
-    
+
     # Tell the worker to shutdown.
     worker.shutdown
 
@@ -294,6 +306,17 @@ You can create additional schedulers as necessary:
       :pool => Workers::Pool.new        # The workers pool used to execute timer callbacks.
     )
 
+#### Bucket Schedulers
+
+The Bucket scheduler class is a specialized scheduler designed to work around lock contention.
+This is accomplished by using many pools (100 by default) each with a small number of workers (1 by default).
+Timers are assigned to a scheduler by their ````hash```` value.
+Most users will never need to use this class, but it is documented here for completeness.
+Both the number of buckets and the number of workers assigned to each bucket are configurable.
+
+    # Create a bucket scheduler.
+    scheduler = Workers::BucketScheduler.new
+
 ## Concurrency and performance
 
 Workers is tested with both JRuby and MRI (C Ruby).

data/Rakefile

@@ -1,13 +1,10 @@
 require "bundler/gem_tasks"
+require "rake/testtask"
 
-desc 'Start an IRB console with Workers loaded'
-task :console do
-  $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
-
-  require 'workers'
-  require 'irb'
-
-  ARGV.clear
-
-  IRB.start
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
 end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "workers"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/workers.rb

@@ -3,6 +3,7 @@ require 'monitor'
 require 'set'
 
 require 'workers/version'
+require 'workers/exceptions'
 require 'workers/helpers'
 require 'workers/worker'
 require 'workers/pool'
@@ -18,7 +19,7 @@ require 'workers/task_group'
 module Workers
   def self.pool
     lock do
-      return @pool ||= Workers::Pool.new
+      @pool ||= Workers::Pool.new
     end
   end
 
@@ -31,7 +32,7 @@ module Workers
 
   def self.scheduler
     lock do
-      return @scheduler ||= Workers::Scheduler.new
+      @scheduler ||= Workers::Scheduler.new
     end
   end
 

data/lib/workers/bucket_scheduler.rb

@@ -18,29 +18,25 @@ module Workers
     def schedule(timer)
       @schedulers[timer.hash % @options[:bucket_size]].schedule(timer)
 
-      return nil
+      nil
     end
 
     def unschedule(timer)
       @schedulers[timer.hash % @options[:bucket_size]].unschedule(timer)
 
-      return nil
+      nil
     end
 
     def wakeup
       @schedulers.each { |s| s.wakeup }
 
-      return nil
+      nil
     end
 
     def dispose
       @schedulers.each { |s| s.dispose }
 
-      return nil
-    end
-
-    def alive?
-      return @schedulers.all? { |s| s.alive? }
+      nil
     end
   end
 end

data/lib/workers/event.rb

@@ -7,7 +7,7 @@ module Workers
       @command = command
       @data = data
 
-      return nil
+      nil
     end
   end
 end

data/lib/workers/exceptions.rb

@@ -0,0 +1,10 @@
+module Workers
+  class WorkersError < RuntimeError; end
+  class UnknownEventError < WorkersError; end
+  class JoinError < WorkersError; end
+  class FailedTaskError < WorkersError; end
+  class InvalidStateError < WorkersError; end
+  class MaxTriesError < WorkersError; end
+  class MissingCallbackError < WorkersError; end
+  class PoolSizeError < WorkersError; end
+end

data/lib/workers/helpers.rb

@@ -17,7 +17,7 @@ module Workers
     end
 
     def concat_e(msg, e = nil)
-      return e ? "#{msg}\nEXCEPTION: #{e.message}\n#{e.backtrace.join("\n")}\n--" : msg
+      e ? "#{msg}\nEXCEPTION: #{e.message}\n#{e.backtrace.join("\n")}\n--" : msg
     end
   end
 end

data/lib/workers/log_proxy.rb

@@ -5,31 +5,31 @@ module Workers
     def initialize(logger)
       @logger = logger.is_a?(Workers::LogProxy) ? logger.logger : logger
 
-      return nil
+      nil
     end
 
     def debug(msg)
       @logger.debug(msg) if @logger
 
-      return nil
+      nil
     end
 
     def info(msg)
       @logger.info(msg) if @logger
 
-      return nil
+      nil
     end
 
     def warn(msg)
       @logger.warn(msg) if @logger
 
-      return nil
+      nil
     end
 
     def error(msg)
       @logger.error(msg) if @logger
 
-      return nil
+      nil
     end
   end
 end

data/lib/workers/periodic_timer.rb

@@ -5,7 +5,7 @@ module Workers
 
       super(delay, options, &block)
 
-      return nil
+      nil
     end
   end
 end

data/lib/workers/pool.rb

@@ -11,22 +11,23 @@ module Workers
       @lock = Monitor.new
       @workers = Set.new
       @size = 0
+      @exception_callback = options[:on_exception]
 
       expand(options[:size] || Workers::Pool::DEFAULT_POOL_SIZE)
 
-      return nil
+      nil
     end
 
     def enqueue(command, data = nil)
       @input_queue.push(Event.new(command, data))
 
-      return nil
+      nil
     end
 
     def perform(&block)
       enqueue(:perform, block)
 
-      return nil
+      nil
     end
 
     def shutdown(&block)
@@ -36,7 +37,7 @@ module Workers
         end
       end
 
-      return nil
+      nil
     end
 
     def join(max_wait = nil)
@@ -44,42 +45,40 @@ module Workers
       @workers.clear
       @size = 0
 
-      return results
+      results
     end
 
-    def dispose
-      @lock.synchronize do
-        shutdown
-        join
-      end
-
-      return nil
+    def dispose(max_wait = nil)
+      shutdown  
+      join(max_wait)
     end
 
     def inspect
-      return "#<#{self.class.to_s}:0x#{(object_id << 1).to_s(16)} size=#{size}>"
+      "#<#{self.class.to_s}:0x#{(object_id << 1).to_s(16)} size=#{size}>"
     end
 
     def size
       @lock.synchronize do
-        return @size
+        @size
       end
     end
 
     def expand(count)
       @lock.synchronize do
         count.times do
-            @workers << @worker_class.new(:input_queue => @input_queue)
-            @size += 1
+          worker = @worker_class.new(:input_queue => @input_queue, :die_on_exception => false,
+                                     :on_exception => @exception_callback, :logger => @logger)
+          @workers << worker
+          @size += 1
         end
       end
 
-      return nil
+      nil
     end
 
     def contract(count, &block)
       @lock.synchronize do
-        raise 'Count is too large.' if count > @size
+        raise Workers::PoolSizeError, 'Count is too large.' if count > @size
 
         count.times do
           callback = Proc.new do |worker|
@@ -92,7 +91,7 @@ module Workers
         end
       end
 
-      return nil
+      nil
     end
 
     def resize(new_size)
@@ -104,7 +103,7 @@ module Workers
         end
       end
 
-      return nil
+      nil
     end
 
     private
@@ -114,7 +113,7 @@ module Workers
         @workers.delete(worker)
       end
 
-      return nil
+      nil
     end
   end
 end

data/lib/workers/scheduler.rb

@@ -9,7 +9,7 @@ module Workers
       @mutex = Mutex.new
       @thread = Thread.new { start_loop }
 
-      return nil
+      nil
     end
 
     def schedule(timer)
@@ -19,7 +19,7 @@ module Workers
 
       wakeup
 
-      return nil
+      nil
     end
 
     def unschedule(timer)
@@ -27,13 +27,13 @@ module Workers
         @schedule.delete(timer)
       end
 
-      return nil
+      nil
     end
 
     def wakeup
       @thread.wakeup
 
-      return nil
+      nil
     end
 
     def dispose
@@ -43,11 +43,11 @@ module Workers
         @thread.kill
       end
 
-      return nil
+      nil
     end
 
     def alive?
-      return @thread && @thread.alive?
+      @thread && @thread.alive?
     end
 
     private
@@ -64,7 +64,7 @@ module Workers
         delay ? sleep(delay) : sleep
       end
 
-      return nil
+      nil
     end
 
     def process_overdue
@@ -84,7 +84,7 @@ module Workers
         @schedule << timer if timer.repeat
       end
 
-      return nil
+      nil
     end
 
     def next_delay

data/lib/workers/task.rb

@@ -12,19 +12,19 @@ module Workers
     def initialize(options = {})
       @logger = Workers::LogProxy.new(options[:logger])
       @input = options[:input] || []
-      @perform = options[:perform] || raise('Perform callback is required.')
+      @perform = options[:perform] || raise(Workers::MissingCallbackError, 'Perform callback is required.')
       @finished = options[:finished]
       @max_tries = options[:max_tries] || 1
       @state = :initialized
       @tries = 0
 
-      raise 'max_tries must be >= 1' unless @max_tries >= 1
+      raise Workers::MaxTriesError, 'max_tries must be >= 1' unless @max_tries >= 1
 
-      return nil
+      nil
     end
 
     def run
-      raise "Invalid state (#{@state})." unless @state == :initialized
+      raise Workers::InvalidStateError, "Invalid state (#{@state})." unless @state == :initialized
 
       @state = :running
 
@@ -43,15 +43,15 @@ module Workers
 
       @finished.call(self)
 
-      return nil
+      nil
     end
 
     def succeeded?
-      return @state == :succeeded
+      @state == :succeeded
     end
 
     def failed?
-      return @state == :failed
+      @state == :failed
     end
   end
 end

data/lib/workers/task_group.rb

@@ -15,7 +15,7 @@ module Workers
       @finished_count = 0
       @conditional = ConditionVariable.new
 
-      return nil
+      nil
     end
 
     def add(options = {}, &block)
@@ -26,7 +26,7 @@ module Workers
 
       @tasks << Workers::Task.new(options)
 
-      return nil
+      nil
     end
 
     def run
@@ -45,15 +45,15 @@ module Workers
         @conditional.wait(@internal_lock)
       end
 
-      return @tasks.all? { |t| t.succeeded? }
+      @tasks.all? { |t| t.succeeded? }
     end
 
     def successes
-      return @tasks.select { |t| t.succeeded? }
+      @tasks.select { |t| t.succeeded? }
     end
 
     def failures
-      return @tasks.select { |t| t.failed? }
+      @tasks.select { |t| t.failed? }
     end
 
     def map(inputs, options = {}, &block)
@@ -70,10 +70,10 @@ module Workers
         m = failure.exception.message
         b = failure.exception.backtrace.join("\n")
 
-        raise "At least one task failed. ARGS=#{a}, TRACE=#{m}\n#{b}\n----------\n"
+        raise Workers::FailedTaskError, "At least one task failed. ARGS=#{a}, TRACE=#{m}\n#{b}\n----------\n"
       end
 
-      return tasks.map { |t| t.result }
+      tasks.map { |t| t.result }
     end
 
     # Convenient mutex to be used by a users's task code that needs serializing.
@@ -81,17 +81,17 @@ module Workers
     def synchronize(&block)
       @external_lock.synchronize { block.call }
 
-      return nil
+      nil
     end
 
     private
 
     def state!(*args)
       unless args.include?(@state)
-        raise "Invalid state (#{@state})."
+        raise Workers::InvalidStateError, "Invalid state (#{@state})."
       end
 
-      return nil
+      nil
     end
 
     def finished(task)
@@ -100,7 +100,7 @@ module Workers
         @conditional.signal if @finished_count >= @tasks.count
       end
 
-      return nil
+      nil
     end
   end
 end

data/lib/workers/timer.rb

@@ -16,23 +16,22 @@ module Workers
       reset
       @scheduler.schedule(self)
 
-      return nil
+      nil
     end
 
     def <=>(other)
-      return sec_remaining <=> other.sec_remaining
+      sec_remaining <=> other.sec_remaining
     end
 
     def sec_remaining
       @mutex.synchronize do
         diff = @fire_at.to_f - Time.now.utc.to_f
-
-        return (diff > 0) ? diff : 0
+        (diff > 0) ? diff : 0
       end
     end
 
     def overdue?
-        return sec_remaining <= 0
+      sec_remaining <= 0
     end
 
     def fire
@@ -40,13 +39,13 @@ module Workers
         @callback.call if @callback
       end
 
-      return nil
+      nil
     end
 
     def cancel
       @scheduler.unschedule(self)
 
-      return nil
+      nil
     end
 
     def reset
@@ -54,7 +53,7 @@ module Workers
         @fire_at = Time.now.utc + @delay
       end
 
-      return nil
+      nil
     end
   end
 end

data/lib/workers/version.rb

@@ -1,3 +1,3 @@
 module Workers
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/workers/worker.rb

@@ -2,82 +2,106 @@ module Workers
   class Worker
     include Workers::Helpers
 
+    attr_accessor :exception
+
     def initialize(options = {})
       @logger = Workers::LogProxy.new(options[:logger])
       @input_queue = options[:input_queue] || Queue.new
       @thread = Thread.new { start_event_loop }
+      @exception_callback = options[:on_exception]
+      @die_on_exception = options.include?(:die_on_exception) ? options[:die_on_exception] : true
 
-      return nil
+      nil
     end
 
     def enqueue(command, data = nil)
       @input_queue.push(Event.new(command, data))
 
-      return nil
+      nil
     end
 
     def perform(&block)
       enqueue(:perform, block)
 
-      return nil
+      nil
     end
 
     def shutdown(&block)
       enqueue(:shutdown, block)
 
-      return nil
+      nil
+    end
+
+    def kill
+      @thread.kill
+
+      nil
     end
 
     def join(max_wait = nil)
-      raise "Worker can't join itself." if @thread == Thread.current
+      raise Workers::JoinError, "Worker can't join itself." if @thread == Thread.current
 
       return true if !@thread.join(max_wait).nil?
 
       @thread.kill and return false
     end
 
+    def dispose(max_wait = nil)
+      shutdown
+      join(max_wait)
+    end
+
     def alive?
-      return @thread && @thread.alive?
+      @thread && @thread.alive?
     end
 
     def inspect
-      return "#<#{self.class.to_s}:0x#{(object_id << 1).to_s(16)} #{alive? ? 'alive' : 'dead'}>"
+      "#<#{self.class.to_s}:0x#{(object_id << 1).to_s(16)} #{alive? ? 'alive' : 'dead'}>"
     end
 
     private
 
     def start_event_loop
-      while true
-        event = @input_queue.pop
-
-        case event.command
-        when :shutdown
-          shutdown_handler(event)
-          return nil
-        when :perform
-          perform_handler(event)
-        else
-          process_event(event)
-        end
+      while process_event; end
+    end
+
+    def process_event
+      event = @input_queue.pop
+
+      case event.command
+      when :shutdown
+        shutdown_handler(event)
+        return false
+      when :perform
+        perform_handler(event)
+      else
+        raise Workers::UnknownEventError, "Unhandled event (#{event.inspect})."
       end
+
+      true
     rescue Exception => e
       exception_handler(e)
+      true
     end
 
     def shutdown_handler(event)
       event.data.call(self) if event.data
+
+      nil
     end
 
     def perform_handler(event)
-      event.data.call if event.data
+      event.data.call
+
+      nil
     end
 
     def exception_handler(e)
-      puts concat_e('Worker event loop died.', e)
-    end
+      @exception = e
+      @exception_callback.call(e) if @exception_callback
+      raise(e) if @die_on_exception
 
-    def process_event(event)
-      raise "Unhandled event (#{event.inspect}). Subclass and override if you need custom events."
+      nil
     end
   end
 end

data/workers.gemspec

@@ -1,19 +1,24 @@
-# -*- encoding: utf-8 -*-
+# coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'workers/version'
 
-Gem::Specification.new do |gem|
-  gem.name          = 'workers'
-  gem.version       = Workers::VERSION
-  gem.authors       = ['Chad Remesch']
-  gem.email         = ['chad@remesch.com']
-  gem.description   = %q{Workers is a Ruby gem for performing work in background threads.}
-  gem.summary       = %q{Workers is a Ruby gem for performing work in background threads. Design goals include high performance, low latency, simple API, customizability, and multi-layered architecture. It provides a number of simple to use classes that solve a wide range of concurrency problems.}
-  gem.homepage      = 'https://github.com/chadrem/workers'
+Gem::Specification.new do |spec|
+  spec.name          = "workers"
+  spec.version       = Workers::VERSION
+  spec.authors       = ["Chad Remesch"]
+  spec.email         = ["chad@remesch.com"]
 
-  gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ['lib']
+  spec.summary       = %q{A Ruby gem for performing work in background threads.}
+  spec.homepage      = "https://github.com/chadrem/workers"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
 end

metadata

@@ -1,16 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: workers
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Chad Remesch
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2015-02-23 00:00:00.000000000 Z
-dependencies: []
-description: Workers is a Ruby gem for performing work in background threads.
+date: 2015-07-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
 email:
 - chad@remesch.com
 executables: []
@@ -18,13 +60,17 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/console
+- bin/setup
 - lib/workers.rb
 - lib/workers/bucket_scheduler.rb
 - lib/workers/event.rb
+- lib/workers/exceptions.rb
 - lib/workers/helpers.rb
 - lib/workers/log_proxy.rb
 - lib/workers/periodic_timer.rb
@@ -37,7 +83,8 @@ files:
 - lib/workers/worker.rb
 - workers.gemspec
 homepage: https://github.com/chadrem/workers
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -58,8 +105,5 @@ rubyforge_project:
 rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
-summary: Workers is a Ruby gem for performing work in background threads. Design goals
-  include high performance, low latency, simple API, customizability, and multi-layered
-  architecture. It provides a number of simple to use classes that solve a wide range
-  of concurrency problems.
+summary: A Ruby gem for performing work in background threads.
 test_files: []