em-worker-pool 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,5 @@
1
+ lib/**/*.rb
2
+ bin/*
3
+ -
4
+ features/**/*.feature
5
+ LICENSE.txt
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source 'http://rubygems.org'
2
+
3
+ group :development do
4
+ gem 'bundler', '>= 1.0.0'
5
+ gem 'jeweler', '>= 1.8.4'
6
+ end
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2013 Scott Tadman
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,87 @@
1
+ # EmWorkerPool
2
+
3
+ This is a simple threaded pool worker system that can process tasks in the
4
+ order they are received.
5
+
6
+ ## Example
7
+
8
+ ```
9
+ pool = EmWorkerPool.new
10
+
11
+ pool.perform do
12
+ # ... Action to be enqueued here
13
+ end
14
+ ```
15
+
16
+ The worker pool has some sensible defaults as to how many workers will be
17
+ created, zero when there's no work, up to twenty when there's enough work
18
+ to be performed. These settings can be customized.
19
+
20
+ Constructor options:
21
+
22
+ * `:worker_class` - What kind of worker to spawn. Should be a Worker subclass.
23
+ * `:workers_min` - The minimum number of workers to have running.
24
+ * `:workers_max` - The maximum number of workers to have running.
25
+ * `:count_per_worker` - The ratio of tasks to workers.
26
+
27
+ The default `EnWorkerPool::Worker` class should suffice for most tasks.
28
+ If necessary, this can be subclassed. This would be useful if the worker
29
+ needs to perform some kind of resource initialization before it's able to
30
+ complete any tasks, such as establishing a database connection.
31
+
32
+ There is a method `after_initialize` that will execute on the worker thread
33
+ immediately after the worker is created. This is useful for performing
34
+ post-initialization functions that would otherwise block the main thread:
35
+
36
+ ```
37
+ class ExampleDatabaseWorker < EmWorkerPool::Worker
38
+ def after_initialize
39
+ # Create a database handle.
40
+ @db = DatabaseDriver::Handle.new
41
+
42
+ # Pass in the database handle as the arguments to the blocks being
43
+ # processed.
44
+ @args = [ @db ]
45
+ end
46
+ end
47
+ ```
48
+
49
+ It's also possible to re-write the `perform` method to pass in additional
50
+ arguments.
51
+
52
+ If you need to do something immediately before or after processing of a block,
53
+ two methods are available. As an example this can be used to record the amount
54
+ of time it took to complete a task:
55
+
56
+ ```
57
+ class ExampleDatabaseWorker < EmWorkerPool::Worker
58
+ def before_perform(block)
59
+ @start_time = Time.now
60
+ end
61
+
62
+ def after_perform(block)
63
+ puts "Took %ds" % (Time.now - @start_time)
64
+ end
65
+ end
66
+ ```
67
+
68
+ If exceptions are generated within the worker thread either because of
69
+ processing a task or otherwise, these are passed back to the EmWorkerPool
70
+ object via the `handle_exception` method. The default behavior is to re-raise
71
+ these, but it's also possible to perform some additional handling here to
72
+ rescue from or ignore them:
73
+
74
+ ```
75
+ class ExampleDatabasePool < EmWorkerPool
76
+ def handle_exception(worker, exception, block = nil)
77
+ # Pass through to a custom exception logger
78
+ ExceptionHandler.log(exception)
79
+ end
80
+ end
81
+ ```
82
+
83
+ ## Copyright
84
+
85
+ Copyright (c) 2013 Scott Tadman, The Working Group Inc.
86
+ See LICENSE.txt for further details.
87
+
@@ -0,0 +1,37 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rubygems'
4
+ require 'bundler'
5
+
6
+ begin
7
+ Bundler.setup(:default, :development)
8
+ rescue Bundler::BundlerError => e
9
+ $stderr.puts e.message
10
+ $stderr.puts "Run `bundle install` to install missing gems"
11
+ exit e.status_code
12
+ end
13
+
14
+ require 'rake'
15
+ require 'jeweler'
16
+
17
+ Jeweler::Tasks.new do |gem|
18
+ # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
19
+ gem.name = "em-worker-pool"
20
+ gem.homepage = "http://github.com/twg/em-worker-pool"
21
+ gem.license = "MIT"
22
+ gem.summary = %Q{EventMachine Thread Pool/Worker System}
23
+ gem.description = %Q{Simple thread pool/worker system for EventMachine}
24
+ gem.email = "scott@twg.ca"
25
+ gem.authors = [ "Scott Tadman" ]
26
+ # dependencies defined in Gemfile
27
+ end
28
+
29
+ Jeweler::RubygemsDotOrgTasks.new
30
+
31
+ require 'rake/testtask'
32
+
33
+ Rake::TestTask.new(:test) do |test|
34
+ test.libs << 'lib' << 'test'
35
+ test.pattern = 'test/**/test_*.rb'
36
+ test.verbose = true
37
+ end
data/VERSION ADDED
@@ -0,0 +1,2 @@
1
+ 0.1.0
2
+
@@ -0,0 +1,54 @@
1
+ # Generated by jeweler
2
+ # DO NOT EDIT THIS FILE DIRECTLY
3
+ # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
4
+ # -*- encoding: utf-8 -*-
5
+
6
+ Gem::Specification.new do |s|
7
+ s.name = "em-worker-pool"
8
+ s.version = "0.1.0"
9
+
10
+ s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
+ s.authors = ["Scott Tadman"]
12
+ s.date = "2013-04-30"
13
+ s.description = "Simple thread pool/worker system for EventMachine"
14
+ s.email = "scott@twg.ca"
15
+ s.extra_rdoc_files = [
16
+ "LICENSE.txt",
17
+ "README.md"
18
+ ]
19
+ s.files = [
20
+ ".document",
21
+ "Gemfile",
22
+ "LICENSE.txt",
23
+ "README.md",
24
+ "Rakefile",
25
+ "VERSION",
26
+ "em-worker-pool.gemspec",
27
+ "lib/em-worker-pool.rb",
28
+ "lib/em_worker_pool/worker.rb",
29
+ "test/helper.rb",
30
+ "test/unit/test_worker_pool.rb",
31
+ "test/unit/test_worker_pool_worker.rb"
32
+ ]
33
+ s.homepage = "http://github.com/twg/em-worker-pool"
34
+ s.licenses = ["MIT"]
35
+ s.require_paths = ["lib"]
36
+ s.rubygems_version = "1.8.25"
37
+ s.summary = "EventMachine Thread Pool/Worker System"
38
+
39
+ if s.respond_to? :specification_version then
40
+ s.specification_version = 3
41
+
42
+ if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
43
+ s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
44
+ s.add_development_dependency(%q<jeweler>, [">= 1.8.4"])
45
+ else
46
+ s.add_dependency(%q<bundler>, [">= 1.0.0"])
47
+ s.add_dependency(%q<jeweler>, [">= 1.8.4"])
48
+ end
49
+ else
50
+ s.add_dependency(%q<bundler>, [">= 1.0.0"])
51
+ s.add_dependency(%q<jeweler>, [">= 1.8.4"])
52
+ end
53
+ end
54
+
@@ -0,0 +1,147 @@
1
+ require 'thread'
2
+
3
+ class EmWorkerPool
4
+ # == Submodules ===========================================================
5
+
6
+ autoload(:Worker, 'em_worker_pool/worker')
7
+
8
+ # == Constants ============================================================
9
+
10
+ OPTIONS_DEFAULT = {
11
+ :worker_class => EmWorkerPool::Worker,
12
+ :workers_min => 0,
13
+ :workers_max => 20,
14
+ :count_per_worker => 1,
15
+ :args => [ ]
16
+ }
17
+
18
+ # == Properties ===========================================================
19
+
20
+ attr_reader *OPTIONS_DEFAULT.keys
21
+
22
+ # == Instance Methods =====================================================
23
+
24
+ # Creates a new instance of Pool with an optional set of options:
25
+ # * :worker_class - The class of worker to spawn when tasks arrive.
26
+ # * :workers_min - The minimum number of workers to have running.
27
+ # * :workers_max - The maximum number of workers to spawn.
28
+ # * :count_per_worker - Ratio of items in queue to workers.
29
+ # * :args - An array of arguments to be passed through to the workers.
30
+ def initialize(options = nil)
31
+ @queue = Queue.new
32
+ @workers = [ ]
33
+
34
+ options = options ? OPTIONS_DEFAULT.merge(options) : OPTIONS_DEFAULT
35
+
36
+ @worker_class = options[:worker_class]
37
+ @workers_min = options[:workers_min]
38
+ @workers_max = options[:workers_max]
39
+ @count_per_worker = options[:count_per_worker]
40
+ @args = options[:args]
41
+
42
+ @workers_min.times do
43
+ self.worker_create!
44
+ end
45
+ end
46
+
47
+ # Returns the number of active worker threads.
48
+ def workers_count
49
+ @workers.length
50
+ end
51
+
52
+ # Returns the number of workers required for the current loading.
53
+ def workers_needed
54
+ n = ((@queue.length + @workers.length - @queue.num_waiting) / @count_per_worker)
55
+
56
+ if (n > @workers_max)
57
+ @workers_max
58
+ elsif (n < @workers_min)
59
+ @workers_min
60
+ else
61
+ n
62
+ end
63
+ end
64
+
65
+ # Makes a blocking call to pop an item from the queue, returning that item.
66
+ # If the queue is empty, also has the effect of sleeping the calling thread
67
+ # until something is pushed into the queue.
68
+ def block_pop
69
+ @queue.pop
70
+ end
71
+
72
+ # Returns true if more workers are needed to satisfy the current backlog,
73
+ # or false otherwise.
74
+ def workers_needed?
75
+ @workers.length < self.workers_needed
76
+ end
77
+
78
+ def worker_needed?(worker)
79
+ @queue.length > 0 or @workers.length <= self.workers_needed
80
+ end
81
+
82
+ # Returns an array of the current workers.
83
+ def workers
84
+ @workers.dup
85
+ end
86
+
87
+ # Returns the current number of workers.
88
+ def workers_count
89
+ @workers.length
90
+ end
91
+
92
+ # Returns true if there are any workers, false otherwise.
93
+ def workers?
94
+ @workers.any?
95
+ end
96
+
97
+ # Returns true if there is some outstanding work to be performed, false
98
+ # otherwise.
99
+ def busy?
100
+ @queue.num_waiting < @workers.length
101
+ end
102
+
103
+ # Returns the number of workers that are waiting for something to do.
104
+ def waiting
105
+ @queue.num_waiting
106
+ end
107
+
108
+ # Returns true if anything is queued, false otherwise. Note that this does
109
+ # not count anything that might be active within a worker.
110
+ def queue?
111
+ @queue.length > 0
112
+ end
113
+
114
+ # Returns the number of items in the queue. Note that this does not count
115
+ # anything that might be active within a worker.
116
+ def queue_size
117
+ @queue.size
118
+ end
119
+
120
+ # Receives reports of exceptions from workers. Default behavior is to re-raise.
121
+ def report_exception!(worker, exception, block = nil)
122
+ raise(exception)
123
+ end
124
+
125
+ # Schedules a block to be acted upon by the workers.
126
+ def perform
127
+ @queue << Proc.new
128
+
129
+ if (self.workers_count < self.workers_needed)
130
+ self.worker_create!
131
+ end
132
+
133
+ true
134
+ end
135
+
136
+ # Called by a worker when it's finished. Should not be called otherwise.
137
+ def worker_finished!(worker)
138
+ @workers.delete(worker)
139
+ worker.join
140
+ end
141
+
142
+ protected
143
+ # Creates a new worker and puts it into the list of available workers.
144
+ def worker_create!
145
+ @workers << worker_class.new(self, *@args)
146
+ end
147
+ end
@@ -0,0 +1,71 @@
1
+ class EmWorkerPool::Worker
2
+ # == Properties ===========================================================
3
+
4
+ attr_reader :pool
5
+ attr_reader :args
6
+ attr_reader :block
7
+
8
+ # == Instance Methods =====================================================
9
+
10
+ # Creates a new worker attached to the provided pool. Optional arguments
11
+ # may be supplied, which are passed on to the blocks it processes.
12
+ def initialize(pool, *args)
13
+ @pool = pool
14
+ @args = args
15
+
16
+ @thread = Thread.new do
17
+ Thread.abort_on_exception = true
18
+ begin
19
+ self.after_initialize
20
+
21
+ while (block = @pool.block_pop)
22
+ begin
23
+ @block = block
24
+ self.before_perform(block)
25
+ perform(&block)
26
+ self.after_perform(block)
27
+ @block = nil
28
+
29
+ unless (@pool.worker_needed?(self))
30
+ @pool.worker_finished!(self)
31
+ break
32
+ end
33
+ rescue => exception
34
+ @pool.handle_exception(self, exception, block)
35
+ end
36
+ end
37
+ rescue => exception
38
+ @pool.handle_exception(self, exception, nil)
39
+ end
40
+ end
41
+ end
42
+
43
+ # Calls the Proc pulled from the queue. Subclasses can implement their own
44
+ # method here which might pass in arguments to the block for contextual
45
+ # purposes.
46
+ def perform
47
+ yield(*@args)
48
+ end
49
+
50
+ # This method is called after the worker is initialized within the thread
51
+ # used by the worker. It can be customized in sub-classes as required.
52
+ def after_initialize
53
+ end
54
+
55
+ # This method is called just before the worker executes the given block.
56
+ # This should be customized in sub-classes to do any additional processing
57
+ # required.
58
+ def before_perform(block)
59
+ end
60
+
61
+ # This method is called just after the worker has finished executing the
62
+ # given block This should be customized in sub-classes to do any additional
63
+ # processing required.
64
+ def after_perform(block)
65
+ end
66
+
67
+ # Called by the pool to reap this thread when it is finished.
68
+ def join
69
+ @thread.join
70
+ end
71
+ end
@@ -0,0 +1,31 @@
1
+ require 'rubygems'
2
+ require 'bundler'
3
+
4
+ begin
5
+ Bundler.setup(:default, :development)
6
+ rescue Bundler::BundlerError => e
7
+ $stderr.puts e.message
8
+ $stderr.puts "Run `bundle install` to install missing gems"
9
+ exit e.status_code
10
+ end
11
+
12
+ require 'test/unit'
13
+
14
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
15
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
16
+
17
+ require 'em-worker-pool'
18
+
19
+ class Test::Unit::TestCase
20
+ def assert_eventually(time = nil, message = nil, &block)
21
+ start_time = Time.now.to_i
22
+
23
+ while (!block.call)
24
+ select(nil, nil, nil, 0.1)
25
+
26
+ if (time and (Time.now.to_i - start_time > time))
27
+ flunk(message || 'assert_eventually timed out')
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,142 @@
1
+ require_relative '../helper'
2
+
3
+ class TestWorkerPool < Test::Unit::TestCase
4
+ def test_defaults
5
+ pool = EmWorkerPool.new
6
+
7
+ assert_equal false, pool.queue?
8
+ assert_equal 0, pool.queue_size
9
+
10
+ assert_equal 0, pool.workers_count
11
+ assert_equal 0, pool.workers_needed
12
+ assert_equal false, pool.workers_needed?
13
+ assert_equal 0, pool.waiting
14
+
15
+ assert_equal [ ], pool.workers
16
+
17
+ options_default = EmWorkerPool::OPTIONS_DEFAULT
18
+
19
+ assert_equal options_default[:worker_class], pool.worker_class
20
+ assert_equal options_default[:workers_min], pool.workers_min
21
+ assert_equal options_default[:workers_max], pool.workers_max
22
+ assert_equal options_default[:count_per_worker], pool.count_per_worker
23
+ assert_equal options_default[:args], pool.args
24
+ end
25
+
26
+ class ExampleWorker < EmWorkerPool::Worker
27
+ attr_reader :after_initialized
28
+ attr_reader :before_performed
29
+ attr_reader :after_performed
30
+
31
+ def after_initialize
32
+ @after_initialized = :after_initialized
33
+ end
34
+
35
+ def before_perform(block)
36
+ @before_performed = :before_performed
37
+ end
38
+
39
+ def after_perform(block)
40
+ @after_performed = :after_performed
41
+ end
42
+ end
43
+
44
+ def test_options
45
+ pool = EmWorkerPool.new(
46
+ :worker_class => ExampleWorker,
47
+ :workers_min => 1,
48
+ :workers_max => 5,
49
+ :count_per_worker => 2,
50
+ :args => [ :example ]
51
+ )
52
+
53
+ assert_equal false, pool.queue?
54
+ assert_equal 0, pool.queue_size
55
+
56
+ assert_equal 1, pool.workers_count
57
+ assert_equal 1, pool.workers_needed
58
+ assert_equal false, pool.workers_needed?
59
+
60
+ assert_equal ExampleWorker, pool.worker_class
61
+ assert_equal 1, pool.workers_min
62
+ assert_equal 5, pool.workers_max
63
+ assert_equal 2, pool.count_per_worker
64
+ assert_equal [ :example ], pool.args
65
+
66
+ worker = pool.workers[0]
67
+
68
+ assert worker
69
+ assert worker.is_a?(ExampleWorker)
70
+
71
+ assert_equal pool, worker.pool
72
+ assert_equal nil, worker.block
73
+ assert_equal [ :example ], worker.args
74
+
75
+ assert_eventually(1) do
76
+ worker.after_initialized
77
+ end
78
+
79
+ assert_equal :after_initialized, worker.after_initialized
80
+ assert_equal nil, worker.before_performed
81
+ assert_equal nil, worker.after_performed
82
+ end
83
+
84
+ def test_simple_tasks
85
+ pool = EmWorkerPool.new
86
+ count = 0
87
+
88
+ 100.times do
89
+ pool.perform do
90
+ count += 1
91
+ end
92
+ end
93
+
94
+ assert_eventually do
95
+ count == 100
96
+ end
97
+ end
98
+
99
+ def test_with_context
100
+ pool = EmWorkerPool.new(
101
+ :args => [ :test, 'arguments' ]
102
+ )
103
+
104
+ args = nil
105
+
106
+ pool.perform do |*_args|
107
+ args = _args
108
+ end
109
+
110
+ assert_eventually(1) do
111
+ args
112
+ end
113
+
114
+ assert_equal [ :test, 'arguments' ], args
115
+ end
116
+
117
+ def test_with_recursion
118
+ pool = EmWorkerPool.new
119
+
120
+ times = 100
121
+ queued = 0
122
+ count = 0
123
+
124
+ times.times do |n|
125
+ pool.perform do
126
+ n.times do
127
+ queued += 1
128
+ pool.perform do |*args|
129
+ count += 1
130
+ end
131
+ end
132
+ end
133
+ end
134
+
135
+ assert_eventually(600) do
136
+ !pool.queue? and !pool.busy?
137
+ end
138
+
139
+ assert_equal (0..times - 1).inject(0) { |s, r| s + r }, queued
140
+ assert_equal queued, count
141
+ end
142
+ end
@@ -0,0 +1,14 @@
1
+ require_relative '../helper'
2
+
3
+ class TestEmWorkerPoolWorker < Test::Unit::TestCase
4
+ def test_defaults
5
+ pool = EmWorkerPool.new
6
+ args = %w[ test arguments ]
7
+
8
+ worker = EmWorkerPool::Worker.new(pool, *args)
9
+
10
+ assert_equal pool, worker.pool
11
+ assert_equal nil, worker.block
12
+ assert_equal args, worker.args
13
+ end
14
+ end
metadata ADDED
@@ -0,0 +1,91 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: em-worker-pool
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Scott Tadman
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-04-30 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: 1.0.0
22
+ type: :development
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.0.0
30
+ - !ruby/object:Gem::Dependency
31
+ name: jeweler
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ! '>='
36
+ - !ruby/object:Gem::Version
37
+ version: 1.8.4
38
+ type: :development
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: 1.8.4
46
+ description: Simple thread pool/worker system for EventMachine
47
+ email: scott@twg.ca
48
+ executables: []
49
+ extensions: []
50
+ extra_rdoc_files:
51
+ - LICENSE.txt
52
+ - README.md
53
+ files:
54
+ - .document
55
+ - Gemfile
56
+ - LICENSE.txt
57
+ - README.md
58
+ - Rakefile
59
+ - VERSION
60
+ - em-worker-pool.gemspec
61
+ - lib/em-worker-pool.rb
62
+ - lib/em_worker_pool/worker.rb
63
+ - test/helper.rb
64
+ - test/unit/test_worker_pool.rb
65
+ - test/unit/test_worker_pool_worker.rb
66
+ homepage: http://github.com/twg/em-worker-pool
67
+ licenses:
68
+ - MIT
69
+ post_install_message:
70
+ rdoc_options: []
71
+ require_paths:
72
+ - lib
73
+ required_ruby_version: !ruby/object:Gem::Requirement
74
+ none: false
75
+ requirements:
76
+ - - ! '>='
77
+ - !ruby/object:Gem::Version
78
+ version: '0'
79
+ required_rubygems_version: !ruby/object:Gem::Requirement
80
+ none: false
81
+ requirements:
82
+ - - ! '>='
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
85
+ requirements: []
86
+ rubyforge_project:
87
+ rubygems_version: 1.8.25
88
+ signing_key:
89
+ specification_version: 3
90
+ summary: EventMachine Thread Pool/Worker System
91
+ test_files: []