worker_roulette 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/worker_roulette/foreman.rb +1 -1
- data/lib/worker_roulette/tradesman.rb +3 -3
- data/lib/worker_roulette/version.rb +1 -1
- data/lib/worker_roulette.rb +32 -7
- data/spec/integration/worker_roulette_spec.rb +10 -7
- data/spec/spec_helper.rb +41 -15
- data/worker_roulette.gemspec +4 -7
- metadata +55 -7
@@ -22,7 +22,7 @@ module WorkerRoulette
|
|
22
22
|
#is added to the job_queue. This is not a big deal bc it just means that
|
23
23
|
#the sender's queue will be processed one slot behind it's rightful place.
|
24
24
|
#This does not effect work_order ordering.
|
25
|
-
@redis_pool.with do |redis|
|
25
|
+
@redis_pool.with({}) do |redis|
|
26
26
|
@count = redis.incr(COUNTER_KEY)
|
27
27
|
redis.multi do
|
28
28
|
redis.zadd(WorkerRoulette::JOB_BOARD, @count, sender)
|
@@ -11,7 +11,7 @@ module WorkerRoulette
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def wait_for_work_orders(on_subscribe_callback = nil, &block)
|
14
|
-
@pubsub_pool.with do |redis|
|
14
|
+
@pubsub_pool.with({}) do |redis|
|
15
15
|
redis.subscribe(WorkerRoulette::JOB_NOTIFICATIONS) do |on|
|
16
16
|
on.subscribe {on_subscribe_callback.call if on_subscribe_callback}
|
17
17
|
on.message {block.call(work_orders!) if block}
|
@@ -20,7 +20,7 @@ module WorkerRoulette
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def work_orders!
|
23
|
-
@client_pool.with do |redis|
|
23
|
+
@client_pool.with({}) do |redis|
|
24
24
|
get_sender_for_next_job(redis)
|
25
25
|
results = redis.multi do
|
26
26
|
redis.lrange(sender, 0, -1)
|
@@ -32,7 +32,7 @@ module WorkerRoulette
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def unsubscribe
|
35
|
-
@pubsub_pool.with {|redis| redis.unsubscribe}
|
35
|
+
@pubsub_pool.with({}) {|redis| redis.unsubscribe}
|
36
36
|
end
|
37
37
|
|
38
38
|
private
|
data/lib/worker_roulette.rb
CHANGED
@@ -1,21 +1,25 @@
|
|
1
1
|
require "worker_roulette/version"
|
2
2
|
require 'redis'
|
3
3
|
require 'hiredis'
|
4
|
-
require 'redis/connection/hiredis'
|
5
|
-
require 'connection_pool'
|
6
4
|
require 'oj'
|
5
|
+
require 'em-synchrony'
|
7
6
|
|
8
7
|
Dir[File.join(File.dirname(__FILE__),'worker_roulette','**','*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
9
8
|
|
9
|
+
class EventMachine::Synchrony::ConnectionPool
|
10
|
+
alias_method :with, :execute
|
11
|
+
end
|
12
|
+
|
10
13
|
module WorkerRoulette
|
11
14
|
JOB_BOARD = "job_board"
|
12
15
|
JOB_NOTIFICATIONS = "new_job_ready"
|
13
16
|
|
14
|
-
def self.start(
|
15
|
-
|
16
|
-
@
|
17
|
-
@
|
18
|
-
@
|
17
|
+
def self.start(config = {})
|
18
|
+
@redis_config = {host: 'localhost', db: 14, driver: :hiredis, timeout: 5, pool_size: 10}.merge(config)
|
19
|
+
@pool_config = Hash[size: @redis_config.delete(:pool_size), timeout: @redis_config.delete(:timeout)]
|
20
|
+
@foreman_connection_pool = connection_pool.new(@pool_config) {Redis.new(@redis_config)}
|
21
|
+
@tradesman_connection_pool = connection_pool.new(@pool_config) {Redis.new(@redis_config)}
|
22
|
+
@pubsub_connection_pool = connection_pool.new(@pool_config) {Redis.new(@redis_config)}
|
19
23
|
end
|
20
24
|
|
21
25
|
def self.foreman(sender)
|
@@ -35,4 +39,25 @@ module WorkerRoulette
|
|
35
39
|
def self.pubsub_connection_pool
|
36
40
|
@pubsub_connection_pool
|
37
41
|
end
|
42
|
+
|
43
|
+
def self.pool_size
|
44
|
+
@pool_config[:size]
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.redis_config
|
48
|
+
@redis_config.dup
|
49
|
+
end
|
50
|
+
|
51
|
+
private
|
52
|
+
def connection_pool
|
53
|
+
if @redis_config[:driver] == :synchrony
|
54
|
+
require 'redis/connection/synchrony'
|
55
|
+
require 'em-synchrony/connection_pool'
|
56
|
+
EM::Synchrony::ConnectionPool
|
57
|
+
else
|
58
|
+
require 'redis/connection/hiredis'
|
59
|
+
require 'connection_pool'
|
60
|
+
ConnectionPool
|
61
|
+
end
|
62
|
+
end
|
38
63
|
end
|
@@ -9,12 +9,10 @@ describe WorkerRoulette do
|
|
9
9
|
let(:work_orders_with_headers) {default_headers.merge({payload: work_orders})}
|
10
10
|
let(:jsonized_work_orders_with_headers) {[Oj.dump(work_orders_with_headers)]}
|
11
11
|
|
12
|
-
let(:redis) {Redis.new}
|
13
|
-
let(:pool_size) {10}
|
12
|
+
let(:redis) {Redis.new(WorkerRoulette.redis_config)}
|
14
13
|
|
15
14
|
before do
|
16
|
-
WorkerRoulette.start
|
17
|
-
Redis.new.flushdb
|
15
|
+
WorkerRoulette.start
|
18
16
|
end
|
19
17
|
|
20
18
|
it "should exist" do
|
@@ -161,13 +159,13 @@ describe WorkerRoulette do
|
|
161
159
|
Thread.new {WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}
|
162
160
|
end.each(&:join)
|
163
161
|
WorkerRoulette.tradesman_connection_pool.with do |pooled_redis|
|
164
|
-
pooled_redis.info["connected_clients"].to_i.should > (pool_size)
|
162
|
+
pooled_redis.info["connected_clients"].to_i.should > (WorkerRoulette.pool_size)
|
165
163
|
end
|
166
164
|
end
|
167
165
|
|
168
166
|
#This may be fixed soon (10 Feb 2014 - https://github.com/redis/redis-rb/pull/389 and https://github.com/redis/redis-rb/issues/364)
|
169
167
|
it "should not be fork() proof -- forking reconnects need to be handled in the calling code (until redis gem is udpated, then we should be fork-proof)" do
|
170
|
-
WorkerRoulette.start
|
168
|
+
WorkerRoulette.start
|
171
169
|
WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}
|
172
170
|
fork do
|
173
171
|
expect {WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}.to raise_error(Redis::InheritedError)
|
@@ -175,7 +173,12 @@ describe WorkerRoulette do
|
|
175
173
|
end
|
176
174
|
|
177
175
|
it "should use optionally non-blocking I/O" do
|
178
|
-
|
176
|
+
EM.synchrony do
|
177
|
+
WorkerRoulette.start(:driver => :synchrony)
|
178
|
+
WorkerRoulette.foreman("muddle_man").enqueue_work_order("foo")
|
179
|
+
WorkerRoulette.tradesman.work_orders!.should == [work_orders_with_headers]
|
180
|
+
EM.stop
|
181
|
+
end
|
179
182
|
end
|
180
183
|
end
|
181
184
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,22 +1,48 @@
|
|
1
1
|
require 'worker_roulette'
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
2
|
+
require 'em-synchrony'
|
3
|
+
require 'simplecov'
|
4
|
+
require 'simplecov-rcov'
|
5
|
+
class SimpleCov::Formatter::MergedFormatter
|
6
|
+
def format(result)
|
7
|
+
SimpleCov::Formatter::HTMLFormatter.new.format(result)
|
8
|
+
SimpleCov::Formatter::RcovFormatter.new.format(result)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
|
12
|
+
SimpleCov.start
|
12
13
|
|
13
14
|
require File.expand_path(File.join("..", "..", "lib", "worker_roulette.rb"), __FILE__)
|
14
15
|
include WorkerRoulette
|
15
16
|
|
16
17
|
Dir[File.join(File.dirname(__FILE__), 'helpers', '**/*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
17
18
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
module RSpec
|
20
|
+
module Core
|
21
|
+
class ExampleGroup
|
22
|
+
|
23
|
+
class << self
|
24
|
+
alias_method :run_alias, :run
|
25
|
+
|
26
|
+
def run(reporter)
|
27
|
+
if EM.reactor_running?
|
28
|
+
run_alias reporter
|
29
|
+
else
|
30
|
+
out = nil
|
31
|
+
EM.synchrony do
|
32
|
+
out = run_alias reporter
|
33
|
+
EM.stop
|
34
|
+
end
|
35
|
+
out
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
RSpec.configure do |c|
|
45
|
+
c.after(:each) do
|
46
|
+
Redis.new(WorkerRoulette.redis_config).flushdb
|
47
|
+
end
|
48
|
+
end
|
data/worker_roulette.gemspec
CHANGED
@@ -17,8 +17,8 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.test_files = spec.files.grep(%r{^(spec)/})
|
18
18
|
spec.require_paths = ["lib"]
|
19
19
|
|
20
|
-
spec.add_dependency 'redis'
|
21
20
|
spec.add_dependency 'oj'
|
21
|
+
spec.add_dependency 'redis', '~> 3.0.7'
|
22
22
|
spec.add_dependency 'hiredis', '~> 0.4.5'
|
23
23
|
spec.add_dependency 'em-synchrony'
|
24
24
|
spec.add_dependency 'connection_pool'
|
@@ -27,10 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.add_development_dependency "rake"
|
28
28
|
spec.add_development_dependency 'rspec'
|
29
29
|
spec.add_development_dependency 'pry-debugger'
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
# spec.add_development_dependency 'simplecov-rcov'
|
34
|
-
# spec.add_development_dependency 'rspec_junit_formatter'
|
35
|
-
# spec.add_development_dependency 'sidekiq'
|
30
|
+
spec.add_development_dependency 'simplecov'
|
31
|
+
spec.add_development_dependency 'simplecov-rcov'
|
32
|
+
spec.add_development_dependency 'rspec_junit_formatter'
|
36
33
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: worker_roulette
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -12,7 +12,7 @@ cert_chain: []
|
|
12
12
|
date: 2014-02-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
15
|
+
name: oj
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
@@ -28,21 +28,21 @@ dependencies:
|
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
|
-
name:
|
31
|
+
name: redis
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
|
-
- -
|
35
|
+
- - ~>
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version:
|
37
|
+
version: 3.0.7
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
|
-
- -
|
43
|
+
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version:
|
45
|
+
version: 3.0.7
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: hiredis
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -155,6 +155,54 @@ dependencies:
|
|
155
155
|
- - ! '>='
|
156
156
|
- !ruby/object:Gem::Version
|
157
157
|
version: '0'
|
158
|
+
- !ruby/object:Gem::Dependency
|
159
|
+
name: simplecov
|
160
|
+
requirement: !ruby/object:Gem::Requirement
|
161
|
+
none: false
|
162
|
+
requirements:
|
163
|
+
- - ! '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
type: :development
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
none: false
|
170
|
+
requirements:
|
171
|
+
- - ! '>='
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
- !ruby/object:Gem::Dependency
|
175
|
+
name: simplecov-rcov
|
176
|
+
requirement: !ruby/object:Gem::Requirement
|
177
|
+
none: false
|
178
|
+
requirements:
|
179
|
+
- - ! '>='
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0'
|
182
|
+
type: :development
|
183
|
+
prerelease: false
|
184
|
+
version_requirements: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
- !ruby/object:Gem::Dependency
|
191
|
+
name: rspec_junit_formatter
|
192
|
+
requirement: !ruby/object:Gem::Requirement
|
193
|
+
none: false
|
194
|
+
requirements:
|
195
|
+
- - ! '>='
|
196
|
+
- !ruby/object:Gem::Version
|
197
|
+
version: '0'
|
198
|
+
type: :development
|
199
|
+
prerelease: false
|
200
|
+
version_requirements: !ruby/object:Gem::Requirement
|
201
|
+
none: false
|
202
|
+
requirements:
|
203
|
+
- - ! '>='
|
204
|
+
- !ruby/object:Gem::Version
|
205
|
+
version: '0'
|
158
206
|
description: Write a gem description
|
159
207
|
email:
|
160
208
|
- classicist@gmail.com
|