worker_roulette 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/Gemfile +4 -0
- data/Guardfile +3 -0
- data/LICENSE.txt +0 -0
- data/README.md +70 -0
- data/Rakefile +25 -0
- data/lib/worker_roulette/foreman.rb +44 -0
- data/lib/worker_roulette/tradesman.rb +39 -0
- data/lib/worker_roulette/version.rb +3 -0
- data/lib/worker_roulette.rb +38 -0
- data/spec/benchmark/perf_test.rb +66 -0
- data/spec/helpers/.gitkeep +1 -0
- data/spec/integration/worker_roulette_spec.rb +182 -0
- data/spec/spec_helper.rb +22 -0
- data/worker_roulette.gemspec +36 -0
- metadata +210 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/Guardfile
ADDED
data/LICENSE.txt
ADDED
File without changes
|
data/README.md
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# WorkerRoulette
|
2
|
+
|
3
|
+
WorkerRoulette is designed to allow large numbers of unique devices, processes, users, or whatever communicate over individual channels without messing up the order of their messages. WorkerRoulette was created to solve two otherwise hard problems. First, other messaging solutions (I'm looking at you RabbitMQ) are not designed to handle very large numbers of queues (30,000+); because WorkerRoulette is built on top of Redis, we have successfully tested it running with millions of queues. Second, other messaging systems assume one (or more) of three things: 1. Your message consumers know the routing key of messages they are interested in processing; 2. Your messages can wait so that only one consumer is processed at a time; 3. You love to complicated write code to put your messages back in order. Sometimes, none of these things is true and that is where WorkerRoulette comes in.
|
4
|
+
|
5
|
+
WorkerRoulette lets you have thousands of competing consumers (distrubted over as many machines as you'd like) processing ordered messages from millions of totally unknown message providers. It does all this and ensures that the messages sent from each message provider are processed in exactly the order it sent them.
|
6
|
+
|
7
|
+
## Usage
|
8
|
+
size_of_connection_pool = 100
|
9
|
+
redis_config = {host: 'localhost', timeout: 5, db: 1}
|
10
|
+
|
11
|
+
WorkerRoulette.start(size_of_connection_pool, redis_config)
|
12
|
+
|
13
|
+
sender_id = :shady
|
14
|
+
foreman = WorkerRoulette.foreman(sender_id)
|
15
|
+
foreman.enqueue_work_order(['hello', 'foreman'])
|
16
|
+
|
17
|
+
tradesman = WorkerRoulette.tradesman
|
18
|
+
messages = tradesman.work_orders! #drain the queue of the next available sender
|
19
|
+
messages.first # => ['hello', 'foreman']
|
20
|
+
|
21
|
+
other_sender_id = :the_real_slim_shady
|
22
|
+
other_foreman = WorkerRoulette.foreman(other_sender_id)
|
23
|
+
other_foreman.enqueue_work_order({'can you get me' => 'the number nine?'})
|
24
|
+
|
25
|
+
messages = tradesman.work_orders! #drain the queue of the next available sender
|
26
|
+
messages.first # => {'can you get me' => 'the number nine?'}
|
27
|
+
|
28
|
+
on_subscribe_callback = -> do
|
29
|
+
puts "Huzzah! We're listening!"
|
30
|
+
foreman.enqueue_work_order('will I see you later?')
|
31
|
+
foreman.enqueue_work_order('can you give me back my dime?')
|
32
|
+
end
|
33
|
+
|
34
|
+
tradesman.wait_for_work_orders(on_subscribe_callback) do |messages| #drain the queue of the next available sender
|
35
|
+
messages # => ['will I see you later', 'can you give me back my dime?']
|
36
|
+
end
|
37
|
+
|
38
|
+
##Caveat Emptor
|
39
|
+
While WorkerRoulette does promise to keep the messages of each consumer processed in order by competing consumers, it does NOT guarantee the order in which the queues themselves will be processed. In general, work is processed in a FIFO order, but for performance reasons this has been left a loose FIFO. For example, if Abdul enqueue_work_orders some ordered messages ('1', '2', and '3') and then so do Mark and Wanda, Mark's messages may be processed first, then it would likely be Abdul's, and then Wanda's. However, even though Mark jumped the line, Abdul's messages will still be processed the order he enqueue_work_orderd them ('1', '2', then '3').
|
40
|
+
|
41
|
+
## Installation
|
42
|
+
|
43
|
+
Add this line to your application's Gemfile:
|
44
|
+
|
45
|
+
gem 'worker_roulette'
|
46
|
+
|
47
|
+
And then execute:
|
48
|
+
|
49
|
+
$ bundle
|
50
|
+
|
51
|
+
Or install it yourself as:
|
52
|
+
|
53
|
+
$ gem install worker_roulette
|
54
|
+
|
55
|
+
## Run the specs
|
56
|
+
|
57
|
+
$ bundle exec rake spec:ci
|
58
|
+
|
59
|
+
## Run the performance tests
|
60
|
+
|
61
|
+
$ bundle exec rake spec:perf
|
62
|
+
|
63
|
+
|
64
|
+
## Contributing
|
65
|
+
|
66
|
+
1. Fork it
|
67
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
68
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
69
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
70
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require "rspec/core/rake_task"
|
4
|
+
|
5
|
+
RSpec::Core::RakeTask.new
|
6
|
+
|
7
|
+
task :default => :spec
|
8
|
+
task :test => :spec
|
9
|
+
|
10
|
+
# def rspec_out_file
|
11
|
+
# require 'rspec_junit_formatter'
|
12
|
+
# "-f RspecJunitFormatter -o results.xml"
|
13
|
+
# end
|
14
|
+
|
15
|
+
desc "Run all unit and integration tests"
|
16
|
+
task :'spec:ci' do
|
17
|
+
rspec_out_file = nil
|
18
|
+
sh "bundle exec rspec #{rspec_out_file} spec"
|
19
|
+
end
|
20
|
+
|
21
|
+
desc "Run perf tests"
|
22
|
+
task :'spec:perf' do
|
23
|
+
rspec_out_file = nil
|
24
|
+
sh "bundle exec ruby ./spec/benchmark/perf_test.rb"
|
25
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module WorkerRoulette
|
2
|
+
class Foreman
|
3
|
+
attr_reader :sender
|
4
|
+
COUNTER_KEY = 'counter_key'
|
5
|
+
|
6
|
+
def initialize(sender, redis_pool)
|
7
|
+
@sender = sender
|
8
|
+
@redis_pool = redis_pool
|
9
|
+
end
|
10
|
+
|
11
|
+
def job_board_key
|
12
|
+
WorkerRoulette::JOB_BOARD
|
13
|
+
end
|
14
|
+
|
15
|
+
def counter_key
|
16
|
+
COUNTER_KEY
|
17
|
+
end
|
18
|
+
|
19
|
+
def enqueue_work_order_without_headers(work_order)
|
20
|
+
#Caveat Emptor: There is a race condition here, but it not serious;
|
21
|
+
#the count may be incremented again by another process before the sender
|
22
|
+
#is added to the job_queue. This is not a big deal bc it just means that
|
23
|
+
#the sender's queue will be processed one slot behind it's rightful place.
|
24
|
+
#This does not effect work_order ordering.
|
25
|
+
@redis_pool.with do |redis|
|
26
|
+
@count = redis.incr(COUNTER_KEY)
|
27
|
+
redis.multi do
|
28
|
+
redis.zadd(WorkerRoulette::JOB_BOARD, @count, sender)
|
29
|
+
redis.rpush(sender, Oj.dump(work_order))
|
30
|
+
redis.publish(WorkerRoulette::JOB_NOTIFICATIONS, WorkerRoulette::JOB_NOTIFICATIONS)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def enqueue_work_order(work_order, headers = {})
|
36
|
+
work_order = {headers: default_headers.merge(headers), payload: work_order}
|
37
|
+
enqueue_work_order_without_headers(work_order)
|
38
|
+
end
|
39
|
+
|
40
|
+
def default_headers
|
41
|
+
Hash[sender: sender]
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module WorkerRoulette
|
2
|
+
class Tradesman
|
3
|
+
attr_reader :sender
|
4
|
+
def initialize(client_pool, pubsub_pool)
|
5
|
+
@client_pool = client_pool
|
6
|
+
@pubsub_pool = pubsub_pool
|
7
|
+
end
|
8
|
+
|
9
|
+
def job_board_key
|
10
|
+
WorkerRoulette::JOB_BOARD
|
11
|
+
end
|
12
|
+
|
13
|
+
def wait_for_work_orders(on_subscribe_callback = nil, &block)
|
14
|
+
@pubsub_pool.with do |redis|
|
15
|
+
redis.subscribe(WorkerRoulette::JOB_NOTIFICATIONS) do |on|
|
16
|
+
on.subscribe {on_subscribe_callback.call if on_subscribe_callback}
|
17
|
+
on.message {redis.unsubscribe; block.call(work_orders!) if block}
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def work_orders!
|
23
|
+
@client_pool.with do |redis|
|
24
|
+
get_sender_for_next_job(redis)
|
25
|
+
results = redis.multi do
|
26
|
+
redis.lrange(sender, 0, -1)
|
27
|
+
redis.del(sender)
|
28
|
+
redis.zrem(WorkerRoulette::JOB_BOARD, sender)
|
29
|
+
end
|
30
|
+
((results || []).first || []).map {|work_order| Oj.load(work_order)}
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
private
|
35
|
+
def get_sender_for_next_job(redis)
|
36
|
+
@sender = (redis.zrange(WorkerRoulette::JOB_BOARD, 0, 0) || []).first.to_s
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require "worker_roulette/version"
|
2
|
+
require 'redis'
|
3
|
+
require 'hiredis'
|
4
|
+
require 'redis/connection/hiredis'
|
5
|
+
require 'connection_pool'
|
6
|
+
require 'oj'
|
7
|
+
|
8
|
+
Dir[File.join(File.dirname(__FILE__),'worker_roulette','**','*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
9
|
+
|
10
|
+
module WorkerRoulette
|
11
|
+
JOB_BOARD = "job_board"
|
12
|
+
JOB_NOTIFICATIONS = "new_job_ready"
|
13
|
+
|
14
|
+
def self.start(pool_size = 10, config = {})
|
15
|
+
config = {host: 'localhost', db: 15, timeout: 5}.merge(config)
|
16
|
+
@foreman_connection_pool = ConnectionPool.new(config.merge Hash[size: pool_size, timeout: config[:timeout]]) {Redis.new}
|
17
|
+
@tradesman_connection_pool = ConnectionPool.new(config.merge Hash[size: pool_size, timeout: config[:timeout]]) {Redis.new}
|
18
|
+
@pubsub_connection_pool = ConnectionPool.new(config.merge Hash[size: pool_size, timeout: config[:timeout]]) {Redis.new}
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.foreman(sender)
|
22
|
+
raise "WorkerRoulette not Started" unless @foreman_connection_pool
|
23
|
+
Foreman.new(sender, @foreman_connection_pool)
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.tradesman
|
27
|
+
raise "WorkerRoulette not Started" unless @tradesman_connection_pool
|
28
|
+
Tradesman.new(@tradesman_connection_pool, @pubsub_connection_pool)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.tradesman_connection_pool
|
32
|
+
@tradesman_connection_pool
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.pubsub_connection_pool
|
36
|
+
@pubsub_connection_pool
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require_relative '../spec_helper'
|
2
|
+
require 'benchmark'
|
3
|
+
require 'eventmachine'
|
4
|
+
|
5
|
+
REDIS_CONNECTION_POOL_SIZE = 100
|
6
|
+
ITERATIONS = 10_000
|
7
|
+
|
8
|
+
work_order = {'ding dong' => "hello_foreman_" * 100}
|
9
|
+
|
10
|
+
WorkerRoulette.start(REDIS_CONNECTION_POOL_SIZE)#{driver: :synchrony}
|
11
|
+
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
12
|
+
|
13
|
+
puts "Redis Connection Pool Size: #{REDIS_CONNECTION_POOL_SIZE}"
|
14
|
+
|
15
|
+
Benchmark.bmbm do |x|
|
16
|
+
x.report "Time to insert and read #{ITERATIONS} large work_orders" do # ~2500 work_orders / second round trip; 50-50 read-write time; CPU and IO bound
|
17
|
+
ITERATIONS.times do |iteration|
|
18
|
+
sender = 'sender_' + iteration.to_s
|
19
|
+
foreman = WorkerRoulette.foreman(sender)
|
20
|
+
foreman.enqueue_work_order(work_order)
|
21
|
+
end
|
22
|
+
|
23
|
+
ITERATIONS.times do |iteration|
|
24
|
+
sender = 'sender_' + iteration.to_s
|
25
|
+
tradesman = WorkerRoulette.tradesman
|
26
|
+
tradesman.work_orders!
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
32
|
+
|
33
|
+
Benchmark.bmbm do |x|
|
34
|
+
x.report "Time for tradesmans to enqueue_work_order and read #{ITERATIONS} large work_orders via pubsub" do # ~1800 work_orders / second round trip
|
35
|
+
ITERATIONS.times do |iteration|
|
36
|
+
p = -> do
|
37
|
+
sender = 'sender_' + iteration.to_s
|
38
|
+
foreman = WorkerRoulette.foreman(sender)
|
39
|
+
foreman.enqueue_work_order(work_order)
|
40
|
+
end
|
41
|
+
tradesman = WorkerRoulette.tradesman
|
42
|
+
tradesman.wait_for_work_orders(p) {|m| m}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
48
|
+
|
49
|
+
# EM.run do
|
50
|
+
# EM.add_timer(6) {puts "em off";EM.stop}
|
51
|
+
# tradesmans = []
|
52
|
+
# foremans = []
|
53
|
+
# @start = Time.now
|
54
|
+
# @end = nil
|
55
|
+
# ITERATIONS.times do |iteration|
|
56
|
+
# s = WorkerRoulette.tradesman
|
57
|
+
# tradesmans << s
|
58
|
+
# sender = 'sender_' + iteration.to_s
|
59
|
+
# foreman = WorkerRoulette.foreman(sender)
|
60
|
+
# a = -> {foreman.enqueue_work_order(work_order)}
|
61
|
+
# s.wait_for_work_orders(a) {|m| @end = Time.now if iteration == (ITERATIONS - 1) }
|
62
|
+
# end
|
63
|
+
# end
|
64
|
+
|
65
|
+
# puts @end - @start
|
66
|
+
# WorkerRoulette.tradesman_connection_pool.with {|r| r.flushdb}
|
@@ -0,0 +1 @@
|
|
1
|
+
.gitkeep
|
@@ -0,0 +1,182 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe WorkerRoulette do
|
4
|
+
let(:sender) {'katie_80'}
|
5
|
+
let(:work_orders) {["hello", "foreman"]}
|
6
|
+
let(:default_headers) {Hash[headers: {sender: sender}]}
|
7
|
+
let(:hello_work_order) {Hash[payload: "hello"]}
|
8
|
+
let(:foreman_work_order) {Hash[payload: "foreman"]}
|
9
|
+
let(:work_orders_with_headers) {default_headers.merge({payload: work_orders})}
|
10
|
+
let(:jsonized_work_orders_with_headers) {[Oj.dump(work_orders_with_headers)]}
|
11
|
+
|
12
|
+
let(:redis) {Redis.new}
|
13
|
+
let(:pool_size) {10}
|
14
|
+
|
15
|
+
before do
|
16
|
+
WorkerRoulette.start(pool_size)
|
17
|
+
Redis.new.flushdb
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should exist" do
|
21
|
+
WorkerRoulette.should_not be_nil
|
22
|
+
end
|
23
|
+
|
24
|
+
context Foreman do
|
25
|
+
let(:subject) {WorkerRoulette.foreman(sender)}
|
26
|
+
|
27
|
+
it "should be working on behalf of a sender" do
|
28
|
+
subject.sender.should == sender
|
29
|
+
end
|
30
|
+
|
31
|
+
it "should be injected with a raw_redis_client so it can do is work" do
|
32
|
+
Redis.any_instance.should_receive(:rpush)
|
33
|
+
subject.enqueue_work_order(:whatever)
|
34
|
+
end
|
35
|
+
|
36
|
+
it "should enqueue_work_order two work_orders in the sender's slot in the switchboard" do
|
37
|
+
subject.enqueue_work_order(work_orders.first)
|
38
|
+
subject.enqueue_work_order(work_orders.last)
|
39
|
+
redis.lrange(sender, 0, -1).should == work_orders.map {|m| Oj.dump(default_headers.merge({payload: m})) }
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should enqueue_work_order an array of work_orders without headers in the sender's slot in the switchboard" do
|
43
|
+
subject.enqueue_work_order_without_headers(work_orders)
|
44
|
+
redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders)]
|
45
|
+
end
|
46
|
+
|
47
|
+
it "should enqueue_work_order an array of work_orders with default headers in the sender's slot in the switchboard" do
|
48
|
+
subject.enqueue_work_order(work_orders)
|
49
|
+
redis.lrange(sender, 0, -1).should == jsonized_work_orders_with_headers
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should enqueue_work_order an array of work_orders with additional headers in the sender's slot in the switchboard" do
|
53
|
+
extra_headers = {foo: :bars}
|
54
|
+
subject.enqueue_work_order(work_orders, extra_headers)
|
55
|
+
work_orders_with_headers[:headers].merge!(extra_headers)
|
56
|
+
redis.lrange(sender, 0, -1).should == [Oj.dump(work_orders_with_headers)]
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should post the sender's id to the job board with an order number" do
|
60
|
+
subject.enqueue_work_order(work_orders.first)
|
61
|
+
subject.enqueue_work_order(work_orders.last)
|
62
|
+
redis.zrange(subject.job_board_key, 0, -1, with_scores: true).should == [[sender.to_s, work_orders.length.to_f]]
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should post the sender_id and work_orders transactionally" do
|
66
|
+
Redis.any_instance.should_receive(:multi)
|
67
|
+
subject.enqueue_work_order(work_orders.first)
|
68
|
+
end
|
69
|
+
|
70
|
+
it "should generate sequential order numbers" do
|
71
|
+
redis.get(subject.counter_key).should == nil
|
72
|
+
subject.enqueue_work_order(work_orders.first)
|
73
|
+
redis.get(subject.counter_key).should == "1"
|
74
|
+
subject.enqueue_work_order(work_orders.last)
|
75
|
+
redis.get(subject.counter_key).should == "2"
|
76
|
+
end
|
77
|
+
|
78
|
+
it "should publish a notification that a new job is ready" do
|
79
|
+
result = nil
|
80
|
+
redis_tradesman = Redis.new
|
81
|
+
redis_tradesman.subscribe(WorkerRoulette::JOB_NOTIFICATIONS) do |on|
|
82
|
+
on.subscribe do |channel, subscription|
|
83
|
+
subject.enqueue_work_order(work_orders)
|
84
|
+
end
|
85
|
+
|
86
|
+
on.message do |channel, notification|
|
87
|
+
result = notification
|
88
|
+
redis_tradesman.unsubscribe(WorkerRoulette::JOB_NOTIFICATIONS)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
result.should == WorkerRoulette::JOB_NOTIFICATIONS
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
context Tradesman do
|
97
|
+
let(:foreman) {WorkerRoulette.foreman(sender)}
|
98
|
+
let(:subject) {WorkerRoulette.tradesman}
|
99
|
+
|
100
|
+
before do
|
101
|
+
foreman.enqueue_work_order(work_orders)
|
102
|
+
end
|
103
|
+
|
104
|
+
it "should be working on behalf of a sender" do
|
105
|
+
subject.work_orders!
|
106
|
+
subject.sender.should == sender
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should be injected with a redis client so it can do its work" do
|
110
|
+
Redis.any_instance.should_receive(:lrange).and_call_original
|
111
|
+
subject.work_orders!
|
112
|
+
end
|
113
|
+
|
114
|
+
it "should drain one set of work_orders from the sender's slot in the switchboard" do
|
115
|
+
subject.work_orders!.should == [work_orders_with_headers]
|
116
|
+
subject.work_orders!.should == []
|
117
|
+
subject.work_orders!.should == [] #does not throw an error if queue is alreay empty
|
118
|
+
end
|
119
|
+
|
120
|
+
it "should drain all the work_orders from the sender's slot in the switchboard" do
|
121
|
+
foreman.enqueue_work_order(work_orders)
|
122
|
+
subject.work_orders!.should == [work_orders_with_headers, work_orders_with_headers]
|
123
|
+
subject.work_orders!.should == []
|
124
|
+
subject.work_orders!.should == [] #does not throw an error if queue is alreay empty
|
125
|
+
end
|
126
|
+
|
127
|
+
it "should take the oldest sender off the job board (FIFO)" do
|
128
|
+
oldest_sender = sender.to_s
|
129
|
+
most_recent_sender = 'most_recent_sender'
|
130
|
+
most_recent_foreman = WorkerRoulette.foreman(most_recent_sender)
|
131
|
+
most_recent_foreman.enqueue_work_order(work_orders)
|
132
|
+
redis.zrange(subject.job_board_key, 0, -1).should == [oldest_sender, most_recent_sender]
|
133
|
+
subject.work_orders!
|
134
|
+
redis.zrange(subject.job_board_key, 0, -1).should == [most_recent_sender]
|
135
|
+
end
|
136
|
+
|
137
|
+
it "should get the sender and work_order list transactionally" do
|
138
|
+
Redis.any_instance.should_receive(:multi).and_call_original
|
139
|
+
subject.work_orders!
|
140
|
+
end
|
141
|
+
|
142
|
+
it "should get the work_orders from the next sender's slot when a new job is ready" do
|
143
|
+
subject.work_orders!
|
144
|
+
subject.should_receive(:work_orders!).and_call_original
|
145
|
+
publisher = -> {foreman.enqueue_work_order(work_orders)}
|
146
|
+
subject.wait_for_work_orders(publisher) do |redis_work_orders|
|
147
|
+
redis_work_orders.should == [work_orders_with_headers]
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
it "should checkout a readlock for a queue and put it back when its done processing; lock should expire after 5 minutes?"
|
152
|
+
it "should eves drop on the job board"
|
153
|
+
|
154
|
+
context "Failure" do
|
155
|
+
it "should not put the sender_id and work_orders back if processing fails bc new work_orders may have been processed while that process failed" do; end
|
156
|
+
end
|
157
|
+
|
158
|
+
context "Concurrent Access" do
|
159
|
+
it "should pool its connections" do
|
160
|
+
Array.new(100) do
|
161
|
+
Thread.new {WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}
|
162
|
+
end.each(&:join)
|
163
|
+
WorkerRoulette.tradesman_connection_pool.with do |pooled_redis|
|
164
|
+
pooled_redis.info["connected_clients"].to_i.should > (pool_size)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
#This may be fixed soon (10 Feb 2014 - https://github.com/redis/redis-rb/pull/389 and https://github.com/redis/redis-rb/issues/364)
|
169
|
+
it "should not be fork() proof -- forking reconnects need to be handled in the calling code (until redis gem is udpated, then we should be fork-proof)" do
|
170
|
+
WorkerRoulette.start(1)
|
171
|
+
WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}
|
172
|
+
fork do
|
173
|
+
expect {WorkerRoulette.tradesman_connection_pool.with {|pooled_redis| pooled_redis.get("foo")}}.to raise_error(Redis::InheritedError)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
it "should use optionally non-blocking I/O" do
|
178
|
+
expect {WorkerRoulette.start(1, :driver => :synchrony)}.not_to raise_error
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'worker_roulette'
|
2
|
+
# require 'simplecov'
|
3
|
+
# require 'simplecov-rcov'
|
4
|
+
# class SimpleCov::Formatter::MergedFormatter
|
5
|
+
# def format(result)
|
6
|
+
# SimpleCov::Formatter::HTMLFormatter.new.format(result)
|
7
|
+
# SimpleCov::Formatter::RcovFormatter.new.format(result)
|
8
|
+
# end
|
9
|
+
# end
|
10
|
+
# SimpleCov.formatter = SimpleCov::Formatter::MergedFormatter
|
11
|
+
# SimpleCov.start
|
12
|
+
|
13
|
+
require File.expand_path(File.join("..", "..", "lib", "worker_roulette.rb"), __FILE__)
|
14
|
+
include WorkerRoulette
|
15
|
+
|
16
|
+
Dir[File.join(File.dirname(__FILE__), 'helpers', '**/*.rb')].sort.each { |file| require file.gsub(".rb", "")}
|
17
|
+
|
18
|
+
# RSpec.configure do |c|
|
19
|
+
# after(:each) do
|
20
|
+
# Redis.new.flushall
|
21
|
+
# end
|
22
|
+
# end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'worker_roulette/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "worker_roulette"
|
8
|
+
spec.version = WorkerRoulette::VERSION
|
9
|
+
spec.authors = ["Paul Saieg"]
|
10
|
+
spec.email = ["classicist@gmail.com"]
|
11
|
+
spec.description = %q{Write a gem description}
|
12
|
+
spec.summary = %q{Write a gem summary}
|
13
|
+
spec.homepage = ""
|
14
|
+
|
15
|
+
spec.files = `git ls-files`.split($/)
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(spec)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_dependency 'redis'
|
21
|
+
spec.add_dependency 'oj'
|
22
|
+
spec.add_dependency 'hiredis', '~> 0.4.5'
|
23
|
+
spec.add_dependency 'em-synchrony'
|
24
|
+
spec.add_dependency 'connection_pool'
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler"
|
27
|
+
spec.add_development_dependency "rake"
|
28
|
+
spec.add_development_dependency 'rspec'
|
29
|
+
spec.add_development_dependency 'pry-debugger'
|
30
|
+
# spec.add_development_dependency 'guard'
|
31
|
+
# spec.add_development_dependency 'guard-rspec'
|
32
|
+
# spec.add_development_dependency 'simplecov'
|
33
|
+
# spec.add_development_dependency 'simplecov-rcov'
|
34
|
+
# spec.add_development_dependency 'rspec_junit_formatter'
|
35
|
+
# spec.add_development_dependency 'sidekiq'
|
36
|
+
end
|
metadata
ADDED
@@ -0,0 +1,210 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: worker_roulette
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Paul Saieg
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2014-02-12 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: redis
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: oj
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: hiredis
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 0.4.5
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.4.5
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: em-synchrony
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: connection_pool
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: bundler
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: rake
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: rspec
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: pry-debugger
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
description: Write a gem description
|
159
|
+
email:
|
160
|
+
- classicist@gmail.com
|
161
|
+
executables: []
|
162
|
+
extensions: []
|
163
|
+
extra_rdoc_files: []
|
164
|
+
files:
|
165
|
+
- .gitignore
|
166
|
+
- .rspec
|
167
|
+
- Gemfile
|
168
|
+
- Guardfile
|
169
|
+
- LICENSE.txt
|
170
|
+
- README.md
|
171
|
+
- Rakefile
|
172
|
+
- lib/worker_roulette.rb
|
173
|
+
- lib/worker_roulette/foreman.rb
|
174
|
+
- lib/worker_roulette/tradesman.rb
|
175
|
+
- lib/worker_roulette/version.rb
|
176
|
+
- spec/benchmark/perf_test.rb
|
177
|
+
- spec/helpers/.gitkeep
|
178
|
+
- spec/integration/worker_roulette_spec.rb
|
179
|
+
- spec/spec_helper.rb
|
180
|
+
- worker_roulette.gemspec
|
181
|
+
homepage: ''
|
182
|
+
licenses: []
|
183
|
+
post_install_message:
|
184
|
+
rdoc_options: []
|
185
|
+
require_paths:
|
186
|
+
- lib
|
187
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
188
|
+
none: false
|
189
|
+
requirements:
|
190
|
+
- - ! '>='
|
191
|
+
- !ruby/object:Gem::Version
|
192
|
+
version: '0'
|
193
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
194
|
+
none: false
|
195
|
+
requirements:
|
196
|
+
- - ! '>='
|
197
|
+
- !ruby/object:Gem::Version
|
198
|
+
version: '0'
|
199
|
+
requirements: []
|
200
|
+
rubyforge_project:
|
201
|
+
rubygems_version: 1.8.25
|
202
|
+
signing_key:
|
203
|
+
specification_version: 3
|
204
|
+
summary: Write a gem summary
|
205
|
+
test_files:
|
206
|
+
- spec/benchmark/perf_test.rb
|
207
|
+
- spec/helpers/.gitkeep
|
208
|
+
- spec/integration/worker_roulette_spec.rb
|
209
|
+
- spec/spec_helper.rb
|
210
|
+
has_rdoc:
|