sbm 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +55 -0
- data/Rakefile +9 -0
- data/bin/sbm +3 -0
- data/lib/sbm.rb +6 -0
- data/lib/sbm/coordinator.rb +87 -0
- data/lib/sbm/runner.rb +92 -0
- data/lib/sbm/version.rb +3 -0
- data/sbm.gemspec +27 -0
- data/spec/coordinator_spec.rb +120 -0
- data/spec/runner_spec.rb +163 -0
- data/spec/spec_helper.rb +9 -0
- metadata +134 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5e18a93f50941dd24fa6348a0ca45ed4c7648dad
|
4
|
+
data.tar.gz: 2653657223ff6fc5ae1862839e8906e233f8fc14
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 21485914dab3bed5742d63d813c8dbf2b8de47dce842bb3c562426f8bf0e5a6956c586884aba8dcc33f93c06552d4302d11d24a8ddb51a8efbc9994d5c2840d2
|
7
|
+
data.tar.gz: c24c64c4c4a463572d23154f1c1b6ffb9613cf2100af5576312ee843a1384e5932397ffe46d95ae5b0c1e2001e63e374041f4a8427f1f047407fac8f0bf14455
|
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--colour
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Darcy Laycock
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# SBM - Simple Batch Manager
|
2
|
+
|
3
|
+
Manages running / coordinating batch processes running across multiple hosts.
|
4
|
+
|
5
|
+
Uses redis as a simple coordinator to ensure split work runs across hosts evenly.
|
6
|
+
|
7
|
+
**Note:** SBM is still a hack. It's untested. Don't use this for production stuff. Please!
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
gem 'sbm'
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install sbm
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
SBM is composed of a set of simple scripts useful for running in shells. The only required
|
26
|
+
variable is `SBM_WORKER` as an environment variable - the name of the node the current job is
|
27
|
+
running on.`
|
28
|
+
|
29
|
+
```bash
|
30
|
+
#!/usr/bin/env bash -e
|
31
|
+
|
32
|
+
export SBM_WORKER="$(hostname)-$$"
|
33
|
+
|
34
|
+
# Used if you have a bunch of different batches with the same name:
|
35
|
+
# export SBM_COORDINATOR='your-groups'
|
36
|
+
|
37
|
+
sbm start-work my-test-batch
|
38
|
+
rake do:your:work
|
39
|
+
sbm complete-work my-test-batch && sbm wait-for my-test-batch 20 # There are 20 nodes running this process
|
40
|
+
|
41
|
+
sbm status
|
42
|
+
|
43
|
+
```
|
44
|
+
|
45
|
+
Wait for simply checks the number of items in the completed set have the correct length.
|
46
|
+
|
47
|
+
Please note that by default it uses redis for this, so to change your default redis use `REDIS_URI`.
|
48
|
+
|
49
|
+
## Contributing
|
50
|
+
|
51
|
+
1. Fork it
|
52
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
53
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
54
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
55
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/bin/sbm
ADDED
data/lib/sbm.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
# Handles the base of coordinating works in sets between
|
2
|
+
# X nodes, with each item having a node identifier.
|
3
|
+
|
4
|
+
require 'redis'
|
5
|
+
|
6
|
+
module SBM
|
7
|
+
class Coordinator
|
8
|
+
|
9
|
+
def self.defaults
|
10
|
+
worker_name = (ENV['SBM_WORKER'] or raise "Please ensure SBM_WORKER is set")
|
11
|
+
coordinator_name = (ENV['SBM_COORDINATOR'] || "worker-coordinator")
|
12
|
+
return new(coordinator_name), Worker.new(worker_name)
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :name, :redis
|
16
|
+
|
17
|
+
def initialize(name)
|
18
|
+
@name = name.to_s
|
19
|
+
@redis = Redis.current
|
20
|
+
end
|
21
|
+
|
22
|
+
class Batch < Struct.new(:name)
|
23
|
+
|
24
|
+
def to_s; name; end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
class Worker < Struct.new(:name)
|
29
|
+
def to_s; name; end
|
30
|
+
end
|
31
|
+
|
32
|
+
def batches
|
33
|
+
redis.smembers(key(:batches)).map { |w| Batch.new(w) }
|
34
|
+
end
|
35
|
+
|
36
|
+
def workers
|
37
|
+
redis.smembers(key(:workers)).map { |w| Worker.new(w) }
|
38
|
+
end
|
39
|
+
|
40
|
+
def started_workers_for_batch(batch)
|
41
|
+
redis.smembers(key(:batches, batch, :started)).map { |w| Worker.new(w) }
|
42
|
+
end
|
43
|
+
|
44
|
+
def completed_workers_for_batch(batch)
|
45
|
+
redis.smembers(key(:batches, batch, :completed)).map { |w| Worker.new(w) }
|
46
|
+
end
|
47
|
+
|
48
|
+
def start(batch, worker)
|
49
|
+
prepare worker, batch
|
50
|
+
redis.sadd key(:batches, batch, :started), worker.to_s
|
51
|
+
redis.srem key(:batches, batch, :completed), worker.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
def complete(batch, worker)
|
55
|
+
prepare worker, batch
|
56
|
+
redis.sadd key(:batches, batch, :completed), worker.to_s
|
57
|
+
end
|
58
|
+
|
59
|
+
# Waits on batch to reach a count, waiting for 15 seconds at a time.
|
60
|
+
def wait_for(batch, worker_count, wait_time = 15)
|
61
|
+
while redis.scard(key(:batches, batch, :completed)) < worker_count
|
62
|
+
sleep wait_time
|
63
|
+
yield if block_given?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def prepare(worker, batch)
|
70
|
+
register_worker worker
|
71
|
+
register_batch batch
|
72
|
+
end
|
73
|
+
|
74
|
+
def register_worker(worker)
|
75
|
+
redis.sadd key(:workers), worker.to_s
|
76
|
+
end
|
77
|
+
|
78
|
+
def register_batch(batch)
|
79
|
+
redis.sadd key(:batches), batch.to_s
|
80
|
+
end
|
81
|
+
|
82
|
+
def key(*args)
|
83
|
+
[name, *args].join(":")
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
data/lib/sbm/runner.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
module SBM
|
2
|
+
class Runner
|
3
|
+
|
4
|
+
USAGES = {
|
5
|
+
'status' => '',
|
6
|
+
'wait-for' => 'batch-name worker-count',
|
7
|
+
'start-batch' => 'batch-name',
|
8
|
+
'complete-batch' => 'batch-name'
|
9
|
+
}
|
10
|
+
|
11
|
+
attr_reader :command, :args, :coordinator, :worker, :output, :error
|
12
|
+
|
13
|
+
def initialize(args, output = STDOUT, error = STDERR)
|
14
|
+
@command = args.first
|
15
|
+
@args = args.drop(1)
|
16
|
+
@output = output
|
17
|
+
@error = error
|
18
|
+
@coordinator, @worker = SBM::Coordinator.defaults
|
19
|
+
end
|
20
|
+
|
21
|
+
def validate_command!
|
22
|
+
if command.nil? or !USAGES.has_key?(command)
|
23
|
+
usage true
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def run
|
28
|
+
validate_command!
|
29
|
+
send command.tr('-', '_').to_sym
|
30
|
+
end
|
31
|
+
|
32
|
+
def status
|
33
|
+
output.puts "Known Workers: #{coordinator.workers.map(&:name).sort.join(", ")}"
|
34
|
+
output.puts "Known Batches: #{coordinator.batches.map(&:name).sort.join(", ")}"
|
35
|
+
output.puts ""
|
36
|
+
output.puts ""
|
37
|
+
coordinator.batches.each do |batch|
|
38
|
+
started = coordinator.started_workers_for_batch batch
|
39
|
+
completed = coordinator.started_workers_for_batch completed
|
40
|
+
output.puts "Batch: #{batch}"
|
41
|
+
output.puts "Number Started: #{started.size}"
|
42
|
+
output.puts "Number Completed: #{completed.size}"
|
43
|
+
output.puts "Number Pending: #{started.size - completed.size}"
|
44
|
+
output.puts "---"
|
45
|
+
output.puts "Started: #{started.map(&:name).sort.join(", ")}"
|
46
|
+
output.puts "Completed: #{completed.map(&:name).sort.join(", ")}"
|
47
|
+
output.puts ""
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def wait_for
|
52
|
+
batch = extract_batch!
|
53
|
+
worker_count = args.shift.to_i
|
54
|
+
if worker_count.zero?
|
55
|
+
error.puts "You must provide a non-zero worker count"
|
56
|
+
usage
|
57
|
+
end
|
58
|
+
coordinator.wait_for batch, worker_count
|
59
|
+
end
|
60
|
+
|
61
|
+
def start_batch
|
62
|
+
batch = extract_batch!
|
63
|
+
coordinator.start batch, worker
|
64
|
+
end
|
65
|
+
|
66
|
+
def complete_batch
|
67
|
+
batch = extract_batch!
|
68
|
+
coordinator.complete batch, worker
|
69
|
+
end
|
70
|
+
|
71
|
+
def usage(invalid_command = false)
|
72
|
+
if invalid_command
|
73
|
+
error.puts "Invalid / unknown command - must be one of #{USAGES.keys.join(", ")}"
|
74
|
+
error.puts "Usage: #$0 #{USAGES.keys.join("|")} [arguments]"
|
75
|
+
exit 1
|
76
|
+
else
|
77
|
+
error.puts "Usage: #$0 #{command} #{USAGES[command]}".strip
|
78
|
+
exit 1
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def extract_batch!
|
83
|
+
batch_name = args.shift
|
84
|
+
if batch_name.to_s.strip.empty?
|
85
|
+
error.puts "You must provide a batch name."
|
86
|
+
usage
|
87
|
+
end
|
88
|
+
Coordinator::Batch.new(batch_name)
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
end
|
data/lib/sbm/version.rb
ADDED
data/sbm.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'sbm/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "sbm"
|
8
|
+
spec.version = SBM::VERSION
|
9
|
+
spec.authors = ["Darcy Laycock"]
|
10
|
+
spec.email = ["sutto@sutto.net"]
|
11
|
+
spec.description = %q{Tools for managed simple batches across N nodes.}
|
12
|
+
spec.summary = %q{Built on redis, provides a basic set of tools that let you process tasks in parallel across N nodes.}
|
13
|
+
spec.homepage = "https://github.com/Sutto"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "redis"
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
24
|
+
spec.add_development_dependency "rake"
|
25
|
+
spec.add_development_dependency "rspec"
|
26
|
+
spec.add_development_dependency "rr"
|
27
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SBM::Coordinator do
|
4
|
+
|
5
|
+
let(:worker_a) { described_class::Worker.new 'elephant' }
|
6
|
+
let(:worker_b) { described_class::Worker.new 'pony' }
|
7
|
+
let(:worker_c) { described_class::Worker.new 'fish' }
|
8
|
+
|
9
|
+
let(:batch_a) { described_class::Batch.new 'fishing' }
|
10
|
+
let(:batch_b) { described_class::Batch.new 'running' }
|
11
|
+
let(:batch_c) { described_class::Batch.new 'rocking' }
|
12
|
+
|
13
|
+
subject { described_class.new 'my-awesome-item' }
|
14
|
+
|
15
|
+
it 'should register the batch and worker on starting' do
|
16
|
+
subject.batches.should be_empty
|
17
|
+
subject.workers.should be_empty
|
18
|
+
subject.start batch_a, worker_a
|
19
|
+
subject.batches.should == [batch_a]
|
20
|
+
subject.workers.should == [worker_a]
|
21
|
+
subject.start batch_a, worker_b
|
22
|
+
subject.batches.should == [batch_a]
|
23
|
+
subject.workers.should =~ [worker_a, worker_b]
|
24
|
+
subject.start batch_b, worker_c
|
25
|
+
subject.batches.should =~ [batch_a, batch_b]
|
26
|
+
subject.workers.should =~ [worker_a, worker_b, worker_c]
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'should register the batch and worker on completing' do
|
30
|
+
subject.batches.should be_empty
|
31
|
+
subject.workers.should be_empty
|
32
|
+
subject.complete batch_a, worker_a
|
33
|
+
subject.batches.should == [batch_a]
|
34
|
+
subject.workers.should == [worker_a]
|
35
|
+
subject.complete batch_a, worker_b
|
36
|
+
subject.batches.should == [batch_a]
|
37
|
+
subject.workers.should =~ [worker_a, worker_b]
|
38
|
+
subject.complete batch_b, worker_c
|
39
|
+
subject.batches.should =~ [batch_a, batch_b]
|
40
|
+
subject.workers.should =~ [worker_a, worker_b, worker_c]
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should return a list of started workers' do
|
44
|
+
subject.started_workers_for_batch(batch_a).should == []
|
45
|
+
subject.started_workers_for_batch(batch_b).should == []
|
46
|
+
subject.started_workers_for_batch(batch_c).should == []
|
47
|
+
subject.start batch_a, worker_a
|
48
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
49
|
+
subject.started_workers_for_batch(batch_b).should == []
|
50
|
+
subject.started_workers_for_batch(batch_c).should == []
|
51
|
+
subject.start batch_a, worker_b
|
52
|
+
subject.started_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
53
|
+
subject.started_workers_for_batch(batch_b).should == []
|
54
|
+
subject.started_workers_for_batch(batch_c).should == []
|
55
|
+
subject.start batch_c, worker_c
|
56
|
+
subject.started_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
57
|
+
subject.started_workers_for_batch(batch_b).should == []
|
58
|
+
subject.started_workers_for_batch(batch_c).should == [worker_c]
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should return a list of completed workers' do
|
62
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
63
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
64
|
+
subject.completed_workers_for_batch(batch_c).should == []
|
65
|
+
subject.complete batch_a, worker_a
|
66
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_a]
|
67
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
68
|
+
subject.completed_workers_for_batch(batch_c).should == []
|
69
|
+
subject.complete batch_a, worker_b
|
70
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
71
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
72
|
+
subject.completed_workers_for_batch(batch_c).should == []
|
73
|
+
subject.complete batch_c, worker_c
|
74
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
75
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
76
|
+
subject.completed_workers_for_batch(batch_c).should == [worker_c]
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'should remove from completed on starting' do
|
80
|
+
subject.started_workers_for_batch(batch_a).should == []
|
81
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
82
|
+
subject.complete batch_a, worker_a
|
83
|
+
subject.started_workers_for_batch(batch_a).should == []
|
84
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_a]
|
85
|
+
subject.start batch_a, worker_a
|
86
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
87
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'should work for the full flow' do
|
91
|
+
subject.started_workers_for_batch(batch_a).should == []
|
92
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
93
|
+
subject.start batch_a, worker_a
|
94
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
95
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
96
|
+
subject.complete batch_a, worker_a
|
97
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
98
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_a]
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should let you wait for a given batch to finish' do
|
102
|
+
encountered = 0
|
103
|
+
mock(subject).sleep(anything).times(3) do
|
104
|
+
encountered += 1
|
105
|
+
if encountered == 2
|
106
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_c]
|
107
|
+
subject.complete batch_a, worker_a
|
108
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_c, worker_a]
|
109
|
+
elsif encountered == 3
|
110
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_c, worker_a]
|
111
|
+
subject.complete batch_a, worker_b
|
112
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_c, worker_a, worker_b]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
subject.complete batch_a, worker_c
|
116
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_c]
|
117
|
+
subject.wait_for batch_a, 3
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
data/spec/runner_spec.rb
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SBM::Runner do
|
4
|
+
|
5
|
+
before do
|
6
|
+
ENV['SBM_WORKER'] = 'xyz'
|
7
|
+
end
|
8
|
+
|
9
|
+
context 'initialization' do
|
10
|
+
|
11
|
+
it 'should setup the default worker' do
|
12
|
+
instance = described_class.new([])
|
13
|
+
worker = instance.worker
|
14
|
+
worker.should be_a SBM::Coordinator::Worker
|
15
|
+
worker.name.should == SBM::Coordinator.defaults[1].name
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should setup the default coordinator' do
|
19
|
+
instance = described_class.new([])
|
20
|
+
coordinator = instance.coordinator
|
21
|
+
coordinator.should be_a SBM::Coordinator
|
22
|
+
coordinator.name.should == SBM::Coordinator.defaults[0].name
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should setup output / error' do
|
26
|
+
instance = described_class.new([])
|
27
|
+
instance.output.should == STDOUT
|
28
|
+
instance.error.should == STDERR
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should allow overriding the output and error' do
|
32
|
+
out = StringIO.new
|
33
|
+
err = StringIO.new
|
34
|
+
instance = described_class.new([], out, err)
|
35
|
+
instance.output.should == out
|
36
|
+
instance.error.should == err
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'should extract the command and args' do
|
40
|
+
instance = described_class.new ['x', 'y', 'z']
|
41
|
+
instance.command.should == 'x'
|
42
|
+
instance.args.should == ['y', 'z']
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'setting up runner stuff' do
|
48
|
+
|
49
|
+
let(:output) { StringIO.new }
|
50
|
+
let(:error) { StringIO.new }
|
51
|
+
|
52
|
+
let(:args) { [] }
|
53
|
+
|
54
|
+
subject do
|
55
|
+
instance = described_class.new args, output, error
|
56
|
+
stub(instance).exit.with_any_args
|
57
|
+
instance
|
58
|
+
end
|
59
|
+
|
60
|
+
context 'with a runner' do
|
61
|
+
|
62
|
+
it 'should let you validate the command' do
|
63
|
+
['status', 'start-batch', 'complete-batch', 'wait-for'].each do |command|
|
64
|
+
instance = described_class.new [command], output, error
|
65
|
+
dont_allow(instance).exit.with_any_args
|
66
|
+
instance.validate_command!
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'should exit with a bad command' do
|
71
|
+
['dfsdf', 'startbatch', 'complete', nil].each do |command|
|
72
|
+
instance = described_class.new [command], output, error
|
73
|
+
mock(instance).exit 1
|
74
|
+
instance.validate_command!
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'should run the command' do
|
79
|
+
args.replace %w(status)
|
80
|
+
mock(subject).status
|
81
|
+
subject.run
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'should work with non-standard commands' do
|
85
|
+
args.replace %w(start-batch)
|
86
|
+
mock(subject).start_batch
|
87
|
+
subject.run
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'should validate on run' do
|
91
|
+
args.replace %w(status)
|
92
|
+
mock(subject).validate_command!
|
93
|
+
subject.run
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
|
98
|
+
context 'starting batches' do
|
99
|
+
|
100
|
+
it 'should be an error without a batch name' do
|
101
|
+
subject.args.should == []
|
102
|
+
mock(subject).exit 1
|
103
|
+
subject.start_batch
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'should work with the coordinator' do
|
107
|
+
subject.args.replace %w(xyz)
|
108
|
+
mock(subject.coordinator).start subject.worker, SBM::Coordinator::Batch.new('xyz')
|
109
|
+
dont_allow(subject).exit
|
110
|
+
subject.start_batch
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
context 'completing batches' do
|
116
|
+
|
117
|
+
it 'should be an error without a batch name' do
|
118
|
+
subject.args.should == []
|
119
|
+
mock(subject).exit 1
|
120
|
+
subject.complete_batch
|
121
|
+
end
|
122
|
+
|
123
|
+
it 'should work with the coordinator' do
|
124
|
+
subject.args.replace %w(xyz)
|
125
|
+
mock(subject.coordinator).complete subject.worker, SBM::Coordinator::Batch.new('xyz')
|
126
|
+
dont_allow(subject).exit
|
127
|
+
subject.complete_batch
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
|
132
|
+
context 'waiting for batches' do
|
133
|
+
|
134
|
+
it 'should be an error without a batch name' do
|
135
|
+
subject.args.replace []
|
136
|
+
mock(subject).exit 1
|
137
|
+
subject.wait_for
|
138
|
+
end
|
139
|
+
|
140
|
+
it 'should be an error without an instance count' do
|
141
|
+
subject.args.replace ['test-batch']
|
142
|
+
mock(subject).exit 1
|
143
|
+
subject.wait_for
|
144
|
+
end
|
145
|
+
|
146
|
+
it 'should be an error with a bad instance count' do
|
147
|
+
subject.args.replace ['test-batch', '0']
|
148
|
+
mock(subject).exit 1
|
149
|
+
subject.wait_for
|
150
|
+
end
|
151
|
+
|
152
|
+
it 'should work with the coordinator' do
|
153
|
+
subject.args.replace %w(xyz 3)
|
154
|
+
mock(subject.coordinator).wait_for SBM::Coordinator::Batch.new('xyz'), 3
|
155
|
+
dont_allow(subject).exit
|
156
|
+
subject.wait_for
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sbm
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Darcy Laycock
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-06-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: redis
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rr
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: Tools for managed simple batches across N nodes.
|
84
|
+
email:
|
85
|
+
- sutto@sutto.net
|
86
|
+
executables:
|
87
|
+
- sbm
|
88
|
+
extensions: []
|
89
|
+
extra_rdoc_files: []
|
90
|
+
files:
|
91
|
+
- .gitignore
|
92
|
+
- .rspec
|
93
|
+
- Gemfile
|
94
|
+
- LICENSE.txt
|
95
|
+
- README.md
|
96
|
+
- Rakefile
|
97
|
+
- bin/sbm
|
98
|
+
- lib/sbm.rb
|
99
|
+
- lib/sbm/coordinator.rb
|
100
|
+
- lib/sbm/runner.rb
|
101
|
+
- lib/sbm/version.rb
|
102
|
+
- sbm.gemspec
|
103
|
+
- spec/coordinator_spec.rb
|
104
|
+
- spec/runner_spec.rb
|
105
|
+
- spec/spec_helper.rb
|
106
|
+
homepage: https://github.com/Sutto
|
107
|
+
licenses:
|
108
|
+
- MIT
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - '>='
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.0.2
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Built on redis, provides a basic set of tools that let you process tasks
|
130
|
+
in parallel across N nodes.
|
131
|
+
test_files:
|
132
|
+
- spec/coordinator_spec.rb
|
133
|
+
- spec/runner_spec.rb
|
134
|
+
- spec/spec_helper.rb
|