sbm 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +55 -0
- data/Rakefile +9 -0
- data/bin/sbm +3 -0
- data/lib/sbm.rb +6 -0
- data/lib/sbm/coordinator.rb +87 -0
- data/lib/sbm/runner.rb +92 -0
- data/lib/sbm/version.rb +3 -0
- data/sbm.gemspec +27 -0
- data/spec/coordinator_spec.rb +120 -0
- data/spec/runner_spec.rb +163 -0
- data/spec/spec_helper.rb +9 -0
- metadata +134 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5e18a93f50941dd24fa6348a0ca45ed4c7648dad
|
4
|
+
data.tar.gz: 2653657223ff6fc5ae1862839e8906e233f8fc14
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 21485914dab3bed5742d63d813c8dbf2b8de47dce842bb3c562426f8bf0e5a6956c586884aba8dcc33f93c06552d4302d11d24a8ddb51a8efbc9994d5c2840d2
|
7
|
+
data.tar.gz: c24c64c4c4a463572d23154f1c1b6ffb9613cf2100af5576312ee843a1384e5932397ffe46d95ae5b0c1e2001e63e374041f4a8427f1f047407fac8f0bf14455
|
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--colour
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 Darcy Laycock
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# SBM - Simple Batch Manager
|
2
|
+
|
3
|
+
Manages running / coordinating batch processes running across multiple hosts.
|
4
|
+
|
5
|
+
Uses redis as a simple coordinator to ensure split work runs across hosts evenly.
|
6
|
+
|
7
|
+
**Note:** SBM is still a hack. It's untested. Don't use this for production stuff. Please!
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
gem 'sbm'
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install sbm
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
SBM is composed of a set of simple scripts useful for running in shells. The only required
|
26
|
+
variable is `SBM_WORKER` as an environment variable - the name of the node the current job is
|
27
|
+
running on.`
|
28
|
+
|
29
|
+
```bash
|
30
|
+
#!/usr/bin/env bash -e
|
31
|
+
|
32
|
+
export SBM_WORKER="$(hostname)-$$"
|
33
|
+
|
34
|
+
# Used if you have a bunch of different batches with the same name:
|
35
|
+
# export SBM_COORDINATOR='your-groups'
|
36
|
+
|
37
|
+
sbm start-work my-test-batch
|
38
|
+
rake do:your:work
|
39
|
+
sbm complete-work my-test-batch && sbm wait-for my-test-batch 20 # There are 20 nodes running this process
|
40
|
+
|
41
|
+
sbm status
|
42
|
+
|
43
|
+
```
|
44
|
+
|
45
|
+
Wait for simply checks the number of items in the completed set have the correct length.
|
46
|
+
|
47
|
+
Please note that by default it uses redis for this, so to change your default redis use `REDIS_URI`.
|
48
|
+
|
49
|
+
## Contributing
|
50
|
+
|
51
|
+
1. Fork it
|
52
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
53
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
54
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
55
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/bin/sbm
ADDED
data/lib/sbm.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
# Handles the base of coordinating works in sets between
|
2
|
+
# X nodes, with each item having a node identifier.
|
3
|
+
|
4
|
+
require 'redis'
|
5
|
+
|
6
|
+
module SBM
|
7
|
+
class Coordinator
|
8
|
+
|
9
|
+
def self.defaults
|
10
|
+
worker_name = (ENV['SBM_WORKER'] or raise "Please ensure SBM_WORKER is set")
|
11
|
+
coordinator_name = (ENV['SBM_COORDINATOR'] || "worker-coordinator")
|
12
|
+
return new(coordinator_name), Worker.new(worker_name)
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :name, :redis
|
16
|
+
|
17
|
+
def initialize(name)
|
18
|
+
@name = name.to_s
|
19
|
+
@redis = Redis.current
|
20
|
+
end
|
21
|
+
|
22
|
+
class Batch < Struct.new(:name)
|
23
|
+
|
24
|
+
def to_s; name; end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
class Worker < Struct.new(:name)
|
29
|
+
def to_s; name; end
|
30
|
+
end
|
31
|
+
|
32
|
+
def batches
|
33
|
+
redis.smembers(key(:batches)).map { |w| Batch.new(w) }
|
34
|
+
end
|
35
|
+
|
36
|
+
def workers
|
37
|
+
redis.smembers(key(:workers)).map { |w| Worker.new(w) }
|
38
|
+
end
|
39
|
+
|
40
|
+
def started_workers_for_batch(batch)
|
41
|
+
redis.smembers(key(:batches, batch, :started)).map { |w| Worker.new(w) }
|
42
|
+
end
|
43
|
+
|
44
|
+
def completed_workers_for_batch(batch)
|
45
|
+
redis.smembers(key(:batches, batch, :completed)).map { |w| Worker.new(w) }
|
46
|
+
end
|
47
|
+
|
48
|
+
def start(batch, worker)
|
49
|
+
prepare worker, batch
|
50
|
+
redis.sadd key(:batches, batch, :started), worker.to_s
|
51
|
+
redis.srem key(:batches, batch, :completed), worker.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
def complete(batch, worker)
|
55
|
+
prepare worker, batch
|
56
|
+
redis.sadd key(:batches, batch, :completed), worker.to_s
|
57
|
+
end
|
58
|
+
|
59
|
+
# Waits on batch to reach a count, waiting for 15 seconds at a time.
|
60
|
+
def wait_for(batch, worker_count, wait_time = 15)
|
61
|
+
while redis.scard(key(:batches, batch, :completed)) < worker_count
|
62
|
+
sleep wait_time
|
63
|
+
yield if block_given?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def prepare(worker, batch)
|
70
|
+
register_worker worker
|
71
|
+
register_batch batch
|
72
|
+
end
|
73
|
+
|
74
|
+
def register_worker(worker)
|
75
|
+
redis.sadd key(:workers), worker.to_s
|
76
|
+
end
|
77
|
+
|
78
|
+
def register_batch(batch)
|
79
|
+
redis.sadd key(:batches), batch.to_s
|
80
|
+
end
|
81
|
+
|
82
|
+
def key(*args)
|
83
|
+
[name, *args].join(":")
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
end
|
data/lib/sbm/runner.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
module SBM
|
2
|
+
class Runner
|
3
|
+
|
4
|
+
USAGES = {
|
5
|
+
'status' => '',
|
6
|
+
'wait-for' => 'batch-name worker-count',
|
7
|
+
'start-batch' => 'batch-name',
|
8
|
+
'complete-batch' => 'batch-name'
|
9
|
+
}
|
10
|
+
|
11
|
+
attr_reader :command, :args, :coordinator, :worker, :output, :error
|
12
|
+
|
13
|
+
def initialize(args, output = STDOUT, error = STDERR)
|
14
|
+
@command = args.first
|
15
|
+
@args = args.drop(1)
|
16
|
+
@output = output
|
17
|
+
@error = error
|
18
|
+
@coordinator, @worker = SBM::Coordinator.defaults
|
19
|
+
end
|
20
|
+
|
21
|
+
def validate_command!
|
22
|
+
if command.nil? or !USAGES.has_key?(command)
|
23
|
+
usage true
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def run
|
28
|
+
validate_command!
|
29
|
+
send command.tr('-', '_').to_sym
|
30
|
+
end
|
31
|
+
|
32
|
+
def status
|
33
|
+
output.puts "Known Workers: #{coordinator.workers.map(&:name).sort.join(", ")}"
|
34
|
+
output.puts "Known Batches: #{coordinator.batches.map(&:name).sort.join(", ")}"
|
35
|
+
output.puts ""
|
36
|
+
output.puts ""
|
37
|
+
coordinator.batches.each do |batch|
|
38
|
+
started = coordinator.started_workers_for_batch batch
|
39
|
+
completed = coordinator.started_workers_for_batch completed
|
40
|
+
output.puts "Batch: #{batch}"
|
41
|
+
output.puts "Number Started: #{started.size}"
|
42
|
+
output.puts "Number Completed: #{completed.size}"
|
43
|
+
output.puts "Number Pending: #{started.size - completed.size}"
|
44
|
+
output.puts "---"
|
45
|
+
output.puts "Started: #{started.map(&:name).sort.join(", ")}"
|
46
|
+
output.puts "Completed: #{completed.map(&:name).sort.join(", ")}"
|
47
|
+
output.puts ""
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def wait_for
|
52
|
+
batch = extract_batch!
|
53
|
+
worker_count = args.shift.to_i
|
54
|
+
if worker_count.zero?
|
55
|
+
error.puts "You must provide a non-zero worker count"
|
56
|
+
usage
|
57
|
+
end
|
58
|
+
coordinator.wait_for batch, worker_count
|
59
|
+
end
|
60
|
+
|
61
|
+
def start_batch
|
62
|
+
batch = extract_batch!
|
63
|
+
coordinator.start batch, worker
|
64
|
+
end
|
65
|
+
|
66
|
+
def complete_batch
|
67
|
+
batch = extract_batch!
|
68
|
+
coordinator.complete batch, worker
|
69
|
+
end
|
70
|
+
|
71
|
+
def usage(invalid_command = false)
|
72
|
+
if invalid_command
|
73
|
+
error.puts "Invalid / unknown command - must be one of #{USAGES.keys.join(", ")}"
|
74
|
+
error.puts "Usage: #$0 #{USAGES.keys.join("|")} [arguments]"
|
75
|
+
exit 1
|
76
|
+
else
|
77
|
+
error.puts "Usage: #$0 #{command} #{USAGES[command]}".strip
|
78
|
+
exit 1
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def extract_batch!
|
83
|
+
batch_name = args.shift
|
84
|
+
if batch_name.to_s.strip.empty?
|
85
|
+
error.puts "You must provide a batch name."
|
86
|
+
usage
|
87
|
+
end
|
88
|
+
Coordinator::Batch.new(batch_name)
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
end
|
data/lib/sbm/version.rb
ADDED
data/sbm.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'sbm/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "sbm"
|
8
|
+
spec.version = SBM::VERSION
|
9
|
+
spec.authors = ["Darcy Laycock"]
|
10
|
+
spec.email = ["sutto@sutto.net"]
|
11
|
+
spec.description = %q{Tools for managed simple batches across N nodes.}
|
12
|
+
spec.summary = %q{Built on redis, provides a basic set of tools that let you process tasks in parallel across N nodes.}
|
13
|
+
spec.homepage = "https://github.com/Sutto"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "redis"
|
22
|
+
|
23
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
24
|
+
spec.add_development_dependency "rake"
|
25
|
+
spec.add_development_dependency "rspec"
|
26
|
+
spec.add_development_dependency "rr"
|
27
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SBM::Coordinator do
|
4
|
+
|
5
|
+
let(:worker_a) { described_class::Worker.new 'elephant' }
|
6
|
+
let(:worker_b) { described_class::Worker.new 'pony' }
|
7
|
+
let(:worker_c) { described_class::Worker.new 'fish' }
|
8
|
+
|
9
|
+
let(:batch_a) { described_class::Batch.new 'fishing' }
|
10
|
+
let(:batch_b) { described_class::Batch.new 'running' }
|
11
|
+
let(:batch_c) { described_class::Batch.new 'rocking' }
|
12
|
+
|
13
|
+
subject { described_class.new 'my-awesome-item' }
|
14
|
+
|
15
|
+
it 'should register the batch and worker on starting' do
|
16
|
+
subject.batches.should be_empty
|
17
|
+
subject.workers.should be_empty
|
18
|
+
subject.start batch_a, worker_a
|
19
|
+
subject.batches.should == [batch_a]
|
20
|
+
subject.workers.should == [worker_a]
|
21
|
+
subject.start batch_a, worker_b
|
22
|
+
subject.batches.should == [batch_a]
|
23
|
+
subject.workers.should =~ [worker_a, worker_b]
|
24
|
+
subject.start batch_b, worker_c
|
25
|
+
subject.batches.should =~ [batch_a, batch_b]
|
26
|
+
subject.workers.should =~ [worker_a, worker_b, worker_c]
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'should register the batch and worker on completing' do
|
30
|
+
subject.batches.should be_empty
|
31
|
+
subject.workers.should be_empty
|
32
|
+
subject.complete batch_a, worker_a
|
33
|
+
subject.batches.should == [batch_a]
|
34
|
+
subject.workers.should == [worker_a]
|
35
|
+
subject.complete batch_a, worker_b
|
36
|
+
subject.batches.should == [batch_a]
|
37
|
+
subject.workers.should =~ [worker_a, worker_b]
|
38
|
+
subject.complete batch_b, worker_c
|
39
|
+
subject.batches.should =~ [batch_a, batch_b]
|
40
|
+
subject.workers.should =~ [worker_a, worker_b, worker_c]
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should return a list of started workers' do
|
44
|
+
subject.started_workers_for_batch(batch_a).should == []
|
45
|
+
subject.started_workers_for_batch(batch_b).should == []
|
46
|
+
subject.started_workers_for_batch(batch_c).should == []
|
47
|
+
subject.start batch_a, worker_a
|
48
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
49
|
+
subject.started_workers_for_batch(batch_b).should == []
|
50
|
+
subject.started_workers_for_batch(batch_c).should == []
|
51
|
+
subject.start batch_a, worker_b
|
52
|
+
subject.started_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
53
|
+
subject.started_workers_for_batch(batch_b).should == []
|
54
|
+
subject.started_workers_for_batch(batch_c).should == []
|
55
|
+
subject.start batch_c, worker_c
|
56
|
+
subject.started_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
57
|
+
subject.started_workers_for_batch(batch_b).should == []
|
58
|
+
subject.started_workers_for_batch(batch_c).should == [worker_c]
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should return a list of completed workers' do
|
62
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
63
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
64
|
+
subject.completed_workers_for_batch(batch_c).should == []
|
65
|
+
subject.complete batch_a, worker_a
|
66
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_a]
|
67
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
68
|
+
subject.completed_workers_for_batch(batch_c).should == []
|
69
|
+
subject.complete batch_a, worker_b
|
70
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
71
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
72
|
+
subject.completed_workers_for_batch(batch_c).should == []
|
73
|
+
subject.complete batch_c, worker_c
|
74
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_a, worker_b]
|
75
|
+
subject.completed_workers_for_batch(batch_b).should == []
|
76
|
+
subject.completed_workers_for_batch(batch_c).should == [worker_c]
|
77
|
+
end
|
78
|
+
|
79
|
+
it 'should remove from completed on starting' do
|
80
|
+
subject.started_workers_for_batch(batch_a).should == []
|
81
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
82
|
+
subject.complete batch_a, worker_a
|
83
|
+
subject.started_workers_for_batch(batch_a).should == []
|
84
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_a]
|
85
|
+
subject.start batch_a, worker_a
|
86
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
87
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'should work for the full flow' do
|
91
|
+
subject.started_workers_for_batch(batch_a).should == []
|
92
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
93
|
+
subject.start batch_a, worker_a
|
94
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
95
|
+
subject.completed_workers_for_batch(batch_a).should == []
|
96
|
+
subject.complete batch_a, worker_a
|
97
|
+
subject.started_workers_for_batch(batch_a).should == [worker_a]
|
98
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_a]
|
99
|
+
end
|
100
|
+
|
101
|
+
it 'should let you wait for a given batch to finish' do
|
102
|
+
encountered = 0
|
103
|
+
mock(subject).sleep(anything).times(3) do
|
104
|
+
encountered += 1
|
105
|
+
if encountered == 2
|
106
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_c]
|
107
|
+
subject.complete batch_a, worker_a
|
108
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_c, worker_a]
|
109
|
+
elsif encountered == 3
|
110
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_c, worker_a]
|
111
|
+
subject.complete batch_a, worker_b
|
112
|
+
subject.completed_workers_for_batch(batch_a).should =~ [worker_c, worker_a, worker_b]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
subject.complete batch_a, worker_c
|
116
|
+
subject.completed_workers_for_batch(batch_a).should == [worker_c]
|
117
|
+
subject.wait_for batch_a, 3
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
data/spec/runner_spec.rb
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe SBM::Runner do
|
4
|
+
|
5
|
+
before do
|
6
|
+
ENV['SBM_WORKER'] = 'xyz'
|
7
|
+
end
|
8
|
+
|
9
|
+
context 'initialization' do
|
10
|
+
|
11
|
+
it 'should setup the default worker' do
|
12
|
+
instance = described_class.new([])
|
13
|
+
worker = instance.worker
|
14
|
+
worker.should be_a SBM::Coordinator::Worker
|
15
|
+
worker.name.should == SBM::Coordinator.defaults[1].name
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should setup the default coordinator' do
|
19
|
+
instance = described_class.new([])
|
20
|
+
coordinator = instance.coordinator
|
21
|
+
coordinator.should be_a SBM::Coordinator
|
22
|
+
coordinator.name.should == SBM::Coordinator.defaults[0].name
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should setup output / error' do
|
26
|
+
instance = described_class.new([])
|
27
|
+
instance.output.should == STDOUT
|
28
|
+
instance.error.should == STDERR
|
29
|
+
end
|
30
|
+
|
31
|
+
it 'should allow overriding the output and error' do
|
32
|
+
out = StringIO.new
|
33
|
+
err = StringIO.new
|
34
|
+
instance = described_class.new([], out, err)
|
35
|
+
instance.output.should == out
|
36
|
+
instance.error.should == err
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'should extract the command and args' do
|
40
|
+
instance = described_class.new ['x', 'y', 'z']
|
41
|
+
instance.command.should == 'x'
|
42
|
+
instance.args.should == ['y', 'z']
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'setting up runner stuff' do
|
48
|
+
|
49
|
+
let(:output) { StringIO.new }
|
50
|
+
let(:error) { StringIO.new }
|
51
|
+
|
52
|
+
let(:args) { [] }
|
53
|
+
|
54
|
+
subject do
|
55
|
+
instance = described_class.new args, output, error
|
56
|
+
stub(instance).exit.with_any_args
|
57
|
+
instance
|
58
|
+
end
|
59
|
+
|
60
|
+
context 'with a runner' do
|
61
|
+
|
62
|
+
it 'should let you validate the command' do
|
63
|
+
['status', 'start-batch', 'complete-batch', 'wait-for'].each do |command|
|
64
|
+
instance = described_class.new [command], output, error
|
65
|
+
dont_allow(instance).exit.with_any_args
|
66
|
+
instance.validate_command!
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'should exit with a bad command' do
|
71
|
+
['dfsdf', 'startbatch', 'complete', nil].each do |command|
|
72
|
+
instance = described_class.new [command], output, error
|
73
|
+
mock(instance).exit 1
|
74
|
+
instance.validate_command!
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'should run the command' do
|
79
|
+
args.replace %w(status)
|
80
|
+
mock(subject).status
|
81
|
+
subject.run
|
82
|
+
end
|
83
|
+
|
84
|
+
it 'should work with non-standard commands' do
|
85
|
+
args.replace %w(start-batch)
|
86
|
+
mock(subject).start_batch
|
87
|
+
subject.run
|
88
|
+
end
|
89
|
+
|
90
|
+
it 'should validate on run' do
|
91
|
+
args.replace %w(status)
|
92
|
+
mock(subject).validate_command!
|
93
|
+
subject.run
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
|
98
|
+
context 'starting batches' do
|
99
|
+
|
100
|
+
it 'should be an error without a batch name' do
|
101
|
+
subject.args.should == []
|
102
|
+
mock(subject).exit 1
|
103
|
+
subject.start_batch
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'should work with the coordinator' do
|
107
|
+
subject.args.replace %w(xyz)
|
108
|
+
mock(subject.coordinator).start subject.worker, SBM::Coordinator::Batch.new('xyz')
|
109
|
+
dont_allow(subject).exit
|
110
|
+
subject.start_batch
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
context 'completing batches' do
|
116
|
+
|
117
|
+
it 'should be an error without a batch name' do
|
118
|
+
subject.args.should == []
|
119
|
+
mock(subject).exit 1
|
120
|
+
subject.complete_batch
|
121
|
+
end
|
122
|
+
|
123
|
+
it 'should work with the coordinator' do
|
124
|
+
subject.args.replace %w(xyz)
|
125
|
+
mock(subject.coordinator).complete subject.worker, SBM::Coordinator::Batch.new('xyz')
|
126
|
+
dont_allow(subject).exit
|
127
|
+
subject.complete_batch
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
|
132
|
+
context 'waiting for batches' do
|
133
|
+
|
134
|
+
it 'should be an error without a batch name' do
|
135
|
+
subject.args.replace []
|
136
|
+
mock(subject).exit 1
|
137
|
+
subject.wait_for
|
138
|
+
end
|
139
|
+
|
140
|
+
it 'should be an error without an instance count' do
|
141
|
+
subject.args.replace ['test-batch']
|
142
|
+
mock(subject).exit 1
|
143
|
+
subject.wait_for
|
144
|
+
end
|
145
|
+
|
146
|
+
it 'should be an error with a bad instance count' do
|
147
|
+
subject.args.replace ['test-batch', '0']
|
148
|
+
mock(subject).exit 1
|
149
|
+
subject.wait_for
|
150
|
+
end
|
151
|
+
|
152
|
+
it 'should work with the coordinator' do
|
153
|
+
subject.args.replace %w(xyz 3)
|
154
|
+
mock(subject.coordinator).wait_for SBM::Coordinator::Batch.new('xyz'), 3
|
155
|
+
dont_allow(subject).exit
|
156
|
+
subject.wait_for
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
end
|
data/spec/spec_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,134 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sbm
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Darcy Laycock
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-06-24 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: redis
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rr
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: Tools for managed simple batches across N nodes.
|
84
|
+
email:
|
85
|
+
- sutto@sutto.net
|
86
|
+
executables:
|
87
|
+
- sbm
|
88
|
+
extensions: []
|
89
|
+
extra_rdoc_files: []
|
90
|
+
files:
|
91
|
+
- .gitignore
|
92
|
+
- .rspec
|
93
|
+
- Gemfile
|
94
|
+
- LICENSE.txt
|
95
|
+
- README.md
|
96
|
+
- Rakefile
|
97
|
+
- bin/sbm
|
98
|
+
- lib/sbm.rb
|
99
|
+
- lib/sbm/coordinator.rb
|
100
|
+
- lib/sbm/runner.rb
|
101
|
+
- lib/sbm/version.rb
|
102
|
+
- sbm.gemspec
|
103
|
+
- spec/coordinator_spec.rb
|
104
|
+
- spec/runner_spec.rb
|
105
|
+
- spec/spec_helper.rb
|
106
|
+
homepage: https://github.com/Sutto
|
107
|
+
licenses:
|
108
|
+
- MIT
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - '>='
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - '>='
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.0.2
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Built on redis, provides a basic set of tools that let you process tasks
|
130
|
+
in parallel across N nodes.
|
131
|
+
test_files:
|
132
|
+
- spec/coordinator_spec.rb
|
133
|
+
- spec/runner_spec.rb
|
134
|
+
- spec/spec_helper.rb
|