disbatch 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/disbatchd +25 -11
- data/lib/disbatch/error.rb +24 -0
- data/lib/disbatch/node.rb +36 -25
- data/lib/disbatch/plugin.rb +4 -3
- data/lib/disbatch/queue.rb +5 -9
- data/lib/disbatch/{queue/task.rb → task.rb} +13 -6
- data/lib/disbatch.rb +9 -3
- metadata +4 -4
- data/lib/disbatch/errors.rb +0 -6
data/bin/disbatchd
CHANGED
@@ -11,6 +11,9 @@ opts = Trollop.options do
|
|
11
11
|
opt 'config', 'the disbatchd config file', :default => '/etc/disbatch/disbatchd.conf'
|
12
12
|
opt 'max', 'maximum number of threads per queue', :default => 10
|
13
13
|
opt 'plugins', 'path(s) to dispatch plugins', :multi => true, :type => :string
|
14
|
+
opt 'mongo_host', 'mongo host'
|
15
|
+
opt 'mongo_port', 'mongo port'
|
16
|
+
opt 'mongo_db', 'mongo db'
|
14
17
|
opt 'force', 'force node registration'
|
15
18
|
end
|
16
19
|
|
@@ -20,25 +23,27 @@ config = opts.merge(config)
|
|
20
23
|
force = config['force'] || false
|
21
24
|
|
22
25
|
node = Disbatch.node.register(force)
|
23
|
-
max = config['max']
|
26
|
+
max = config.has_key?('max') ? config['max'] : 10
|
24
27
|
|
25
|
-
config.has_key?('plugins') && config['plugins'].each { |dir| Disbatch::Plugin.init_all(dir) }
|
26
28
|
Disbatch::Plugin.init_all(File.dirname(File.expand_path(__FILE__)) + '/../lib/disbatch/plugin/**/*.rb')
|
29
|
+
config.has_key?('plugins') && config['plugins'].each { |dir| Disbatch::Plugin.init_all(dir) }
|
30
|
+
|
31
|
+
config.has_key?('mongo_host') && Disbatch.mongo_host = config['mongo_host']
|
32
|
+
config.has_key?('mongo_port') && Disbatch.mongo_port = config['mongo_port']
|
33
|
+
config.has_key?('mongo_db') && Disbatch.mongo_db = config['mongo_db']
|
27
34
|
|
28
35
|
trap('TERM') { node.release; exit }
|
29
36
|
trap('INT') { node.release; exit }
|
30
37
|
trap('QUIT') { node.release; exit }
|
31
38
|
|
32
|
-
runners = {}
|
39
|
+
@runners = {}
|
33
40
|
|
34
|
-
|
35
|
-
|
36
|
-
EventMachine::add_periodic_timer(5) do
|
41
|
+
def update_queues
|
37
42
|
|
38
43
|
Disbatch::Queue.get_all.each do |queue|
|
39
44
|
|
40
45
|
# runner already exists
|
41
|
-
next if runners.has_key?(queue.id)
|
46
|
+
next if @runners.has_key?(queue.id)
|
42
47
|
|
43
48
|
# node configured to ignore
|
44
49
|
# next if queue.nodes_ignore.include?(node.id)
|
@@ -51,20 +56,29 @@ EventMachine::run do
|
|
51
56
|
|
52
57
|
puts "Adding #{queue.plugin} runner for #{queue.id}"
|
53
58
|
|
54
|
-
runners[queue.id] = { 'tg' => ThreadGroup.new, 'queue' => queue }
|
59
|
+
@runners[queue.id] = { 'tg' => ThreadGroup.new, 'queue' => queue }
|
55
60
|
|
56
61
|
end
|
57
62
|
|
63
|
+
end
|
64
|
+
|
65
|
+
update_queues
|
66
|
+
|
67
|
+
EventMachine::run do
|
68
|
+
|
69
|
+
EventMachine::add_periodic_timer(5) do
|
70
|
+
|
71
|
+
update_queues
|
58
72
|
|
59
|
-
runners.each do |id, runner|
|
73
|
+
@runners.each do |id, runner|
|
60
74
|
puts "#{id}: #{runner['queue'].plugin}: #{runner['tg'].list.length}"
|
61
75
|
end
|
62
76
|
|
63
77
|
end
|
64
78
|
|
65
|
-
EventMachine::add_periodic_timer(
|
79
|
+
EventMachine::add_periodic_timer(0.5) do
|
66
80
|
|
67
|
-
runners.each do |id, runner|
|
81
|
+
@runners.each do |id, runner|
|
68
82
|
|
69
83
|
tg = runner['tg']
|
70
84
|
queue = runner['queue']
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Disbatch
|
2
|
+
|
3
|
+
# Raised when attempting to open an invalid node
|
4
|
+
class NoNodeError < RuntimeError; end
|
5
|
+
|
6
|
+
# Raised when attempting to open an owned node
|
7
|
+
class RegisteredNodeError < RuntimeError; end
|
8
|
+
|
9
|
+
# Raised when attempting to open a node twice
|
10
|
+
class AlreadyRegisteredNodeError < RuntimeError; end
|
11
|
+
|
12
|
+
# Raised when attempting to release an unregistered node
|
13
|
+
class NotRegisteredNodeError < RuntimeError; end
|
14
|
+
|
15
|
+
# Raised when specifying a non-existant plugin
|
16
|
+
class NoPluginError < RuntimeError; end
|
17
|
+
|
18
|
+
# Raised when specifying an invalid plugin
|
19
|
+
class InvalidPluginError < RuntimeError; end
|
20
|
+
|
21
|
+
# Raised when specifying a non-existent queue
|
22
|
+
class NoQueueError < RuntimeError; end
|
23
|
+
|
24
|
+
end
|
data/lib/disbatch/node.rb
CHANGED
@@ -9,7 +9,7 @@ class Disbatch::Node
|
|
9
9
|
#
|
10
10
|
# @param [String] id
|
11
11
|
def initialize(id)
|
12
|
-
@
|
12
|
+
@registered = false
|
13
13
|
@id = id
|
14
14
|
end
|
15
15
|
|
@@ -18,7 +18,8 @@ class Disbatch::Node
|
|
18
18
|
# @param [String] id
|
19
19
|
def self.get(id = Disbatch.node_id)
|
20
20
|
Mongo.try do
|
21
|
-
Disbatch.db[:nodes].find_one({:_id => id})
|
21
|
+
raise Disbatch::NoNodeError unless Disbatch.db[:nodes].find_one({:_id => id})
|
22
|
+
new(id)
|
22
23
|
end
|
23
24
|
end
|
24
25
|
|
@@ -45,32 +46,38 @@ class Disbatch::Node
|
|
45
46
|
|
46
47
|
# Register node
|
47
48
|
def register(force = false)
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
}
|
60
|
-
|
49
|
+
raise Disbatch::AlreadyRegisteredNodeError if @registered
|
50
|
+
|
51
|
+
begin
|
52
|
+
doc = Mongo.try do
|
53
|
+
Disbatch.db[:nodes].find_and_modify({
|
54
|
+
:query => force ? { :_id => @id } : { :_id => @id, :pid => nil },
|
55
|
+
:update => { :$set => {
|
56
|
+
:pid => Process.pid,
|
57
|
+
:version => Disbatch::VERSION,
|
58
|
+
:spec_version => Disbatch::SPEC_VERSION
|
59
|
+
} }
|
60
|
+
})
|
61
|
+
end
|
62
|
+
rescue
|
63
|
+
doc = Mongo.try do
|
64
|
+
Disbatch.db[:nodes].find({ :_id => @id })
|
65
|
+
end
|
66
|
+
|
67
|
+
if doc.count == 0
|
68
|
+
raise Disbatch::NoNodeError
|
69
|
+
else
|
70
|
+
raise Disbatch::RegisteredNodeError
|
71
|
+
end
|
61
72
|
end
|
62
73
|
|
63
|
-
|
64
|
-
|
65
|
-
else
|
66
|
-
@open = true
|
67
|
-
self
|
68
|
-
end
|
74
|
+
@registered = true
|
75
|
+
self
|
69
76
|
end
|
70
77
|
|
71
78
|
# Release ownership of node
|
72
79
|
def release
|
73
|
-
|
80
|
+
raise Disbatch::NotRegisteredNodeError unless @registered
|
74
81
|
|
75
82
|
doc = Mongo.try do
|
76
83
|
Disbatch.db[:nodes].find_and_modify({
|
@@ -86,13 +93,14 @@ class Disbatch::Node
|
|
86
93
|
if doc.nil?
|
87
94
|
false
|
88
95
|
else
|
89
|
-
@
|
96
|
+
@registered = false
|
90
97
|
true
|
91
98
|
end
|
92
99
|
end
|
93
100
|
|
94
|
-
|
95
|
-
|
101
|
+
# Check if process is registered as this node.
|
102
|
+
def registered?
|
103
|
+
@registered
|
96
104
|
end
|
97
105
|
|
98
106
|
# Check equality with another node object
|
@@ -102,4 +110,7 @@ class Disbatch::Node
|
|
102
110
|
@id == node.id
|
103
111
|
end
|
104
112
|
|
113
|
+
# Deprecated.
|
114
|
+
alias :open? :registered?
|
115
|
+
|
105
116
|
end
|
data/lib/disbatch/plugin.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
module Disbatch::Plugin
|
2
2
|
|
3
|
-
|
4
3
|
attr_reader :plugins
|
5
4
|
|
6
5
|
@plugins = {}
|
@@ -9,7 +8,7 @@ module Disbatch::Plugin
|
|
9
8
|
def register(plugin)
|
10
9
|
name = plugin.to_s
|
11
10
|
|
12
|
-
|
11
|
+
raise Disbatch::InvalidPluginError unless plugin.respond_to?(:execute)
|
13
12
|
|
14
13
|
@plugins[name] = plugin
|
15
14
|
puts "Registered #{name}"
|
@@ -17,6 +16,7 @@ module Disbatch::Plugin
|
|
17
16
|
|
18
17
|
# Return a plugin by name
|
19
18
|
def [](name)
|
19
|
+
raise Disbatch::NoPluginError unless @plugins.has_key?(name)
|
20
20
|
@plugins[name]
|
21
21
|
end
|
22
22
|
|
@@ -24,7 +24,6 @@ module Disbatch::Plugin
|
|
24
24
|
def init(file)
|
25
25
|
begin
|
26
26
|
load file
|
27
|
-
puts "Loaded #{file}"
|
28
27
|
rescue
|
29
28
|
puts "Error loading #{file}"
|
30
29
|
end
|
@@ -37,4 +36,6 @@ module Disbatch::Plugin
|
|
37
36
|
|
38
37
|
extend self
|
39
38
|
|
39
|
+
init_all('disbatch/plugin/**/*')
|
40
|
+
|
40
41
|
end
|
data/lib/disbatch/queue.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# Represents a Disbatch queue
|
2
2
|
class Disbatch::Queue
|
3
3
|
|
4
|
-
require 'disbatch/
|
4
|
+
require 'disbatch/task'
|
5
5
|
|
6
6
|
attr_reader :plugin, :id
|
7
7
|
|
@@ -24,9 +24,7 @@ class Disbatch::Queue
|
|
24
24
|
Disbatch.db[:queues].find_one({:_id => id})
|
25
25
|
end
|
26
26
|
|
27
|
-
|
28
|
-
new(doc['class'], id)
|
29
|
-
end
|
27
|
+
raise Disbatch::NoQueueError if doc.nil?
|
30
28
|
end
|
31
29
|
|
32
30
|
# Get all existing queues
|
@@ -46,8 +44,7 @@ class Disbatch::Queue
|
|
46
44
|
# @option opts [Array] nodes_pin
|
47
45
|
# @option opts [Array] nodes_ignore
|
48
46
|
def self.create(plugin, opts={})
|
49
|
-
|
50
|
-
# Throw exception if plugin doesn't exist?
|
47
|
+
raise Disbatch::NoPluginError unless Disbatch::Plugin[plugin]
|
51
48
|
|
52
49
|
id = opts[:id] || BSON::ObjectId.new.to_s
|
53
50
|
maxthreads = opts[:maxthreads] || 10
|
@@ -70,7 +67,6 @@ class Disbatch::Queue
|
|
70
67
|
unless doc.nil?
|
71
68
|
new(plugin, id)
|
72
69
|
end
|
73
|
-
|
74
70
|
end
|
75
71
|
|
76
72
|
# Number of pending tasks
|
@@ -80,13 +76,13 @@ class Disbatch::Queue
|
|
80
76
|
|
81
77
|
# Push a new task onto the queue
|
82
78
|
def push(parameters)
|
83
|
-
Disbatch::
|
79
|
+
Disbatch::Task.create(self, parameters)
|
84
80
|
self
|
85
81
|
end
|
86
82
|
|
87
83
|
# Pop a task off the queue
|
88
84
|
def pop
|
89
|
-
Disbatch::
|
85
|
+
Disbatch::Task.take(self)
|
90
86
|
end
|
91
87
|
|
92
88
|
# Check equality with another queue object
|
@@ -1,12 +1,19 @@
|
|
1
1
|
# Represents a Disbatch task
|
2
|
-
class Disbatch::
|
3
|
-
|
4
|
-
require 'pp'
|
2
|
+
class Disbatch::Task
|
5
3
|
|
6
4
|
attr_reader :queue, :parameters, :id
|
7
5
|
|
8
6
|
private_class_method :new
|
9
7
|
|
8
|
+
module Status
|
9
|
+
BLOCKED = -4
|
10
|
+
TERMINATED = -3
|
11
|
+
CREATED = -2
|
12
|
+
CLAIMED = -1
|
13
|
+
RUNNING = 0
|
14
|
+
CONCLUDED = 1
|
15
|
+
end
|
16
|
+
|
10
17
|
# Create a task object
|
11
18
|
#
|
12
19
|
# @param [Disbatch::Queue] queue
|
@@ -24,8 +31,8 @@ class Disbatch::Queue::Task
|
|
24
31
|
def self.take(queue)
|
25
32
|
doc = Mongo.try do
|
26
33
|
Disbatch.db[:tasks].find_and_modify({
|
27
|
-
:query => { :queue => queue.id, :status =>
|
28
|
-
:update => { :$set => { :node => Disbatch.node.id, :status =>
|
34
|
+
:query => { :queue => queue.id, :status => Status::CREATED },
|
35
|
+
:update => { :$set => { :node => Disbatch.node.id, :status => Status::CLAIMED, } }
|
29
36
|
})
|
30
37
|
end
|
31
38
|
|
@@ -46,7 +53,7 @@ class Disbatch::Queue::Task
|
|
46
53
|
:ctime => Time.now,
|
47
54
|
:mtime => Time.now,
|
48
55
|
:node => -1,
|
49
|
-
:status =>
|
56
|
+
:status => Status::CREATED,
|
50
57
|
:stdout => '',
|
51
58
|
:stderr => '',
|
52
59
|
:log => []
|
data/lib/disbatch.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
module Disbatch
|
2
2
|
|
3
3
|
# engine version
|
4
|
-
VERSION = 'rdisbatch 0.0.
|
4
|
+
VERSION = 'rdisbatch 0.0.9'
|
5
5
|
# specification version
|
6
6
|
SPEC_VERSION = '1.9'
|
7
7
|
|
8
|
-
attr_accessor :
|
8
|
+
attr_accessor :mongo_host
|
9
|
+
attr_accessor :mongo_port
|
9
10
|
attr_accessor :mongo_db
|
10
11
|
attr_accessor :mongo_opts
|
11
12
|
|
@@ -16,6 +17,7 @@ module Disbatch
|
|
16
17
|
require 'mongo'
|
17
18
|
require 'socket'
|
18
19
|
|
20
|
+
require 'disbatch/error'
|
19
21
|
require 'disbatch/node'
|
20
22
|
require 'disbatch/queue'
|
21
23
|
require 'disbatch/plugin'
|
@@ -33,7 +35,11 @@ module Disbatch
|
|
33
35
|
|
34
36
|
# Return and cache the local node object
|
35
37
|
def node
|
36
|
-
|
38
|
+
begin
|
39
|
+
@node ||= Disbatch::Node.get(@node_id)
|
40
|
+
rescue Disbatch::NoNodeError
|
41
|
+
Disbatch::Node.create(@node_id)
|
42
|
+
end
|
37
43
|
end
|
38
44
|
|
39
45
|
extend self
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: disbatch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.9
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Matthew Berg
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date:
|
13
|
+
date: 2012-01-17 00:00:00 -05:00
|
14
14
|
default_executable:
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
@@ -85,10 +85,10 @@ files:
|
|
85
85
|
- lib/disbatch.rb
|
86
86
|
- lib/disbatch/queue.rb
|
87
87
|
- lib/disbatch/node.rb
|
88
|
-
- lib/disbatch/
|
88
|
+
- lib/disbatch/error.rb
|
89
|
+
- lib/disbatch/task.rb
|
89
90
|
- lib/disbatch/plugin.rb
|
90
91
|
- lib/disbatch/plugin/hello.rb
|
91
|
-
- lib/disbatch/queue/task.rb
|
92
92
|
- doc/class_list.html
|
93
93
|
- doc/frames.html
|
94
94
|
- doc/_index.html
|