manband 0.6.1 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README +3 -0
- data/bin/env.sh +7 -0
- data/bin/jobhandler.rb +8 -3
- data/bin/workflowhandler.rb +10 -5
- data/lib/manband/bandmanager.rb +2 -1
- data/lib/manband/flowconfig.rb +7 -0
- data/lib/manband/job.rb +33 -3
- data/lib/manband/store.rb +1 -1
- data/lib/manband/util.rb +7 -5
- data/lib/manband/workflow.rb +6 -2
- metadata +6 -5
data/README
CHANGED
@@ -108,6 +108,9 @@ Webband may use this gem, or use current files with same hierarchy
|
|
108
108
|
|
109
109
|
## History
|
110
110
|
|
111
|
+
0.7:
|
112
|
+
Add IF management
|
113
|
+
Fix some message dispatching issues
|
111
114
|
0.6.1: Increase file path max size
|
112
115
|
0.6.0: Remove mb-minion dependency. New bunny gem is incompatible now, code has been rewriten to remove the dependency and use only amqp.
|
113
116
|
|
data/bin/env.sh
ADDED
data/bin/jobhandler.rb
CHANGED
@@ -28,7 +28,7 @@ require 'manband/store.rb'
|
|
28
28
|
handler = Time.now.to_s
|
29
29
|
|
30
30
|
log = Logger.new(STDOUT)
|
31
|
-
log.level = Logger::
|
31
|
+
log.level = Logger::DEBUG
|
32
32
|
|
33
33
|
options = {}
|
34
34
|
|
@@ -116,7 +116,8 @@ EventMachine.run do
|
|
116
116
|
exchange = channel.direct()
|
117
117
|
queue = channel.queue('manband.node'+options[:queue], :auto_delete => false, :durable => true)
|
118
118
|
queue.bind(exchange).subscribe(:ack => true) do |metadata, message|
|
119
|
-
|
119
|
+
log.debug "Received msg: "+message.to_s
|
120
|
+
doack = true
|
120
121
|
args = JSON.parse(message)
|
121
122
|
msg = JSON.parse(args["msg"])
|
122
123
|
if args["operation"] == OP_DESTROY
|
@@ -137,6 +138,8 @@ EventMachine.run do
|
|
137
138
|
log.error "Error: job does not exists "+msg["id"]
|
138
139
|
else
|
139
140
|
if args["operation"] == OP_RUN
|
141
|
+
myjob.setAck(channel,metadata.delivery_tag)
|
142
|
+
doack = false
|
140
143
|
if msg["instance"]!=nil
|
141
144
|
myjob.run(handler, msg["instance"].to_i)
|
142
145
|
else
|
@@ -153,7 +156,9 @@ EventMachine.run do
|
|
153
156
|
end
|
154
157
|
|
155
158
|
|
156
|
-
|
159
|
+
if doack
|
160
|
+
channel.acknowledge(metadata.delivery_tag, false)
|
161
|
+
end
|
157
162
|
end
|
158
163
|
|
159
164
|
end
|
data/bin/workflowhandler.rb
CHANGED
@@ -30,7 +30,7 @@ require 'manband/job.rb'
|
|
30
30
|
require 'manband/bandmanager.rb'
|
31
31
|
|
32
32
|
log = Logger.new(STDOUT)
|
33
|
-
log.level = Logger::
|
33
|
+
log.level = Logger::DEBUG
|
34
34
|
|
35
35
|
|
36
36
|
@@options = {}
|
@@ -108,6 +108,7 @@ EventMachine.run do
|
|
108
108
|
queue.bind(exchange).subscribe(:ack => true) do |metadata, message|
|
109
109
|
|
110
110
|
begin
|
111
|
+
log.debug "Received msg: "+message.to_s
|
111
112
|
args = JSON.parse(message)
|
112
113
|
msg = JSON.parse(args["msg"])
|
113
114
|
rescue Exception => e
|
@@ -130,12 +131,12 @@ EventMachine.run do
|
|
130
131
|
end
|
131
132
|
if args["operation"] == OP_SKIP
|
132
133
|
savemessage(msg["id"],args["operation"],args["msg"])
|
133
|
-
workflow = WorkFlow.get(msg["
|
134
|
+
workflow = WorkFlow.get(msg["workflow"])
|
134
135
|
if workflow == nil
|
135
|
-
log.error "Error: workflow does not exists "+msg["
|
136
|
+
log.error "Error: workflow does not exists "+msg["workflow"]
|
136
137
|
else
|
137
138
|
workflow.update(:status => STATUS_RUNNING)
|
138
|
-
myjob = Job.get(msg["
|
139
|
+
myjob = Job.get(msg["id"])
|
139
140
|
myjob.runnext(true)
|
140
141
|
end
|
141
142
|
end
|
@@ -156,15 +157,19 @@ EventMachine.run do
|
|
156
157
|
if args["operation"] == OP_FINISH
|
157
158
|
savemessage(msg["workflow"],args["operation"],args["msg"])
|
158
159
|
myjob = Job.get(msg["id"])
|
160
|
+
doSkip = false
|
159
161
|
if myjob == nil
|
160
162
|
log.error "Error: job does not exists "+msg["id"]
|
161
163
|
else
|
164
|
+
if myjob.status == STATUS_SKIP
|
165
|
+
doSkip = true
|
166
|
+
end
|
162
167
|
if msg["handler"]!=nil && myjob.handler != msg["handler"]
|
163
168
|
log.debug "Finish received from a different handler "+myjob.handler+" vs "+msg["handler"]+", skipping message"
|
164
169
|
else
|
165
170
|
myjob.finish
|
166
171
|
if myjob.isover?
|
167
|
-
myjob.runnext
|
172
|
+
myjob.runnext(doSkip)
|
168
173
|
end
|
169
174
|
end
|
170
175
|
end
|
data/lib/manband/bandmanager.rb
CHANGED
@@ -134,9 +134,10 @@ class BandManager
|
|
134
134
|
# Request workflow management
|
135
135
|
msg = '{ "id" : "'+workflow.id.to_s+'", "root" : "'+rootjob.id.to_s+'"}'
|
136
136
|
if debug
|
137
|
+
msg = '{ "id" : "'+rootjob.id.to_s+'","workflow" : "'+workflow.id.to_s+'", "node" : "'+rootjob.id.to_s+'"}'
|
137
138
|
Utils.publish("manband.master", { "operation" => OP_SKIP, "msg" => msg })
|
138
139
|
else
|
139
|
-
|
140
|
+
Utils.publish("manband.master", { "operation" => OP_START, "msg" => msg })
|
140
141
|
end
|
141
142
|
end
|
142
143
|
end
|
data/lib/manband/flowconfig.rb
CHANGED
@@ -35,6 +35,13 @@ OP_CLEAN = "clean" # Delete work dirs of workflow
|
|
35
35
|
OP_DESTROY = "destroy" # Delete workflow and its work dirs
|
36
36
|
OP_STORE = "store" # Store result to S3
|
37
37
|
|
38
|
+
# Basic actor
|
39
|
+
ACTOR = -1
|
40
|
+
# If actor, i.e. ony one of the following jobs will be executed. Selected job match the exit code of the command. Type of the node will be set to selected output
|
41
|
+
# WARNING: input of IF actors MUST NOT be a pattern matchng multiple files. The IF condition must be executed only once, not on a bunch of files.
|
42
|
+
IF_ACTOR = -2
|
43
|
+
|
44
|
+
|
38
45
|
# This class holds the base config
|
39
46
|
class FlowConfig
|
40
47
|
@@workdir='/tmp'
|
data/lib/manband/job.rb
CHANGED
@@ -34,7 +34,19 @@ class Job
|
|
34
34
|
end
|
35
35
|
|
36
36
|
@@log = Logger.new(STDOUT)
|
37
|
-
@@log.level = Logger::
|
37
|
+
@@log.level = Logger::DEBUG
|
38
|
+
|
39
|
+
@channel = nil
|
40
|
+
@delivery_tag = nil
|
41
|
+
|
42
|
+
def setAck(channel,tag)
|
43
|
+
@channel = channel
|
44
|
+
@tag = tag
|
45
|
+
end
|
46
|
+
|
47
|
+
def ack()
|
48
|
+
@channel.acknowledge(@tag, false)
|
49
|
+
end
|
38
50
|
|
39
51
|
property :id, Serial
|
40
52
|
property :wid, Integer # id of the worflow
|
@@ -48,6 +60,7 @@ class Job
|
|
48
60
|
property :queue, String, :default => "" # Queue to handle the job
|
49
61
|
property :error, Text, :default => "[]" # JSON Array of error, per instance
|
50
62
|
property :store, Integer, :default => STORE_NO # Storage status
|
63
|
+
property :type, Integer, :default => ACTOR # Type of the job
|
51
64
|
|
52
65
|
# Update job status and launch the command locally
|
53
66
|
# Sends a finish message or an error message according to the job status.
|
@@ -67,9 +80,17 @@ class Job
|
|
67
80
|
if instance>0
|
68
81
|
workdir = workdir + "/node" + instance.to_s
|
69
82
|
end
|
83
|
+
|
70
84
|
EventMachine.defer do
|
71
85
|
err = runcommand(workdir,instance)
|
72
|
-
|
86
|
+
# In the case of IF_ACTOR, exit code is not an error
|
87
|
+
# but the job to select. Reset error code after updating the type.
|
88
|
+
if @type == IF_ACTOR
|
89
|
+
@type = err + 1
|
90
|
+
curjob.update(:type => (err + 1))
|
91
|
+
err = 0
|
92
|
+
end
|
93
|
+
if err == nil || err > 0
|
73
94
|
# An error occured
|
74
95
|
jobmsg = '{ "workflow" : "'+@wid.to_s+'" , "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
|
75
96
|
sendmessage(OP_ERROR,jobmsg)
|
@@ -82,6 +103,7 @@ class Job
|
|
82
103
|
jobmsg = '{ "workflow" : "'+@wid.to_s+'" , "node" : "'+@node+'", "id" : "'+@id.to_s+'", "handler" : "'+curhandler.to_s+'", "instance" : "'+instance.to_s+'" }'
|
83
104
|
sendmessage(OP_FINISH,jobmsg)
|
84
105
|
end
|
106
|
+
ack()
|
85
107
|
end # End EventMachine
|
86
108
|
end
|
87
109
|
|
@@ -119,6 +141,7 @@ class Job
|
|
119
141
|
end
|
120
142
|
@@log.debug cmd
|
121
143
|
system(cmd)
|
144
|
+
return $?.exitstatus
|
122
145
|
end
|
123
146
|
|
124
147
|
# Change instance counter
|
@@ -245,12 +268,17 @@ class Job
|
|
245
268
|
workflow.isover?
|
246
269
|
return true
|
247
270
|
end
|
271
|
+
# Which next job is it?
|
272
|
+
# Start at 1 because 0 is a valid status code, consider if type from value 1
|
273
|
+
count = 1
|
274
|
+
# For each job to follow
|
248
275
|
nexts.each do |link|
|
249
276
|
job = Job.get(link.to)
|
250
277
|
#nexts.each do |nextnode|
|
251
278
|
#nextnode.strip!
|
252
279
|
#queue = workflow.getnodequeue(nextnode)
|
253
|
-
|
280
|
+
# Explicit skip, or IF actor, we skip all actors but one
|
281
|
+
if skip==true || @status == STATUS_SKIP || (@type>0 && count!= @type)
|
254
282
|
jobstatus = STATUS_SKIP
|
255
283
|
operation = OP_SKIP
|
256
284
|
commands = [ "skipcommand"]
|
@@ -259,6 +287,7 @@ class Job
|
|
259
287
|
operation = OP_RUN
|
260
288
|
commands = workflow.getnodecommand(job.node)
|
261
289
|
end
|
290
|
+
count=count+1
|
262
291
|
if commands == nil
|
263
292
|
@@log.error "Could not get command for node "+job.id.to_s
|
264
293
|
job.update(:command => "", :status => STATUS_ERROR, :instances => 0, :maxinstances => 0, :workdir => FlowConfig.getjobdir(workflow.workdir))
|
@@ -293,6 +322,7 @@ class Job
|
|
293
322
|
queue = "manband.node"+jobqueue
|
294
323
|
end
|
295
324
|
if queue != nil
|
325
|
+
@@log.debug "Send #"+operation+","+msg
|
296
326
|
Utils.enqueue(queue, { "operation" => operation, "msg" => msg })
|
297
327
|
logMessage(operation,msg)
|
298
328
|
end
|
data/lib/manband/store.rb
CHANGED
data/lib/manband/util.rb
CHANGED
@@ -26,7 +26,8 @@ class Utils
|
|
26
26
|
msg = data.to_json
|
27
27
|
queue = @@channel.queue(dest, :auto_delete => false, :durable => true)
|
28
28
|
queue.bind(@@exchange)
|
29
|
-
|
29
|
+
#@@exchange.publish(msg, :routing_key => dest)
|
30
|
+
@@channel.direct("").publish(msg, :routing_key => dest)
|
30
31
|
end
|
31
32
|
|
32
33
|
# Creates a context and sends a message
|
@@ -38,10 +39,11 @@ class Utils
|
|
38
39
|
EventMachine.run do
|
39
40
|
connection = AMQP.connect(FlowConfig.config())
|
40
41
|
channel = AMQP::Channel.new(connection)
|
41
|
-
exchange = channel.direct()
|
42
|
-
queue = channel.queue(dest, :auto_delete => false, :durable => true)
|
43
|
-
queue.bind(exchange)
|
44
|
-
exchange.publish(msg) do
|
42
|
+
#exchange = channel.direct()
|
43
|
+
#queue = channel.queue(dest, :auto_delete => false, :durable => true)
|
44
|
+
#queue.bind(exchange)
|
45
|
+
#exchange.publish(msg, :routing_key => dest) do
|
46
|
+
channel.direct("").publish(msg, :routing_key => dest) do
|
45
47
|
connection.close { EventMachine.stop }
|
46
48
|
end
|
47
49
|
end
|
data/lib/manband/workflow.rb
CHANGED
@@ -19,7 +19,7 @@ class WorkFlow
|
|
19
19
|
include DataMapper::Resource
|
20
20
|
|
21
21
|
@@log = Logger.new(STDOUT)
|
22
|
-
@@log.level = Logger::
|
22
|
+
@@log.level = Logger::DEBUG
|
23
23
|
|
24
24
|
property :id, Serial # An auto-increment integer key
|
25
25
|
property :uid, Text # user id
|
@@ -109,11 +109,15 @@ class WorkFlow
|
|
109
109
|
@@log.debug "Add store option for job "+job
|
110
110
|
store = STORE_DO
|
111
111
|
end
|
112
|
+
type = ACTOR
|
113
|
+
if fworkflow["workflow"][job]["type"]=='if'
|
114
|
+
type = IF_ACTOR
|
115
|
+
end
|
112
116
|
workdir = FlowConfig.getjobdir(@workdir)
|
113
117
|
if curnode == "root"
|
114
118
|
workdir = self.workdir + "/root";
|
115
119
|
end
|
116
|
-
newjob = Job.new(:wid => @id, :node => job, :command => "", :status => status, :instances => 0, :maxinstances => 0, :queue => queue, :workdir => FlowConfig.getjobdir(@workdir), :store => store)
|
120
|
+
newjob = Job.new(:wid => @id, :node => job, :command => "", :status => status, :instances => 0, :maxinstances => 0, :queue => queue, :workdir => FlowConfig.getjobdir(@workdir), :store => store, :type => type)
|
117
121
|
newjob.save
|
118
122
|
if id != nil
|
119
123
|
@@log.debug "Add link "+id.to_s+"->"+newjob.id.to_s+","+newjob.node
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manband
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 3
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 7
|
9
|
+
- 0
|
10
|
+
version: 0.7.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Olivier Sallou
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-07-
|
18
|
+
date: 2012-07-18 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
name: amqp
|
@@ -125,6 +125,7 @@ extra_rdoc_files:
|
|
125
125
|
- README
|
126
126
|
- LICENSE
|
127
127
|
files:
|
128
|
+
- bin/env.sh
|
128
129
|
- bin/manband.rb
|
129
130
|
- bin/remoteband.rb
|
130
131
|
- bin/jobhandler.rb
|