rbbt-util 5.19.35 → 5.19.36
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/etc/app.d/base.rb +4 -0
- data/etc/app.d/init.rb +1 -0
- data/lib/rbbt/annotations.rb +1 -0
- data/lib/rbbt/rest/client/get.rb +6 -1
- data/lib/rbbt/rest/client/step.rb +30 -8
- data/lib/rbbt/tsv/accessor.rb +2 -1
- data/lib/rbbt/tsv/util.rb +10 -6
- data/lib/rbbt/util/R/plot.rb +10 -2
- data/lib/rbbt/util/concurrency/processes/worker.rb +1 -1
- data/lib/rbbt/util/log.rb +1 -1
- data/lib/rbbt/util/log/progress/report.rb +1 -1
- data/lib/rbbt/util/misc/exceptions.rb +2 -0
- data/lib/rbbt/util/misc/inspect.rb +1 -1
- data/lib/rbbt/util/misc/multipart_payload.rb +138 -0
- data/lib/rbbt/util/misc/pipes.rb +2 -1
- data/lib/rbbt/workflow/accessor.rb +36 -5
- data/lib/rbbt/workflow/definition.rb +1 -1
- data/lib/rbbt/workflow/step.rb +7 -4
- data/lib/rbbt/workflow/step/run.rb +96 -31
- data/share/rbbt_commands/app/start +20 -5
- data/share/rbbt_commands/workflow/server +57 -60
- data/share/rbbt_commands/workflow/task +2 -2
- data/test/rbbt/util/test_misc.rb +4 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72a73b569032a1fce45e26a98b2e5e57c1abd83c
|
4
|
+
data.tar.gz: 1bb8e14d02dbc52654120389c6c1ff303ec25d05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6126957c9cb71705cfa47105941752e32033b714e7972dac3b211179c0e23c3c7e7a7d693922f0c7bc141e9c4955b15936ebd1418daa80150fdab5ef950c07cb
|
7
|
+
data.tar.gz: 263a74be39a0ed6e0fee8d2e3c913f77767744fc54b4c0e7d5c469013ef1bd48cf134b5efbcb1c00aacd036a174a552ec2260a64f75156744f5bed1324a9089a
|
data/etc/app.d/base.rb
CHANGED
@@ -1,9 +1,13 @@
|
|
1
1
|
#{{{ MODULES AND HELPERS
|
2
|
+
register Sinatra::MultiRoute
|
2
3
|
|
3
4
|
register Sinatra::RbbtRESTMain
|
4
5
|
register Sinatra::RbbtRESTEntity
|
5
6
|
register Sinatra::RbbtRESTFileServer # Remove to prevent serving files
|
6
7
|
register Sinatra::RbbtRESTKnowledgeBase
|
8
|
+
register Sinatra::RbbtRESTWorkflow
|
9
|
+
|
10
|
+
helpers Sinatra::RbbtToolHelper
|
7
11
|
helpers Sinatra::RbbtMiscHelpers
|
8
12
|
|
9
13
|
#{{{ SESSIONS
|
data/etc/app.d/init.rb
CHANGED
data/lib/rbbt/annotations.rb
CHANGED
@@ -230,6 +230,7 @@ module Annotation
|
|
230
230
|
end
|
231
231
|
|
232
232
|
def clean_and_setup_hash(object, hash)
|
233
|
+
object.instance_variable_set(:@annotation_values, nil) if object.instance_variable_get(:@annotation_values).nil?
|
233
234
|
annotation_values = object.instance_variable_get(:@annotation_values)
|
234
235
|
annotation_values = annotation_values.nil? ? {} : annotation_values.dup
|
235
236
|
annotation_values.instance_variable_set(:@annotation_md5, nil)
|
data/lib/rbbt/rest/client/get.rb
CHANGED
@@ -83,13 +83,18 @@ class WorkflowRESTClient
|
|
83
83
|
end
|
84
84
|
|
85
85
|
def self.post_jobname(url, params = {})
|
86
|
+
Log.stack caller
|
86
87
|
Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
|
87
88
|
params = params.merge({ :_format => 'jobname' })
|
88
89
|
params = fix_params params
|
89
90
|
|
90
|
-
capture_exception do
|
91
|
+
name = capture_exception do
|
91
92
|
RestClient.post(URI.encode(url), params)
|
92
93
|
end
|
94
|
+
|
95
|
+
Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}: #{name}" }
|
96
|
+
|
97
|
+
name
|
93
98
|
end
|
94
99
|
|
95
100
|
def self.post_json(url, params = {})
|
@@ -1,30 +1,36 @@
|
|
1
|
-
require 'excon'
|
2
1
|
class WorkflowRESTClient
|
3
2
|
class RemoteStep < Step
|
4
3
|
|
5
4
|
attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec
|
6
5
|
|
7
6
|
def self.get_streams(inputs)
|
7
|
+
new_inputs = {}
|
8
8
|
inputs.each do |k,v|
|
9
|
-
if Step === v
|
10
|
-
|
11
|
-
|
9
|
+
if Step === v or RemoteStep === v
|
10
|
+
v.run
|
11
|
+
new_inputs[k] = v.load
|
12
|
+
else
|
13
|
+
new_inputs[k] = v
|
12
14
|
end
|
13
15
|
end
|
16
|
+
new_inputs
|
14
17
|
end
|
15
18
|
|
19
|
+
|
16
20
|
def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false)
|
17
21
|
@base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
|
18
22
|
@mutex = Mutex.new
|
19
|
-
RemoteStep.get_streams @inputs
|
23
|
+
@inputs = RemoteStep.get_streams @inputs
|
20
24
|
end
|
21
25
|
|
22
26
|
def name
|
23
27
|
return nil if @is_exec
|
28
|
+
return @path if @url.nil?
|
24
29
|
(Array === @url ? @url.first : @url).split("/").last
|
25
30
|
end
|
26
31
|
|
27
32
|
def task_name
|
33
|
+
return task unless @url
|
28
34
|
init_job
|
29
35
|
(Array === @url ? @url.first : @url).split("/")[-2]
|
30
36
|
end
|
@@ -41,6 +47,7 @@ class WorkflowRESTClient
|
|
41
47
|
end
|
42
48
|
|
43
49
|
def status
|
50
|
+
return nil if @url.nil?
|
44
51
|
begin
|
45
52
|
info[:status]
|
46
53
|
ensure
|
@@ -82,8 +89,11 @@ class WorkflowRESTClient
|
|
82
89
|
end
|
83
90
|
|
84
91
|
def path
|
85
|
-
|
86
|
-
|
92
|
+
if @url
|
93
|
+
@url + '?_format=raw'
|
94
|
+
else
|
95
|
+
[base_url, task, Misc.fingerprint(inputs)] * "/"
|
96
|
+
end
|
87
97
|
end
|
88
98
|
|
89
99
|
def run(noload = false)
|
@@ -99,6 +109,7 @@ class WorkflowRESTClient
|
|
99
109
|
end
|
100
110
|
end
|
101
111
|
end
|
112
|
+
|
102
113
|
return @result if noload == :stream
|
103
114
|
noload ? path + '?_format=raw' : @result
|
104
115
|
end
|
@@ -186,7 +197,18 @@ class WorkflowRESTClient
|
|
186
197
|
RestClient::Request.execute(:method => :post, :url => url, :payload => task_params, :block_response => bl)
|
187
198
|
end
|
188
199
|
|
189
|
-
Zlib::GzipReader.new(sout)
|
200
|
+
reader = Zlib::GzipReader.new(sout)
|
201
|
+
Misc.open_pipe do |sin|
|
202
|
+
while c = reader.read(1015)
|
203
|
+
sin.write c
|
204
|
+
end
|
205
|
+
sin.close
|
206
|
+
@done = true
|
207
|
+
end
|
208
|
+
#nsout, nsin = Misc.pipe
|
209
|
+
#Misc.consume_stream(reader, true, nsin, true) do @done = true end
|
210
|
+
#iii :ret
|
211
|
+
#nsout
|
190
212
|
end
|
191
213
|
end
|
192
214
|
|
data/lib/rbbt/tsv/accessor.rb
CHANGED
@@ -19,6 +19,7 @@ module TSV
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def entity_options
|
22
|
+
@entity_options ||= nil
|
22
23
|
if @entity_options.nil?
|
23
24
|
@entity_options = namespace ? {:namespace => namespace, :organism => namespace} : {}
|
24
25
|
@entity_templates = nil
|
@@ -122,7 +123,7 @@ module TSV
|
|
122
123
|
end
|
123
124
|
|
124
125
|
def write?
|
125
|
-
@writable
|
126
|
+
@writable ||= false
|
126
127
|
end
|
127
128
|
|
128
129
|
def self._extended(data)
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -133,13 +133,17 @@ module TSV
|
|
133
133
|
StringIO.new file
|
134
134
|
end
|
135
135
|
when (defined? Step and Step)
|
136
|
-
file.
|
137
|
-
|
138
|
-
if stream
|
139
|
-
stream
|
136
|
+
if file.respond_to?(:base_url) and file.result
|
137
|
+
file.result
|
140
138
|
else
|
141
|
-
file.
|
142
|
-
|
139
|
+
file.grace
|
140
|
+
stream = file.get_stream
|
141
|
+
if stream
|
142
|
+
stream
|
143
|
+
else
|
144
|
+
file.join
|
145
|
+
get_stream(file.path)
|
146
|
+
end
|
143
147
|
end
|
144
148
|
when Array
|
145
149
|
Misc.open_pipe do |sin|
|
data/lib/rbbt/util/R/plot.rb
CHANGED
@@ -10,6 +10,14 @@ module R
|
|
10
10
|
sources = [:plot, Rbbt.share.Rlib["svg.R"].find(:lib), options[:source]].flatten.compact
|
11
11
|
options.delete :source
|
12
12
|
|
13
|
+
fast = options[:fast]
|
14
|
+
|
15
|
+
if fast
|
16
|
+
save_method = "rbbt.SVG.save.fast"
|
17
|
+
else
|
18
|
+
save_method = "rbbt.SVG.save"
|
19
|
+
end
|
20
|
+
|
13
21
|
if data
|
14
22
|
data.each do |k,v|
|
15
23
|
v = Array === v ? v : [v]
|
@@ -48,7 +56,7 @@ module R
|
|
48
56
|
data.R <<-EOF, sources, options
|
49
57
|
plot = { #{script} }
|
50
58
|
|
51
|
-
|
59
|
+
#{save_method}('#{tmpfile}', plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
|
52
60
|
data = NULL
|
53
61
|
EOF
|
54
62
|
|
@@ -61,7 +69,7 @@ module R
|
|
61
69
|
R.run <<-EOF, sources, options
|
62
70
|
plot = { #{script} }
|
63
71
|
|
64
|
-
|
72
|
+
#{save_method}('#{tmpfile}', plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
|
65
73
|
data = NULL
|
66
74
|
EOF
|
67
75
|
Open.read(tmpfile).gsub(/(glyph\d+-\d+)/, '\1-' + File.basename(tmpfile))
|
@@ -70,7 +70,7 @@ class RbbtProcessQueue
|
|
70
70
|
|
71
71
|
initial = Misc.memory_use(Process.pid)
|
72
72
|
memory_cap = multiplier * initial
|
73
|
-
Log.medium "Worker #{Process.pid} started with #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
|
73
|
+
Log.medium "Worker for #{Process.pid} started with pid #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
|
74
74
|
|
75
75
|
@monitor_thread = Thread.new do
|
76
76
|
begin
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -104,7 +104,7 @@ module Log
|
|
104
104
|
end
|
105
105
|
|
106
106
|
def save
|
107
|
-
info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max}
|
107
|
+
info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
|
108
108
|
Open.write(@file, info.to_yaml)
|
109
109
|
end
|
110
110
|
|
@@ -0,0 +1,138 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'rbbt-util'
|
3
|
+
|
4
|
+
class Net::HTTPGenericRequest
|
5
|
+
alias old_send_request_with_body_stream send_request_with_body_stream
|
6
|
+
|
7
|
+
def send_request_with_body_stream(*args)
|
8
|
+
if chunked?
|
9
|
+
Thread.new do
|
10
|
+
old_send_request_with_body_stream(*args)
|
11
|
+
end
|
12
|
+
else
|
13
|
+
old_send_request_with_body_stream(*args)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module RbbtMutiplartPayload
|
19
|
+
BOUNDARY = "Rbbt_Param_Stream"
|
20
|
+
EOL = "\r\n"
|
21
|
+
|
22
|
+
def self.input_header(name, filename = nil)
|
23
|
+
|
24
|
+
if filename
|
25
|
+
head_text = 'Content-Disposition: form-data; name="' + name + '"; filename="' + filename + '"'
|
26
|
+
else
|
27
|
+
head_text = 'Content-Disposition: form-data; name="' + name + '"'
|
28
|
+
end
|
29
|
+
|
30
|
+
content_transfer_text = "Content-Transfer-Encoding: binary"
|
31
|
+
|
32
|
+
content_type_text = 'Content-Type: text/plain'
|
33
|
+
|
34
|
+
head_text + EOL + content_transfer_text + EOL + content_type_text + EOL
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.add_input(name, content, filename = nil)
|
38
|
+
header = input_header(name, filename)
|
39
|
+
"--" + BOUNDARY + EOL + header + EOL + content + EOL
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.add_stream(io, name, content, filename = nil)
|
43
|
+
header = input_header(name, filename)
|
44
|
+
io.write "--" + BOUNDARY + EOL + header + EOL + EOL
|
45
|
+
|
46
|
+
while c = content.read(1024)
|
47
|
+
io.write c
|
48
|
+
end
|
49
|
+
content.close
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.close_stream(io)
|
53
|
+
io.write "--" + BOUNDARY + "--" + EOL + EOL
|
54
|
+
io.write EOL
|
55
|
+
io.close
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.post_data_stream(inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
|
59
|
+
sout, sin = Misc.pipe
|
60
|
+
|
61
|
+
Thread.new do
|
62
|
+
inputs.each do |input,content|
|
63
|
+
input = input.to_s
|
64
|
+
next if stream_input and input == stream_input.to_s
|
65
|
+
content_str = case content
|
66
|
+
when String
|
67
|
+
if Misc.is_filename?(content) and File.exists?(content)
|
68
|
+
File.read(content)
|
69
|
+
else
|
70
|
+
content
|
71
|
+
end
|
72
|
+
when File, IO
|
73
|
+
content.read
|
74
|
+
when nil
|
75
|
+
"nil"
|
76
|
+
else
|
77
|
+
content.to_s
|
78
|
+
end
|
79
|
+
str = RbbtMutiplartPayload.add_input(input, content_str)
|
80
|
+
sin.write str
|
81
|
+
end
|
82
|
+
|
83
|
+
RbbtMutiplartPayload.add_stream(sin, stream_input.to_s, stream_io, stream_filename) if stream_input
|
84
|
+
RbbtMutiplartPayload.close_stream(sin)
|
85
|
+
end
|
86
|
+
|
87
|
+
sout
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.issue(url, inputs = nil, stream_input = nil, stream_io = nil, stream_filename = nil)
|
91
|
+
|
92
|
+
uri = URI(url)
|
93
|
+
req = Net::HTTP::Post.new(uri.path)
|
94
|
+
|
95
|
+
IndiferentHash.setup(inputs)
|
96
|
+
|
97
|
+
if stream_input
|
98
|
+
stream_io ||= TSV.get_stream inputs[stream_input]
|
99
|
+
stream_filename ||= case inputs[stream_input]
|
100
|
+
when String
|
101
|
+
inputs[stream_input]
|
102
|
+
when File
|
103
|
+
inputs[stream_input].filename
|
104
|
+
else
|
105
|
+
'file'
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
sout = RbbtMutiplartPayload.post_data_stream inputs, stream_input, stream_io, stream_filename
|
110
|
+
|
111
|
+
if stream_input
|
112
|
+
req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY + '; stream=' + stream_input.to_s
|
113
|
+
req.body_stream = sout
|
114
|
+
req.add_field "Transfer-Encoding", "chunked"
|
115
|
+
else
|
116
|
+
req.content_type = "multipart/form-data; boundary=" + RbbtMutiplartPayload::BOUNDARY
|
117
|
+
req.body = sout.read
|
118
|
+
end
|
119
|
+
|
120
|
+
Misc.open_pipe do |sin|
|
121
|
+
Net::HTTP.start(uri.hostname, uri.port) do |http|
|
122
|
+
http.request(req) do |res|
|
123
|
+
if Net::HTTPRedirection === res
|
124
|
+
sin.write res["location"]
|
125
|
+
elsif stream_input
|
126
|
+
res.read_body do |c|
|
127
|
+
sin.write c
|
128
|
+
end
|
129
|
+
else
|
130
|
+
sin.write res.body
|
131
|
+
end
|
132
|
+
sin.close
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -159,7 +159,7 @@ module Misc
|
|
159
159
|
str
|
160
160
|
end
|
161
161
|
|
162
|
-
def self.consume_stream(io, in_thread = false, into = nil, into_close = true)
|
162
|
+
def self.consume_stream(io, in_thread = false, into = nil, into_close = true, &block)
|
163
163
|
return if Path === io
|
164
164
|
return unless io.respond_to? :read
|
165
165
|
if io.respond_to? :closed? and io.closed?
|
@@ -189,6 +189,7 @@ module Misc
|
|
189
189
|
io.close unless io.closed?
|
190
190
|
into.close if into and into_close and not into.closed?
|
191
191
|
into.join if into and into_close and into.respond_to?(:joined?) and not into.joined?
|
192
|
+
block.call if block_given?
|
192
193
|
rescue Aborted
|
193
194
|
Log.medium "Consume stream aborted #{Misc.fingerprint io}"
|
194
195
|
io.abort if io.respond_to? :abort
|
@@ -38,6 +38,10 @@ class Step
|
|
38
38
|
path.nil? ? nil : path + '.info'
|
39
39
|
end
|
40
40
|
|
41
|
+
def self.pid_file(path)
|
42
|
+
path.nil? ? nil : path + '.pid'
|
43
|
+
end
|
44
|
+
|
41
45
|
def self.step_info(path)
|
42
46
|
begin
|
43
47
|
Open.open(info_file(path)) do |f|
|
@@ -61,6 +65,10 @@ class Step
|
|
61
65
|
path.sub(/.*\/#{Regexp.quote task.name.to_s}\/(.*)/, '\1')
|
62
66
|
end
|
63
67
|
|
68
|
+
def short_path
|
69
|
+
[task_name, name] * "/"
|
70
|
+
end
|
71
|
+
|
64
72
|
def task_name
|
65
73
|
@task_name ||= task.name
|
66
74
|
end
|
@@ -71,6 +79,10 @@ class Step
|
|
71
79
|
@info_file ||= Step.info_file(path)
|
72
80
|
end
|
73
81
|
|
82
|
+
def pid_file
|
83
|
+
@pid_file ||= Step.pid_file(path)
|
84
|
+
end
|
85
|
+
|
74
86
|
def info_lock
|
75
87
|
@info_lock = begin
|
76
88
|
path = Persist.persistence_path(info_file + '.lock', {:dir => Step.lock_dir})
|
@@ -118,6 +130,16 @@ class Step
|
|
118
130
|
end
|
119
131
|
end
|
120
132
|
|
133
|
+
def init_info
|
134
|
+
return nil if @exec or info_file.nil?
|
135
|
+
Open.lock(info_file, :lock => info_lock) do
|
136
|
+
i = {:status => :init}
|
137
|
+
@info_cache = i
|
138
|
+
Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
|
139
|
+
@info_cache_time = Time.now
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
121
143
|
def set_info(key, value)
|
122
144
|
return nil if @exec or info_file.nil?
|
123
145
|
value = Annotated.purge value if defined? Annotated
|
@@ -286,7 +308,7 @@ class Step
|
|
286
308
|
end
|
287
309
|
|
288
310
|
def started?
|
289
|
-
Open.exists?
|
311
|
+
Open.exists?(path) or Open.exists?(pid_file)
|
290
312
|
end
|
291
313
|
|
292
314
|
def dirty?
|
@@ -302,7 +324,7 @@ class Step
|
|
302
324
|
end
|
303
325
|
|
304
326
|
def running?
|
305
|
-
return nil if not Open.exists?
|
327
|
+
return nil if not Open.exists? pid_file
|
306
328
|
return nil if info[:pid].nil?
|
307
329
|
|
308
330
|
pid = @pid || info[:pid]
|
@@ -468,13 +490,22 @@ module Workflow
|
|
468
490
|
deps.each do |dep|
|
469
491
|
case dep
|
470
492
|
when Array
|
471
|
-
wf, t = dep
|
493
|
+
wf, t, o = dep
|
472
494
|
wf.rec_dependencies(t).each do |d|
|
473
495
|
if Array === d
|
474
|
-
|
496
|
+
new = d
|
475
497
|
else
|
476
|
-
|
498
|
+
new = [dep.first, d]
|
499
|
+
end
|
500
|
+
if Hash === o and not o.empty?
|
501
|
+
if Hash === new.last
|
502
|
+
hash = new.last
|
503
|
+
o.each{|k,v| hash[k] ||= v}
|
504
|
+
else
|
505
|
+
new.push o
|
506
|
+
end
|
477
507
|
end
|
508
|
+
all_deps << new
|
478
509
|
end
|
479
510
|
when String, Symbol
|
480
511
|
all_deps.concat rec_dependencies(dep.to_sym)
|
@@ -59,7 +59,7 @@ module Workflow
|
|
59
59
|
(defined? WorkflowRESTClient and WorkflowRESTClient === dependency.first) or
|
60
60
|
Hash === dependency.last
|
61
61
|
|
62
|
-
dependency = ([self] + dependency) unless Module === dependency.first
|
62
|
+
dependency = ([self] + dependency) unless Module === dependency.first or (defined? WorkflowRESTClient and WorkflowRESTClient === dependency.first)
|
63
63
|
@dependencies << dependency
|
64
64
|
else
|
65
65
|
@dependencies.concat dependency
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -201,8 +201,9 @@ class Step
|
|
201
201
|
|
202
202
|
def self.clean(path)
|
203
203
|
info_file = Step.info_file path
|
204
|
+
pid_file = Step.pid_file path
|
204
205
|
files_dir = Step.files_dir path
|
205
|
-
if Open.exists?(path) or Open.exists?(
|
206
|
+
if Open.exists?(path) or Open.exists?(pid_file)
|
206
207
|
begin
|
207
208
|
self.abort if self.running?
|
208
209
|
rescue Exception
|
@@ -213,10 +214,11 @@ class Step
|
|
213
214
|
|
214
215
|
Misc.insist do
|
215
216
|
Open.rm info_file if Open.exists? info_file
|
216
|
-
Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
|
217
|
+
#Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
|
217
218
|
Open.rm path if Open.exists? path
|
218
|
-
Open.rm path + '.lock' if Open.exists? path + '.lock'
|
219
|
+
#Open.rm path + '.lock' if Open.exists? path + '.lock'
|
219
220
|
Open.rm_rf files_dir if Open.exists? files_dir
|
221
|
+
Open.rm pid_file if Open.exists? pid_file
|
220
222
|
end
|
221
223
|
end
|
222
224
|
end
|
@@ -236,7 +238,8 @@ class Step
|
|
236
238
|
|
237
239
|
new_dependencies = []
|
238
240
|
dependencies.each{|step|
|
239
|
-
|
241
|
+
r = step.rec_dependencies
|
242
|
+
new_dependencies.concat r
|
240
243
|
new_dependencies << step
|
241
244
|
}
|
242
245
|
new_dependencies.uniq
|
@@ -143,7 +143,11 @@ class Step
|
|
143
143
|
next if (dependency.done? and not dependency.dirty?) or
|
144
144
|
(dependency.streaming? and dependency.running?) or
|
145
145
|
(defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency and not (dependency.error? or dependency.aborted?))
|
146
|
-
|
146
|
+
|
147
|
+
dependency.clean if dependency.aborted? or (dependency.started? and not dependency.running? and not dependency.error?)
|
148
|
+
|
149
|
+
raise DependencyError, [dependency.short_path, dependency.messages.last] * ": " if dependency.error?
|
150
|
+
|
147
151
|
dupping << dependency unless dependencies.include? dependency
|
148
152
|
end
|
149
153
|
|
@@ -154,30 +158,83 @@ class Step
|
|
154
158
|
|
155
159
|
begin
|
156
160
|
|
157
|
-
if
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
161
|
+
if dependency.done?
|
162
|
+
Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
|
163
|
+
next
|
164
|
+
end
|
165
|
+
|
166
|
+
if not dependency.started?
|
167
|
+
Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
|
168
|
+
dependency.run(:stream)
|
169
|
+
raise TryAgain
|
170
|
+
end
|
171
|
+
|
172
|
+
dependency.grace
|
173
|
+
|
174
|
+
if dependency.aborted?
|
175
|
+
dependency.clean
|
176
|
+
raise TryAgain
|
177
|
+
end
|
178
|
+
|
179
|
+
if dependency.error?
|
180
|
+
raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
|
181
|
+
end
|
182
|
+
|
183
|
+
if dependency.streaming?
|
184
|
+
Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
|
185
|
+
next
|
171
186
|
end
|
172
187
|
|
188
|
+
Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
|
189
|
+
begin
|
190
|
+
dependency.join
|
191
|
+
raise TryAgain unless dependency.done?
|
192
|
+
rescue Aborted
|
193
|
+
raise TryAgain
|
194
|
+
end
|
195
|
+
Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
|
196
|
+
|
197
|
+
#if not dependency.done?
|
198
|
+
# if dependency.started?
|
199
|
+
# dependency.grace
|
200
|
+
# if dependency.error?
|
201
|
+
# raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
|
202
|
+
# elsif dependency.streaming?
|
203
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
|
204
|
+
# else
|
205
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
|
206
|
+
# begin
|
207
|
+
# dependency.join unless dependency.streaming?
|
208
|
+
# rescue Aborted
|
209
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} #{Log.color :red, "aborted"} -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
|
210
|
+
# if dependency.aborted?
|
211
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} #{Log.color :red, "aborted cleaning"} -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
|
212
|
+
# dependency.clean
|
213
|
+
# raise TryAgain
|
214
|
+
# end
|
215
|
+
# end
|
216
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
|
217
|
+
# end
|
218
|
+
# else
|
219
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :blue, self.short_path}"
|
220
|
+
# dependency.run(:stream)
|
221
|
+
# dependency.grace
|
222
|
+
# dependency.join unless dependency.streaming?
|
223
|
+
# end
|
224
|
+
#else
|
225
|
+
# Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.short_path}"
|
226
|
+
#end
|
227
|
+
|
228
|
+
rescue TryAgain
|
229
|
+
retry
|
173
230
|
rescue Aborted
|
174
|
-
Log.error "Aborted dep. #{Log.color :red, dependency.
|
231
|
+
Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
|
175
232
|
raise $!
|
176
233
|
rescue Interrupt
|
177
|
-
Log.error "Interrupted while in dep. #{Log.color :red, dependency.
|
234
|
+
Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
|
178
235
|
raise $!
|
179
236
|
rescue Exception
|
180
|
-
Log.error "Exception in dep. #{ Log.color :red, dependency.
|
237
|
+
Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
|
181
238
|
raise $!
|
182
239
|
end
|
183
240
|
end
|
@@ -189,13 +246,14 @@ class Step
|
|
189
246
|
begin
|
190
247
|
@mutex.synchronize do
|
191
248
|
no_load = no_load ? :stream : false
|
192
|
-
result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do
|
249
|
+
result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do
|
193
250
|
if Step === Step.log_relay_step and not self == Step.log_relay_step
|
194
251
|
relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
|
195
252
|
end
|
196
|
-
@exec = false
|
197
253
|
|
198
|
-
|
254
|
+
@exec = false
|
255
|
+
Open.write(pid_file, Process.pid.to_s)
|
256
|
+
init_info
|
199
257
|
|
200
258
|
log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
|
201
259
|
|
@@ -204,16 +262,17 @@ class Step
|
|
204
262
|
:issued => (issue_time = Time.now),
|
205
263
|
:name => name,
|
206
264
|
:clean_name => clean_name,
|
207
|
-
:dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
|
208
265
|
})
|
209
266
|
|
210
267
|
dup_inputs
|
211
268
|
begin
|
212
269
|
run_dependencies
|
213
270
|
rescue Exception
|
271
|
+
FileUtils.rm pid_file if File.exists?(pid_file)
|
214
272
|
stop_dependencies
|
215
273
|
raise $!
|
216
274
|
end
|
275
|
+
set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
|
217
276
|
|
218
277
|
set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
|
219
278
|
|
@@ -223,7 +282,6 @@ class Step
|
|
223
282
|
begin
|
224
283
|
result = _exec
|
225
284
|
rescue Aborted
|
226
|
-
stop_dependencies
|
227
285
|
log(:aborted, "Aborted")
|
228
286
|
raise $!
|
229
287
|
rescue Exception
|
@@ -231,9 +289,9 @@ class Step
|
|
231
289
|
|
232
290
|
# HACK: This fixes an strange behaviour in 1.9.3 where some
|
233
291
|
# backtrace strings are coded in ASCII-8BIT
|
292
|
+
backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
|
234
293
|
set_info :backtrace, backtrace
|
235
294
|
log(:error, "#{$!.class}: #{$!.message}")
|
236
|
-
backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
|
237
295
|
stop_dependencies
|
238
296
|
raise $!
|
239
297
|
end
|
@@ -266,6 +324,7 @@ class Step
|
|
266
324
|
Log.exception $!
|
267
325
|
ensure
|
268
326
|
join
|
327
|
+
FileUtils.rm pid_file if File.exists?(pid_file)
|
269
328
|
end
|
270
329
|
end
|
271
330
|
stream.abort_callback = Proc.new do
|
@@ -273,6 +332,8 @@ class Step
|
|
273
332
|
log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
|
274
333
|
rescue
|
275
334
|
Log.exception $!
|
335
|
+
stop_dependencies
|
336
|
+
FileUtils.rm pid_file if File.exists?(pid_file)
|
276
337
|
end
|
277
338
|
end
|
278
339
|
else
|
@@ -280,6 +341,7 @@ class Step
|
|
280
341
|
set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
|
281
342
|
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|
282
343
|
log :done, "#{Log.color :magenta, "Completed"} step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
|
344
|
+
FileUtils.rm pid_file if File.exists?(pid_file)
|
283
345
|
end
|
284
346
|
|
285
347
|
result
|
@@ -294,6 +356,7 @@ class Step
|
|
294
356
|
end
|
295
357
|
rescue Exception
|
296
358
|
exception $!
|
359
|
+
stop_dependencies
|
297
360
|
raise $!
|
298
361
|
end
|
299
362
|
end
|
@@ -302,10 +365,10 @@ class Step
|
|
302
365
|
return self if done? and not dirty?
|
303
366
|
|
304
367
|
if error? or aborted?
|
305
|
-
if force
|
368
|
+
if force or aborted?
|
306
369
|
clean
|
307
370
|
else
|
308
|
-
raise "Error in job: #{status}"
|
371
|
+
raise "Error in job: #{status} - #{self.path}"
|
309
372
|
end
|
310
373
|
end
|
311
374
|
|
@@ -326,6 +389,7 @@ class Step
|
|
326
389
|
RbbtSemaphore.wait_semaphore(semaphore) if semaphore
|
327
390
|
FileUtils.mkdir_p File.dirname(path) unless File.exists? File.dirname(path)
|
328
391
|
begin
|
392
|
+
@forked = true
|
329
393
|
res = run true
|
330
394
|
set_info :forked, true
|
331
395
|
rescue Aborted
|
@@ -386,13 +450,14 @@ class Step
|
|
386
450
|
Log.medium "Could not abort #{path}: same process"
|
387
451
|
false
|
388
452
|
else
|
389
|
-
Log.medium "Aborting #{path}: #{ @pid }"
|
453
|
+
Log.medium "Aborting pid #{path}: #{ @pid }"
|
390
454
|
begin
|
391
|
-
Process.kill("
|
455
|
+
Process.kill("INT", @pid)
|
392
456
|
Process.waitpid @pid
|
393
457
|
rescue Exception
|
394
458
|
Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
|
395
459
|
end
|
460
|
+
Log.medium "Aborted pid #{path}: #{ @pid }"
|
396
461
|
true
|
397
462
|
end
|
398
463
|
end
|
@@ -422,7 +487,7 @@ class Step
|
|
422
487
|
begin
|
423
488
|
stop_dependencies
|
424
489
|
abort_stream
|
425
|
-
abort_pid
|
490
|
+
abort_pid if defined? @forked and @forked
|
426
491
|
rescue Aborted
|
427
492
|
Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
|
428
493
|
retry
|
@@ -463,7 +528,7 @@ class Step
|
|
463
528
|
end
|
464
529
|
|
465
530
|
def soft_grace
|
466
|
-
until
|
531
|
+
until done? or File.exists?(info_file)
|
467
532
|
sleep 1
|
468
533
|
end
|
469
534
|
self
|
@@ -506,7 +571,7 @@ class Step
|
|
506
571
|
pid = nil
|
507
572
|
dependencies.each{|dep| dep.join }
|
508
573
|
end
|
509
|
-
sleep 1 until path.exists?
|
574
|
+
sleep 1 until path.exists? or error? or aborted?
|
510
575
|
self
|
511
576
|
ensure
|
512
577
|
set_info :joined, true
|
@@ -20,6 +20,8 @@ $ rbbt app start [options] <app_name>
|
|
20
20
|
-R--Rserve_session* Rserve session to use, otherwise start new one
|
21
21
|
-wd--workdir* Change the working directory of the workflow
|
22
22
|
--views* Directory with view templates
|
23
|
+
--stream Activate streaming of workflow tasks
|
24
|
+
--options* Additional options for server (e.g. option1=value1;option2=value2)
|
23
25
|
EOF
|
24
26
|
|
25
27
|
if options[:help]
|
@@ -55,19 +57,32 @@ Misc.in_dir(app_dir) do
|
|
55
57
|
ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
|
56
58
|
ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
|
57
59
|
|
60
|
+
if options[:stream]
|
61
|
+
raise "No streaming available for any server other than puma" unless options[:server] =~ /^puma|unico/
|
62
|
+
ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
|
63
|
+
end
|
64
|
+
|
58
65
|
config_ru_file = File.exists?('./config.ru') ? './config.ru' : Rbbt.share['config.ru'].find
|
59
66
|
|
60
|
-
if
|
61
|
-
|
67
|
+
if options[:options]
|
68
|
+
options[:options].split(";").each do |pair|
|
69
|
+
name, _sep, value = pair.partition("=")
|
70
|
+
name = name[1..-1].to_sym if name[0] == ':'
|
71
|
+
options[name] = value
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
case server
|
76
|
+
when 'unicorn'
|
77
|
+
system ENV, "unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}"
|
78
|
+
when 'puma_alt'
|
79
|
+
system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
|
62
80
|
else
|
63
81
|
options[:config] = config_ru_file
|
64
82
|
Rack::Server.start(options)
|
65
83
|
end
|
66
84
|
end
|
67
85
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
86
|
#!/usr/bin/env ruby
|
72
87
|
|
73
88
|
require 'rbbt-util'
|
@@ -1,33 +1,28 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
require 'rbbt-util'
|
3
4
|
require 'rbbt/util/simpleopt'
|
4
|
-
require 'rbbt/workflow'
|
5
5
|
|
6
|
-
#
|
7
|
-
#require 'modular-scale'
|
8
|
-
|
9
|
-
require 'rbbt/rest/main'
|
10
|
-
require 'rbbt/rest/entity'
|
11
|
-
require 'rbbt/rest/workflow'
|
12
|
-
require 'rbbt/rest/file_server'
|
13
|
-
require 'rbbt/rest/knowledge_base'
|
14
|
-
require 'rbbt/rest/helpers'
|
15
|
-
require 'rbbt/rest/web_tool'
|
6
|
+
$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
|
16
7
|
|
17
8
|
options = SOPT.setup <<EOF
|
18
9
|
Start an rbbt app
|
19
10
|
|
20
|
-
$ rbbt
|
11
|
+
$ rbbt workflow server [options] <Workflow>
|
21
12
|
|
22
13
|
-h--help Print this help
|
23
14
|
-e--environment* Execution environment: production or development
|
24
|
-
-
|
15
|
+
-Ho--Host* Host name
|
16
|
+
-B--Bind* Bind IP
|
25
17
|
-p--port* TCP port
|
26
18
|
-s--server* Server type: thin, webrick, unicorn, etc
|
27
19
|
-f--finder Start server with finder functionality
|
28
20
|
-R--Rserve_session* Rserve session to use, otherwise start new one
|
21
|
+
-wd--workdir* Change the working directory of the workflow
|
29
22
|
-w--workflows* List of additional workflows to load
|
30
23
|
--views* Directory with view templates
|
24
|
+
--stream Activate streaming of workflow tasks
|
25
|
+
--options* Additional options for server (e.g. option1=value1;option2=value2)
|
31
26
|
EOF
|
32
27
|
|
33
28
|
if options[:help]
|
@@ -39,57 +34,59 @@ if options[:help]
|
|
39
34
|
exit 0
|
40
35
|
end
|
41
36
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
ENV["RServe-session"] = options[:RServe_session] || workflow
|
46
|
-
|
47
|
-
raise rbbt_usage unless workflow
|
48
|
-
|
49
|
-
wf = Workflow.require_workflow workflow
|
50
|
-
|
51
|
-
$title = wf.to_s
|
52
|
-
|
53
|
-
load Rbbt.etc['app.d/init.rb'].find
|
54
|
-
|
55
|
-
app = class WorkflowRest < Sinatra::Base; self end
|
56
|
-
|
57
|
-
app.register Sinatra::RbbtRESTWorkflow
|
58
|
-
app.register Sinatra::MultiRoute
|
59
|
-
app.register Sinatra::RbbtRESTKnowledgeBase
|
60
|
-
app.helpers Sinatra::RbbtToolHelper
|
61
|
-
|
62
|
-
app.get '/' do
|
63
|
-
redirect to(File.join('/', wf.to_s))
|
64
|
-
end
|
65
|
-
app.instance_eval Rbbt.etc['app.d/base.rb'].read
|
66
|
-
app.use Rack::Deflater
|
67
|
-
|
68
|
-
load Rbbt.etc['app.d/resources.rb'].find
|
69
|
-
|
70
|
-
app.class_eval do
|
71
|
-
eval Rbbt.etc['app.d/finder.rb'].read
|
37
|
+
if options[:workdir]
|
38
|
+
require 'rbbt/workflow'
|
39
|
+
Workflow.workdir = options[:workdir]
|
72
40
|
end
|
73
41
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
_wf = Workflow.require_workflow name
|
78
|
-
WorkflowRest.add_workflow _wf, true
|
79
|
-
end
|
80
|
-
end
|
42
|
+
options[:Port] ||= options[:port]
|
43
|
+
options[:Host] ||= "0.0.0.0"
|
44
|
+
options[:Bind] ||= "0.0.0.0"
|
81
45
|
|
82
|
-
|
46
|
+
workflow = ARGV.shift
|
47
|
+
workflows = options[:workflows] || ""
|
83
48
|
|
84
|
-
|
49
|
+
workflow = File.expand_path(workflow) if File.exists?(workflow)
|
85
50
|
|
51
|
+
ENV["RServe-session"] = options[:RServe_session] || workflow
|
86
52
|
|
87
|
-
|
88
|
-
|
53
|
+
server = options[:server] || 'puma'
|
54
|
+
|
55
|
+
TmpFile.with_file do |app_dir|
|
56
|
+
Misc.in_dir(app_dir) do
|
57
|
+
app_dir = Path.setup(app_dir.dup)
|
58
|
+
Open.write(app_dir.etc.target_workflow.find, workflow)
|
59
|
+
Open.write(app_dir.etc.workflows.find, workflows.split(/,\s/)*"\n") if workflows
|
60
|
+
|
61
|
+
require 'rack'
|
62
|
+
ENV["RBBT_FINDER"] = "true" if options.include?(:finder)
|
63
|
+
ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
|
64
|
+
ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
|
65
|
+
|
66
|
+
if options[:stream]
|
67
|
+
raise "No streaming available for any server other than puma" unless options[:server] == 'puma'
|
68
|
+
ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'
|
69
|
+
end
|
70
|
+
|
71
|
+
config_ru_file = File.exists?('./workflow_config.ru') ? './config.ru' : Rbbt.share['workflow_config.ru'].find
|
72
|
+
|
73
|
+
|
74
|
+
if options[:options]
|
75
|
+
options[:options].split(";").each do |pair|
|
76
|
+
name, _sep, value = pair.partition("=")
|
77
|
+
name = name[1..-1].to_sym if name[0] == ':'
|
78
|
+
options[name] = value
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
case server
|
83
|
+
when 'unicorn'
|
84
|
+
`unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:port] || "2887"}`
|
85
|
+
when 'puma_alt'
|
86
|
+
`puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
|
87
|
+
else
|
88
|
+
options[:config] = config_ru_file
|
89
|
+
Rack::Server.start(options)
|
90
|
+
end
|
91
|
+
end
|
89
92
|
end
|
90
|
-
|
91
|
-
WorkflowRest.port = options[:port] || 4567
|
92
|
-
WorkflowRest.bind = options[:bind] || "0.0.0.0"
|
93
|
-
WorkflowRest.environment = options[:environment] || "development"
|
94
|
-
WorkflowRest.server = options[:server] if options[:server]
|
95
|
-
WorkflowRest.run!
|
@@ -441,8 +441,8 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
|
|
441
441
|
puts res.to_s
|
442
442
|
end
|
443
443
|
when Step
|
444
|
-
if
|
445
|
-
io =
|
444
|
+
if res.streaming?
|
445
|
+
io = TSV.get_stream res
|
446
446
|
while block = io.read(2048) do
|
447
447
|
out.write block
|
448
448
|
end #unless io.closed?
|
data/test/rbbt/util/test_misc.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.19.
|
4
|
+
version: 5.19.36
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -261,6 +261,7 @@ files:
|
|
261
261
|
- lib/rbbt/util/misc/lock.rb
|
262
262
|
- lib/rbbt/util/misc/manipulation.rb
|
263
263
|
- lib/rbbt/util/misc/math.rb
|
264
|
+
- lib/rbbt/util/misc/multipart_payload.rb
|
264
265
|
- lib/rbbt/util/misc/objects.rb
|
265
266
|
- lib/rbbt/util/misc/omics.rb
|
266
267
|
- lib/rbbt/util/misc/options.rb
|