scout-gear 2.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +65 -2
- data/Rakefile +2 -0
- data/VERSION +1 -1
- data/bin/scout +233 -24
- data/lib/scout/cmd.rb +344 -0
- data/lib/scout/concurrent_stream.rb +259 -0
- data/lib/scout/exceptions.rb +15 -8
- data/lib/scout/indiferent_hash/options.rb +8 -26
- data/lib/scout/log/color.rb +2 -2
- data/lib/scout/log/fingerprint.rb +11 -1
- data/lib/scout/log/progress/report.rb +0 -1
- data/lib/scout/log/progress/util.rb +1 -1
- data/lib/scout/log/progress.rb +4 -4
- data/lib/scout/log.rb +10 -2
- data/lib/scout/meta_extension.rb +19 -3
- data/lib/scout/misc/digest.rb +56 -0
- data/lib/scout/misc/filesystem.rb +26 -0
- data/lib/scout/misc/format.rb +17 -6
- data/lib/scout/misc/insist.rb +56 -0
- data/lib/scout/misc/monitor.rb +23 -0
- data/lib/scout/misc.rb +5 -11
- data/lib/scout/open/lock.rb +61 -0
- data/lib/scout/open/remote.rb +120 -0
- data/lib/scout/open/stream.rb +373 -0
- data/lib/scout/open/util.rb +225 -0
- data/lib/scout/open.rb +169 -0
- data/lib/scout/path/find.rb +68 -21
- data/lib/scout/path/tmpfile.rb +8 -0
- data/lib/scout/path/util.rb +14 -1
- data/lib/scout/path.rb +6 -30
- data/lib/scout/persist/open.rb +17 -0
- data/lib/scout/persist/path.rb +15 -0
- data/lib/scout/persist/serialize.rb +151 -0
- data/lib/scout/persist.rb +54 -0
- data/lib/scout/resource/path.rb +20 -0
- data/lib/scout/resource/produce/rake.rb +69 -0
- data/lib/scout/resource/produce.rb +246 -0
- data/lib/scout/resource/scout.rb +3 -0
- data/lib/scout/resource/util.rb +48 -0
- data/lib/scout/resource.rb +39 -0
- data/lib/scout/simple_opt/accessor.rb +1 -1
- data/lib/scout/simple_opt/doc.rb +29 -23
- data/lib/scout/simple_opt/parse.rb +4 -3
- data/lib/scout/tmpfile.rb +39 -1
- data/lib/scout/workflow/definition.rb +78 -0
- data/lib/scout/workflow/documentation.rb +83 -0
- data/lib/scout/workflow/step/info.rb +77 -0
- data/lib/scout/workflow/step/load.rb +18 -0
- data/lib/scout/workflow/step.rb +132 -0
- data/lib/scout/workflow/task/inputs.rb +114 -0
- data/lib/scout/workflow/task.rb +155 -0
- data/lib/scout/workflow/usage.rb +314 -0
- data/lib/scout/workflow/util.rb +11 -0
- data/lib/scout/workflow.rb +40 -0
- data/lib/scout-gear.rb +4 -0
- data/lib/scout.rb +1 -0
- data/lib/workflow-scout.rb +2 -0
- data/scout-gear.gemspec +77 -5
- data/scout_commands/alias +48 -0
- data/scout_commands/find +83 -0
- data/scout_commands/glob +0 -0
- data/scout_commands/rbbt +23 -0
- data/scout_commands/workflow/info +29 -0
- data/scout_commands/workflow/list +27 -0
- data/scout_commands/workflow/task +58 -0
- data/scout_commands/workflow/task_old +706 -0
- data/test/scout/indiferent_hash/test_options.rb +11 -1
- data/test/scout/misc/test_digest.rb +30 -0
- data/test/scout/misc/test_filesystem.rb +30 -0
- data/test/scout/misc/test_insist.rb +13 -0
- data/test/scout/open/test_lock.rb +52 -0
- data/test/scout/open/test_remote.rb +25 -0
- data/test/scout/open/test_stream.rb +515 -0
- data/test/scout/open/test_util.rb +73 -0
- data/test/scout/path/test_find.rb +28 -0
- data/test/scout/persist/test_open.rb +37 -0
- data/test/scout/persist/test_path.rb +37 -0
- data/test/scout/persist/test_serialize.rb +114 -0
- data/test/scout/resource/test_path.rb +40 -0
- data/test/scout/resource/test_produce.rb +62 -0
- data/test/scout/resource/test_util.rb +27 -0
- data/test/scout/simple_opt/test_doc.rb +16 -0
- data/test/scout/test_cmd.rb +85 -0
- data/test/scout/test_concurrent_stream.rb +29 -0
- data/test/scout/test_meta_extension.rb +9 -0
- data/test/scout/test_misc.rb +0 -7
- data/test/scout/test_open.rb +146 -0
- data/test/scout/test_path.rb +3 -1
- data/test/scout/test_persist.rb +83 -0
- data/test/scout/test_resource.rb +26 -0
- data/test/scout/test_workflow.rb +87 -0
- data/test/scout/workflow/step/test_info.rb +30 -0
- data/test/scout/workflow/step/test_load.rb +65 -0
- data/test/scout/workflow/task/test_inputs.rb +182 -0
- data/test/scout/workflow/test_definition.rb +0 -0
- data/test/scout/workflow/test_documentation.rb +30 -0
- data/test/scout/workflow/test_step.rb +36 -0
- data/test/scout/workflow/test_task.rb +179 -0
- data/test/scout/workflow/test_usage.rb +35 -0
- data/test/scout/workflow/test_util.rb +17 -0
- data/test/test_helper.rb +17 -0
- data/test/test_scout-gear.rb +0 -0
- metadata +75 -3
data/lib/scout/cmd.rb
ADDED
@@ -0,0 +1,344 @@
|
|
1
|
+
require_relative 'indiferent_hash'
|
2
|
+
require_relative 'concurrent_stream'
|
3
|
+
require_relative 'log'
|
4
|
+
require_relative 'open/stream'
|
5
|
+
require 'stringio'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
module CMD
|
9
|
+
|
10
|
+
TOOLS = IndiferentHash.setup({})
|
11
|
+
def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
|
12
|
+
TOOLS[tool] = [claim, test, block, cmd]
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.conda(tool, env = nil, channel = 'bioconda')
|
16
|
+
if env
|
17
|
+
CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
|
18
|
+
else
|
19
|
+
CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.get_tool(tool)
|
24
|
+
return tool.to_s unless TOOLS[tool]
|
25
|
+
|
26
|
+
@@init_cmd_tool ||= IndiferentHash.setup({})
|
27
|
+
if !@@init_cmd_tool[tool]
|
28
|
+
claim, test, block, cmd = TOOLS[tool]
|
29
|
+
begin
|
30
|
+
if test
|
31
|
+
CMD.cmd(test + " ")
|
32
|
+
else
|
33
|
+
CMD.cmd("#{tool} --help")
|
34
|
+
end
|
35
|
+
rescue
|
36
|
+
if claim
|
37
|
+
claim.produce
|
38
|
+
else
|
39
|
+
block.call
|
40
|
+
end
|
41
|
+
end
|
42
|
+
version_txt = ""
|
43
|
+
version = nil
|
44
|
+
["--version", "-version", "--help", ""].each do |f|
|
45
|
+
begin
|
46
|
+
version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
|
47
|
+
version = CMD.scan_version_text(version_txt, tool)
|
48
|
+
break if version
|
49
|
+
rescue
|
50
|
+
Log.exception $!
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
@@init_cmd_tool[tool] = version || true
|
55
|
+
|
56
|
+
return cmd if cmd
|
57
|
+
end
|
58
|
+
|
59
|
+
tool.to_s
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.scan_version_text(text, cmd = nil)
|
63
|
+
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
64
|
+
text.split("\n").each do |line|
|
65
|
+
next unless line =~ /\W#{cmd}\W/i
|
66
|
+
m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
67
|
+
return m[1] if m
|
68
|
+
m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
|
69
|
+
return m[1] if m
|
70
|
+
next unless line =~ /\Wversion\W/i
|
71
|
+
m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
72
|
+
return m[1] if m
|
73
|
+
end
|
74
|
+
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
75
|
+
return m[1] if m
|
76
|
+
m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
|
77
|
+
return m[1] if m
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
def self.versions
|
81
|
+
return {} unless defined? @@init_cmd_tool
|
82
|
+
@@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.bash(cmd)
|
86
|
+
cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
|
87
|
+
CMD.cmd(cmd, :autojoin => true)
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.process_cmd_options(options = {})
|
91
|
+
add_dashes = IndiferentHash.process_options options, :add_option_dashes
|
92
|
+
|
93
|
+
string = ""
|
94
|
+
options.each do |option, value|
|
95
|
+
raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
|
96
|
+
#raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
|
97
|
+
value = value.gsub("'","\\'") if value.to_s.include? "'"
|
98
|
+
|
99
|
+
option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
|
100
|
+
|
101
|
+
case
|
102
|
+
when value.nil? || FalseClass === value
|
103
|
+
next
|
104
|
+
when TrueClass === value
|
105
|
+
string << "#{option} "
|
106
|
+
else
|
107
|
+
if option.to_s.chars.to_a.last == "="
|
108
|
+
string << "#{option}'#{value}' "
|
109
|
+
else
|
110
|
+
string << "#{option} '#{value}' "
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
string.strip
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.cmd(tool, cmd = nil, options = {}, &block)
|
119
|
+
options, cmd = cmd, nil if Hash === cmd
|
120
|
+
|
121
|
+
options = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
|
122
|
+
in_content = options.delete(:in)
|
123
|
+
stderr = options.delete(:stderr)
|
124
|
+
post = options.delete(:post)
|
125
|
+
pipe = options.delete(:pipe)
|
126
|
+
log = options.delete(:log)
|
127
|
+
no_fail = options.delete(:no_fail)
|
128
|
+
no_fail = options.delete(:nofail) if no_fail.nil?
|
129
|
+
no_wait = options.delete(:no_wait)
|
130
|
+
xvfb = options.delete(:xvfb)
|
131
|
+
bar = options.delete(:progress_bar)
|
132
|
+
save_stderr = options.delete(:save_stderr)
|
133
|
+
autojoin = options.delete(:autojoin)
|
134
|
+
autojoin = no_wait if autojoin.nil?
|
135
|
+
|
136
|
+
dont_close_in = options.delete(:dont_close_in)
|
137
|
+
|
138
|
+
log = true if log.nil?
|
139
|
+
|
140
|
+
if cmd.nil? && ! Symbol === tool
|
141
|
+
cmd = tool
|
142
|
+
else
|
143
|
+
tool = get_tool(tool)
|
144
|
+
if cmd.nil?
|
145
|
+
cmd = tool
|
146
|
+
else
|
147
|
+
cmd = tool + ' ' + cmd
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
151
|
+
|
152
|
+
case xvfb
|
153
|
+
when TrueClass
|
154
|
+
cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
|
155
|
+
when String
|
156
|
+
cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
|
157
|
+
when String
|
158
|
+
end
|
159
|
+
|
160
|
+
if stderr == true
|
161
|
+
stderr = Log::HIGH
|
162
|
+
end
|
163
|
+
|
164
|
+
cmd_options = process_cmd_options options
|
165
|
+
if cmd =~ /'\{opt\}'/
|
166
|
+
cmd.sub!('\'{opt}\'', cmd_options)
|
167
|
+
else
|
168
|
+
cmd << " " << cmd_options
|
169
|
+
end
|
170
|
+
|
171
|
+
in_content = StringIO.new in_content if String === in_content
|
172
|
+
|
173
|
+
sin, sout, serr, wait_thr = begin
|
174
|
+
Open3.popen3(ENV, cmd)
|
175
|
+
rescue
|
176
|
+
Log.warn $!.message
|
177
|
+
raise ProcessFailed, nil, cmd unless no_fail
|
178
|
+
return
|
179
|
+
end
|
180
|
+
pid = wait_thr.pid
|
181
|
+
|
182
|
+
Log.debug{"CMD: [#{pid}] #{cmd}" if log}
|
183
|
+
|
184
|
+
if in_content.respond_to?(:read)
|
185
|
+
in_thread = Thread.new(Thread.current) do |parent|
|
186
|
+
Thread.current.report_on_exception = false if no_fail
|
187
|
+
begin
|
188
|
+
begin
|
189
|
+
while c = in_content.readpartial(Open::BLOCK_SIZE)
|
190
|
+
sin << c
|
191
|
+
end
|
192
|
+
rescue EOFError
|
193
|
+
end
|
194
|
+
sin.close unless sin.closed?
|
195
|
+
|
196
|
+
unless dont_close_in
|
197
|
+
in_content.close unless in_content.closed?
|
198
|
+
in_content.join if in_content.respond_to? :join
|
199
|
+
end
|
200
|
+
rescue
|
201
|
+
Log.error "Error in CMD [#{pid}] #{cmd}: #{$!.message}" unless no_fail
|
202
|
+
raise $!
|
203
|
+
end
|
204
|
+
end
|
205
|
+
else
|
206
|
+
in_thread = nil
|
207
|
+
sin.close
|
208
|
+
end
|
209
|
+
|
210
|
+
pids = [pid]
|
211
|
+
|
212
|
+
if pipe
|
213
|
+
|
214
|
+
ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
|
215
|
+
|
216
|
+
sout.callback = post if post
|
217
|
+
|
218
|
+
if (Integer === stderr and log) || bar
|
219
|
+
err_thread = Thread.new do
|
220
|
+
Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
|
221
|
+
begin
|
222
|
+
while line = serr.gets
|
223
|
+
bar.process(line) if bar
|
224
|
+
sout.log = line
|
225
|
+
sout.std_err << line if save_stderr
|
226
|
+
Log.log "STDERR [#{pid}]: " + line, stderr if log
|
227
|
+
end
|
228
|
+
serr.close
|
229
|
+
rescue
|
230
|
+
Log.exception $!
|
231
|
+
raise $!
|
232
|
+
end
|
233
|
+
end
|
234
|
+
else
|
235
|
+
err_thread = Open.consume_stream(serr, true)
|
236
|
+
end
|
237
|
+
|
238
|
+
sout.threads = [in_thread, err_thread, wait_thr].compact
|
239
|
+
|
240
|
+
sout
|
241
|
+
else
|
242
|
+
|
243
|
+
if bar
|
244
|
+
err = ""
|
245
|
+
err_thread = Thread.new do
|
246
|
+
while not serr.eof?
|
247
|
+
line = serr.gets
|
248
|
+
bar.process(line)
|
249
|
+
err << line if Integer === stderr and log
|
250
|
+
end
|
251
|
+
serr.close
|
252
|
+
end
|
253
|
+
elsif log and Integer === stderr
|
254
|
+
err = ""
|
255
|
+
err_thread = Thread.new do
|
256
|
+
while not serr.eof?
|
257
|
+
err << serr.gets
|
258
|
+
end
|
259
|
+
serr.close
|
260
|
+
end
|
261
|
+
else
|
262
|
+
Open.consume_stream(serr, true)
|
263
|
+
#serr.close
|
264
|
+
err_thread = nil
|
265
|
+
err = ""
|
266
|
+
end
|
267
|
+
|
268
|
+
ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
|
269
|
+
|
270
|
+
begin
|
271
|
+
out = StringIO.new sout.read
|
272
|
+
sout.close unless sout.closed?
|
273
|
+
|
274
|
+
status = wait_thr.value
|
275
|
+
if not status.success? and not no_fail
|
276
|
+
if !err.empty?
|
277
|
+
raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
|
278
|
+
else
|
279
|
+
raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
|
280
|
+
end
|
281
|
+
else
|
282
|
+
Log.log err, stderr if Integer === stderr and log
|
283
|
+
end
|
284
|
+
out
|
285
|
+
ensure
|
286
|
+
post.call if post
|
287
|
+
end
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
def self.cmd_pid(*args)
|
292
|
+
all_args = *args
|
293
|
+
|
294
|
+
bar = all_args.last[:progress_bar] if Hash === all_args.last
|
295
|
+
|
296
|
+
all_args << {} unless Hash === all_args.last
|
297
|
+
|
298
|
+
level = all_args.last[:log] || 0
|
299
|
+
level = 0 if TrueClass === level
|
300
|
+
level = 10 if FalseClass === level
|
301
|
+
level = level.to_i
|
302
|
+
|
303
|
+
all_args.last[:log] = true
|
304
|
+
all_args.last[:pipe] = true
|
305
|
+
|
306
|
+
io = cmd(*all_args)
|
307
|
+
pid = io.pids.first
|
308
|
+
|
309
|
+
line = "" if bar
|
310
|
+
starting = true
|
311
|
+
while c = io.getc
|
312
|
+
if starting
|
313
|
+
if pid
|
314
|
+
Log.logn "STDOUT [#{pid}]: ", level
|
315
|
+
else
|
316
|
+
Log.logn "STDOUT: ", level
|
317
|
+
end
|
318
|
+
starting = false
|
319
|
+
end
|
320
|
+
STDERR << c if Log.severity <= level
|
321
|
+
line << c if bar
|
322
|
+
if c == "\n"
|
323
|
+
bar.process(line) if bar
|
324
|
+
starting = true
|
325
|
+
line = "" if bar
|
326
|
+
end
|
327
|
+
end
|
328
|
+
begin
|
329
|
+
io.join
|
330
|
+
bar.remove if bar
|
331
|
+
rescue
|
332
|
+
bar.remove(true) if bar
|
333
|
+
raise $!
|
334
|
+
end
|
335
|
+
|
336
|
+
nil
|
337
|
+
end
|
338
|
+
|
339
|
+
def self.cmd_log(*args)
|
340
|
+
cmd_pid(*args)
|
341
|
+
nil
|
342
|
+
end
|
343
|
+
|
344
|
+
end
|
@@ -0,0 +1,259 @@
|
|
1
|
+
require_relative 'indiferent_hash'
|
2
|
+
|
3
|
+
module AbortedStream
|
4
|
+
attr_accessor :exception
|
5
|
+
def self.setup(obj, exception = nil)
|
6
|
+
obj.extend AbortedStream
|
7
|
+
obj.exception = exception
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
module ConcurrentStream
|
12
|
+
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
|
13
|
+
|
14
|
+
def self.setup(stream, options = {}, &block)
|
15
|
+
|
16
|
+
threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
|
17
|
+
stream.extend ConcurrentStream unless ConcurrentStream === stream
|
18
|
+
|
19
|
+
stream.threads ||= []
|
20
|
+
stream.pids ||= []
|
21
|
+
stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
|
22
|
+
stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
|
23
|
+
stream.autojoin = autojoin unless autojoin.nil?
|
24
|
+
stream.no_fail = no_fail unless no_fail.nil?
|
25
|
+
stream.std_err = ""
|
26
|
+
|
27
|
+
stream.pair = pair unless pair.nil?
|
28
|
+
|
29
|
+
callback = block if block_given?
|
30
|
+
if callback
|
31
|
+
if stream.callback
|
32
|
+
old_callback = stream.callback
|
33
|
+
stream.callback = Proc.new do
|
34
|
+
old_callback.call
|
35
|
+
callback.call
|
36
|
+
end
|
37
|
+
else
|
38
|
+
stream.callback = callback
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
if abort_callback
|
43
|
+
if stream.abort_callback
|
44
|
+
old_abort_callback = stream.abort_callback
|
45
|
+
stream.abort_callback = Proc.new do
|
46
|
+
old_abort_callback.call
|
47
|
+
abort_callback.call
|
48
|
+
end
|
49
|
+
else
|
50
|
+
stream.abort_callback = abort_callback
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
|
55
|
+
|
56
|
+
stream.lockfile = lockfile unless lockfile.nil?
|
57
|
+
|
58
|
+
stream.aborted = false
|
59
|
+
|
60
|
+
stream
|
61
|
+
end
|
62
|
+
|
63
|
+
def annotate(stream)
|
64
|
+
ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
|
65
|
+
stream
|
66
|
+
end
|
67
|
+
|
68
|
+
def clear
|
69
|
+
@threads = @pids = @callback = @abort_callback = @joined = nil
|
70
|
+
end
|
71
|
+
|
72
|
+
def joined?
|
73
|
+
@joined
|
74
|
+
end
|
75
|
+
|
76
|
+
def aborted?
|
77
|
+
@aborted
|
78
|
+
end
|
79
|
+
|
80
|
+
def join_threads
|
81
|
+
if @threads
|
82
|
+
@threads.each do |t|
|
83
|
+
next if t == Thread.current
|
84
|
+
begin
|
85
|
+
t.join
|
86
|
+
if Process::Status === t.value
|
87
|
+
if ! (t.value.success? || no_fail)
|
88
|
+
|
89
|
+
if log
|
90
|
+
msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
|
91
|
+
else
|
92
|
+
msg = "Error joining #{self.filename || self.inspect}"
|
93
|
+
end
|
94
|
+
|
95
|
+
raise ConcurrentStreamProcessFailed.new t.pid, msg, self
|
96
|
+
end
|
97
|
+
end
|
98
|
+
rescue Exception
|
99
|
+
if no_fail
|
100
|
+
Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
|
101
|
+
else
|
102
|
+
Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
|
103
|
+
stream_raise_exception $!
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
@threads = []
|
109
|
+
end
|
110
|
+
|
111
|
+
def join_pids
|
112
|
+
if @pids and @pids.any?
|
113
|
+
@pids.each do |pid|
|
114
|
+
begin
|
115
|
+
Process.waitpid(pid, Process::WUNTRACED)
|
116
|
+
stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
|
117
|
+
rescue Errno::ECHILD
|
118
|
+
end
|
119
|
+
end
|
120
|
+
@pids = []
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def join_callback
|
125
|
+
if @callback and not joined?
|
126
|
+
begin
|
127
|
+
@callback.call
|
128
|
+
ensure
|
129
|
+
@callback = nil
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def join
|
135
|
+
begin
|
136
|
+
join_threads
|
137
|
+
join_pids
|
138
|
+
join_callback
|
139
|
+
close unless closed?
|
140
|
+
ensure
|
141
|
+
@joined = true
|
142
|
+
lockfile.unlock if lockfile and lockfile.locked?
|
143
|
+
raise stream_exception if stream_exception
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def abort_threads(exception = nil)
|
148
|
+
return unless @threads and @threads.any?
|
149
|
+
name = Log.fingerprint(Thread.current)
|
150
|
+
name += " - file:#{filename}" if filename
|
151
|
+
Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
|
152
|
+
|
153
|
+
@threads.each do |t|
|
154
|
+
next if t == Thread.current
|
155
|
+
Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
|
156
|
+
t.raise((exception.nil? ? Aborted.new : exception))
|
157
|
+
end
|
158
|
+
|
159
|
+
@threads.each do |t|
|
160
|
+
next if t == Thread.current
|
161
|
+
begin
|
162
|
+
t.join unless t == Thread.current
|
163
|
+
rescue Aborted
|
164
|
+
rescue Exception
|
165
|
+
Log.debug "Thread (#{name}) exception: #{$!.message}"
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def abort_pids
|
171
|
+
@pids.each do |pid|
|
172
|
+
begin
|
173
|
+
Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
|
174
|
+
Process.kill :INT, pid
|
175
|
+
rescue Errno::ESRCH
|
176
|
+
end
|
177
|
+
end if @pids
|
178
|
+
@pids = []
|
179
|
+
end
|
180
|
+
|
181
|
+
def abort(exception = nil)
|
182
|
+
self.stream_exception ||= exception
|
183
|
+
if @aborted
|
184
|
+
Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
|
185
|
+
return
|
186
|
+
else
|
187
|
+
Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
|
188
|
+
end
|
189
|
+
AbortedStream.setup(self, exception)
|
190
|
+
@aborted = true
|
191
|
+
begin
|
192
|
+
@abort_callback.call exception if @abort_callback
|
193
|
+
|
194
|
+
abort_threads(exception)
|
195
|
+
abort_pids
|
196
|
+
|
197
|
+
@callback = nil
|
198
|
+
@abort_callback = nil
|
199
|
+
|
200
|
+
if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
|
201
|
+
Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
|
202
|
+
@pair.abort exception
|
203
|
+
end
|
204
|
+
ensure
|
205
|
+
close unless closed?
|
206
|
+
|
207
|
+
if lockfile and lockfile.locked?
|
208
|
+
lockfile.unlock
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
def close(*args)
|
214
|
+
if autojoin
|
215
|
+
begin
|
216
|
+
super(*args)
|
217
|
+
rescue
|
218
|
+
Log.exception $!
|
219
|
+
self.abort
|
220
|
+
self.join
|
221
|
+
stream_raise_exception $!
|
222
|
+
ensure
|
223
|
+
self.join if self.closed? or self.eof?
|
224
|
+
end
|
225
|
+
else
|
226
|
+
super(*args)
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
def read(*args)
|
231
|
+
begin
|
232
|
+
super(*args)
|
233
|
+
ensure
|
234
|
+
begin
|
235
|
+
close unless closed?
|
236
|
+
rescue Exception
|
237
|
+
raise $! if ConcurrentStreamProcessFailed === $!
|
238
|
+
end if autojoin && ! closed? && eof?
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
def add_callback(&block)
|
243
|
+
old_callback = callback
|
244
|
+
@callback = Proc.new do
|
245
|
+
old_callback.call if old_callback
|
246
|
+
block.call
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
def stream_raise_exception(exception)
|
251
|
+
threads.each do |thread|
|
252
|
+
thread.raise exception
|
253
|
+
end
|
254
|
+
self.stream_exception = exception
|
255
|
+
|
256
|
+
self.abort
|
257
|
+
end
|
258
|
+
|
259
|
+
end
|
data/lib/scout/exceptions.rb
CHANGED
@@ -13,6 +13,12 @@ end
|
|
13
13
|
|
14
14
|
class Aborted < StandardError; end
|
15
15
|
|
16
|
+
class ParameterException < ScoutException; end
|
17
|
+
class MissingParameterException < ParameterException
|
18
|
+
def initialize(parameter)
|
19
|
+
super("Missing parameter '#{parameter}'")
|
20
|
+
end
|
21
|
+
end
|
16
22
|
class ProcessFailed < StandardError;
|
17
23
|
attr_accessor :pid, :msg
|
18
24
|
def initialize(pid = Process.pid, msg = nil)
|
@@ -41,7 +47,13 @@ end
|
|
41
47
|
|
42
48
|
class OpenURLError < StandardError; end
|
43
49
|
|
44
|
-
class DontClose < Exception
|
50
|
+
class DontClose < Exception
|
51
|
+
attr_accessor :payload
|
52
|
+
def initialize(payload = nil)
|
53
|
+
@payload = payload
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
45
57
|
|
46
58
|
class KeepLocked < Exception
|
47
59
|
attr_accessor :payload
|
@@ -57,13 +69,9 @@ class KeepBar < Exception
|
|
57
69
|
end
|
58
70
|
end
|
59
71
|
|
60
|
-
|
72
|
+
class LockInterrupted < TryAgain; end
|
73
|
+
|
61
74
|
#
|
62
|
-
#class MissingParameterException < ParameterException
|
63
|
-
# def initialize(parameter)
|
64
|
-
# super("Missing parameter '#{parameter}'")
|
65
|
-
# end
|
66
|
-
#end
|
67
75
|
#class ClosedStream < StandardError; end
|
68
76
|
#class OpenGzipError < StandardError; end
|
69
77
|
#
|
@@ -76,7 +84,6 @@ end
|
|
76
84
|
#end
|
77
85
|
#
|
78
86
|
#class SemaphoreInterrupted < TryAgain; end
|
79
|
-
#class LockInterrupted < TryAgain; end
|
80
87
|
#
|
81
88
|
#class RemoteServerError < StandardError; end
|
82
89
|
#
|
@@ -1,32 +1,24 @@
|
|
1
1
|
module IndiferentHash
|
2
2
|
def self.add_defaults(options, defaults = {})
|
3
|
-
options
|
3
|
+
options = string2hash options if String === options
|
4
4
|
IndiferentHash.setup(options)
|
5
|
-
|
6
|
-
|
7
|
-
new_options = options.dup
|
8
|
-
when String === options
|
9
|
-
new_options = string2hash options
|
10
|
-
else
|
11
|
-
raise "Format of '#{options.inspect}' not understood. It should be a hash"
|
12
|
-
end
|
5
|
+
|
6
|
+
defaults = string2hash defaults if String === defaults
|
13
7
|
|
14
8
|
defaults.each do |key, value|
|
15
9
|
next if options.include? key
|
16
10
|
|
17
|
-
|
11
|
+
options[key] = value
|
18
12
|
end
|
19
13
|
|
20
|
-
|
21
|
-
|
22
|
-
options.replace new_options
|
14
|
+
options
|
23
15
|
end
|
24
16
|
|
25
17
|
def self.process_options(hash, *keys)
|
26
18
|
IndiferentHash.setup(hash)
|
27
19
|
|
28
20
|
defaults = keys.pop if Hash === keys.last
|
29
|
-
|
21
|
+
hash = IndiferentHash.add_defaults hash, defaults if defaults
|
30
22
|
|
31
23
|
if keys.length == 1
|
32
24
|
hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s)
|
@@ -102,7 +94,7 @@ module IndiferentHash
|
|
102
94
|
options = {}
|
103
95
|
|
104
96
|
string.split('#').each do |str|
|
105
|
-
key,
|
97
|
+
key, _, value = str.partition "="
|
106
98
|
|
107
99
|
key = key[1..-1].to_sym if key[0] == ":"
|
108
100
|
|
@@ -114,17 +106,7 @@ module IndiferentHash
|
|
114
106
|
options[key] = value.to_f and next if value =~ /^\d*\.\d+$/
|
115
107
|
options[key] = true and next if value == "true"
|
116
108
|
options[key] = false and next if value == "false"
|
117
|
-
options[key] = value
|
118
|
-
|
119
|
-
options[key] = begin
|
120
|
-
saved_safe = $SAFE
|
121
|
-
$SAFE = 0
|
122
|
-
eval(value)
|
123
|
-
rescue Exception
|
124
|
-
value
|
125
|
-
ensure
|
126
|
-
$SAFE = saved_safe
|
127
|
-
end
|
109
|
+
options[key] = value
|
128
110
|
end
|
129
111
|
|
130
112
|
IndiferentHash.setup(options)
|