scout-gear 2.0.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +65 -2
  3. data/Rakefile +2 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +233 -24
  6. data/lib/scout/cmd.rb +344 -0
  7. data/lib/scout/concurrent_stream.rb +259 -0
  8. data/lib/scout/exceptions.rb +15 -8
  9. data/lib/scout/indiferent_hash/options.rb +8 -26
  10. data/lib/scout/log/color.rb +2 -2
  11. data/lib/scout/log/fingerprint.rb +11 -1
  12. data/lib/scout/log/progress/report.rb +0 -1
  13. data/lib/scout/log/progress/util.rb +1 -1
  14. data/lib/scout/log/progress.rb +4 -4
  15. data/lib/scout/log.rb +10 -2
  16. data/lib/scout/meta_extension.rb +19 -3
  17. data/lib/scout/misc/digest.rb +56 -0
  18. data/lib/scout/misc/filesystem.rb +26 -0
  19. data/lib/scout/misc/format.rb +17 -6
  20. data/lib/scout/misc/insist.rb +56 -0
  21. data/lib/scout/misc/monitor.rb +23 -0
  22. data/lib/scout/misc.rb +5 -11
  23. data/lib/scout/open/lock.rb +61 -0
  24. data/lib/scout/open/remote.rb +120 -0
  25. data/lib/scout/open/stream.rb +373 -0
  26. data/lib/scout/open/util.rb +225 -0
  27. data/lib/scout/open.rb +169 -0
  28. data/lib/scout/path/find.rb +68 -21
  29. data/lib/scout/path/tmpfile.rb +8 -0
  30. data/lib/scout/path/util.rb +14 -1
  31. data/lib/scout/path.rb +6 -30
  32. data/lib/scout/persist/open.rb +17 -0
  33. data/lib/scout/persist/path.rb +15 -0
  34. data/lib/scout/persist/serialize.rb +151 -0
  35. data/lib/scout/persist.rb +54 -0
  36. data/lib/scout/resource/path.rb +20 -0
  37. data/lib/scout/resource/produce/rake.rb +69 -0
  38. data/lib/scout/resource/produce.rb +246 -0
  39. data/lib/scout/resource/scout.rb +3 -0
  40. data/lib/scout/resource/util.rb +48 -0
  41. data/lib/scout/resource.rb +39 -0
  42. data/lib/scout/simple_opt/accessor.rb +1 -1
  43. data/lib/scout/simple_opt/doc.rb +29 -23
  44. data/lib/scout/simple_opt/parse.rb +4 -3
  45. data/lib/scout/tmpfile.rb +39 -1
  46. data/lib/scout/workflow/definition.rb +78 -0
  47. data/lib/scout/workflow/documentation.rb +83 -0
  48. data/lib/scout/workflow/step/info.rb +77 -0
  49. data/lib/scout/workflow/step/load.rb +18 -0
  50. data/lib/scout/workflow/step.rb +132 -0
  51. data/lib/scout/workflow/task/inputs.rb +114 -0
  52. data/lib/scout/workflow/task.rb +155 -0
  53. data/lib/scout/workflow/usage.rb +314 -0
  54. data/lib/scout/workflow/util.rb +11 -0
  55. data/lib/scout/workflow.rb +40 -0
  56. data/lib/scout-gear.rb +4 -0
  57. data/lib/scout.rb +1 -0
  58. data/lib/workflow-scout.rb +2 -0
  59. data/scout-gear.gemspec +77 -5
  60. data/scout_commands/alias +48 -0
  61. data/scout_commands/find +83 -0
  62. data/scout_commands/glob +0 -0
  63. data/scout_commands/rbbt +23 -0
  64. data/scout_commands/workflow/info +29 -0
  65. data/scout_commands/workflow/list +27 -0
  66. data/scout_commands/workflow/task +58 -0
  67. data/scout_commands/workflow/task_old +706 -0
  68. data/test/scout/indiferent_hash/test_options.rb +11 -1
  69. data/test/scout/misc/test_digest.rb +30 -0
  70. data/test/scout/misc/test_filesystem.rb +30 -0
  71. data/test/scout/misc/test_insist.rb +13 -0
  72. data/test/scout/open/test_lock.rb +52 -0
  73. data/test/scout/open/test_remote.rb +25 -0
  74. data/test/scout/open/test_stream.rb +515 -0
  75. data/test/scout/open/test_util.rb +73 -0
  76. data/test/scout/path/test_find.rb +28 -0
  77. data/test/scout/persist/test_open.rb +37 -0
  78. data/test/scout/persist/test_path.rb +37 -0
  79. data/test/scout/persist/test_serialize.rb +114 -0
  80. data/test/scout/resource/test_path.rb +40 -0
  81. data/test/scout/resource/test_produce.rb +62 -0
  82. data/test/scout/resource/test_util.rb +27 -0
  83. data/test/scout/simple_opt/test_doc.rb +16 -0
  84. data/test/scout/test_cmd.rb +85 -0
  85. data/test/scout/test_concurrent_stream.rb +29 -0
  86. data/test/scout/test_meta_extension.rb +9 -0
  87. data/test/scout/test_misc.rb +0 -7
  88. data/test/scout/test_open.rb +146 -0
  89. data/test/scout/test_path.rb +3 -1
  90. data/test/scout/test_persist.rb +83 -0
  91. data/test/scout/test_resource.rb +26 -0
  92. data/test/scout/test_workflow.rb +87 -0
  93. data/test/scout/workflow/step/test_info.rb +30 -0
  94. data/test/scout/workflow/step/test_load.rb +65 -0
  95. data/test/scout/workflow/task/test_inputs.rb +182 -0
  96. data/test/scout/workflow/test_definition.rb +0 -0
  97. data/test/scout/workflow/test_documentation.rb +30 -0
  98. data/test/scout/workflow/test_step.rb +36 -0
  99. data/test/scout/workflow/test_task.rb +179 -0
  100. data/test/scout/workflow/test_usage.rb +35 -0
  101. data/test/scout/workflow/test_util.rb +17 -0
  102. data/test/test_helper.rb +17 -0
  103. data/test/test_scout-gear.rb +0 -0
  104. metadata +75 -3
data/lib/scout/cmd.rb ADDED
@@ -0,0 +1,344 @@
1
+ require_relative 'indiferent_hash'
2
+ require_relative 'concurrent_stream'
3
+ require_relative 'log'
4
+ require_relative 'open/stream'
5
+ require 'stringio'
6
+ require 'open3'
7
+
8
+ module CMD
9
+
10
+ TOOLS = IndiferentHash.setup({})
11
+ def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
12
+ TOOLS[tool] = [claim, test, block, cmd]
13
+ end
14
+
15
+ def self.conda(tool, env = nil, channel = 'bioconda')
16
+ if env
17
+ CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
18
+ else
19
+ CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
20
+ end
21
+ end
22
+
23
+ def self.get_tool(tool)
24
+ return tool.to_s unless TOOLS[tool]
25
+
26
+ @@init_cmd_tool ||= IndiferentHash.setup({})
27
+ if !@@init_cmd_tool[tool]
28
+ claim, test, block, cmd = TOOLS[tool]
29
+ begin
30
+ if test
31
+ CMD.cmd(test + " ")
32
+ else
33
+ CMD.cmd("#{tool} --help")
34
+ end
35
+ rescue
36
+ if claim
37
+ claim.produce
38
+ else
39
+ block.call
40
+ end
41
+ end
42
+ version_txt = ""
43
+ version = nil
44
+ ["--version", "-version", "--help", ""].each do |f|
45
+ begin
46
+ version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
47
+ version = CMD.scan_version_text(version_txt, tool)
48
+ break if version
49
+ rescue
50
+ Log.exception $!
51
+ end
52
+ end
53
+
54
+ @@init_cmd_tool[tool] = version || true
55
+
56
+ return cmd if cmd
57
+ end
58
+
59
+ tool.to_s
60
+ end
61
+
62
+ def self.scan_version_text(text, cmd = nil)
63
+ cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
64
+ text.split("\n").each do |line|
65
+ next unless line =~ /\W#{cmd}\W/i
66
+ m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
67
+ return m[1] if m
68
+ m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
69
+ return m[1] if m
70
+ next unless line =~ /\Wversion\W/i
71
+ m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
72
+ return m[1] if m
73
+ end
74
+ m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
75
+ return m[1] if m
76
+ m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
77
+ return m[1] if m
78
+ nil
79
+ end
80
+ def self.versions
81
+ return {} unless defined? @@init_cmd_tool
82
+ @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
83
+ end
84
+
85
+ def self.bash(cmd)
86
+ cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
87
+ CMD.cmd(cmd, :autojoin => true)
88
+ end
89
+
90
+ def self.process_cmd_options(options = {})
91
+ add_dashes = IndiferentHash.process_options options, :add_option_dashes
92
+
93
+ string = ""
94
+ options.each do |option, value|
95
+ raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
96
+ #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
97
+ value = value.gsub("'","\\'") if value.to_s.include? "'"
98
+
99
+ option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
100
+
101
+ case
102
+ when value.nil? || FalseClass === value
103
+ next
104
+ when TrueClass === value
105
+ string << "#{option} "
106
+ else
107
+ if option.to_s.chars.to_a.last == "="
108
+ string << "#{option}'#{value}' "
109
+ else
110
+ string << "#{option} '#{value}' "
111
+ end
112
+ end
113
+ end
114
+
115
+ string.strip
116
+ end
117
+
118
+ def self.cmd(tool, cmd = nil, options = {}, &block)
119
+ options, cmd = cmd, nil if Hash === cmd
120
+
121
+ options = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
122
+ in_content = options.delete(:in)
123
+ stderr = options.delete(:stderr)
124
+ post = options.delete(:post)
125
+ pipe = options.delete(:pipe)
126
+ log = options.delete(:log)
127
+ no_fail = options.delete(:no_fail)
128
+ no_fail = options.delete(:nofail) if no_fail.nil?
129
+ no_wait = options.delete(:no_wait)
130
+ xvfb = options.delete(:xvfb)
131
+ bar = options.delete(:progress_bar)
132
+ save_stderr = options.delete(:save_stderr)
133
+ autojoin = options.delete(:autojoin)
134
+ autojoin = no_wait if autojoin.nil?
135
+
136
+ dont_close_in = options.delete(:dont_close_in)
137
+
138
+ log = true if log.nil?
139
+
140
+ if cmd.nil? && ! Symbol === tool
141
+ cmd = tool
142
+ else
143
+ tool = get_tool(tool)
144
+ if cmd.nil?
145
+ cmd = tool
146
+ else
147
+ cmd = tool + ' ' + cmd
148
+ end
149
+
150
+ end
151
+
152
+ case xvfb
153
+ when TrueClass
154
+ cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
155
+ when String
156
+ cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
157
+ when String
158
+ end
159
+
160
+ if stderr == true
161
+ stderr = Log::HIGH
162
+ end
163
+
164
+ cmd_options = process_cmd_options options
165
+ if cmd =~ /'\{opt\}'/
166
+ cmd.sub!('\'{opt}\'', cmd_options)
167
+ else
168
+ cmd << " " << cmd_options
169
+ end
170
+
171
+ in_content = StringIO.new in_content if String === in_content
172
+
173
+ sin, sout, serr, wait_thr = begin
174
+ Open3.popen3(ENV, cmd)
175
+ rescue
176
+ Log.warn $!.message
177
+ raise ProcessFailed, nil, cmd unless no_fail
178
+ return
179
+ end
180
+ pid = wait_thr.pid
181
+
182
+ Log.debug{"CMD: [#{pid}] #{cmd}" if log}
183
+
184
+ if in_content.respond_to?(:read)
185
+ in_thread = Thread.new(Thread.current) do |parent|
186
+ Thread.current.report_on_exception = false if no_fail
187
+ begin
188
+ begin
189
+ while c = in_content.readpartial(Open::BLOCK_SIZE)
190
+ sin << c
191
+ end
192
+ rescue EOFError
193
+ end
194
+ sin.close unless sin.closed?
195
+
196
+ unless dont_close_in
197
+ in_content.close unless in_content.closed?
198
+ in_content.join if in_content.respond_to? :join
199
+ end
200
+ rescue
201
+ Log.error "Error in CMD [#{pid}] #{cmd}: #{$!.message}" unless no_fail
202
+ raise $!
203
+ end
204
+ end
205
+ else
206
+ in_thread = nil
207
+ sin.close
208
+ end
209
+
210
+ pids = [pid]
211
+
212
+ if pipe
213
+
214
+ ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
215
+
216
+ sout.callback = post if post
217
+
218
+ if (Integer === stderr and log) || bar
219
+ err_thread = Thread.new do
220
+ Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
221
+ begin
222
+ while line = serr.gets
223
+ bar.process(line) if bar
224
+ sout.log = line
225
+ sout.std_err << line if save_stderr
226
+ Log.log "STDERR [#{pid}]: " + line, stderr if log
227
+ end
228
+ serr.close
229
+ rescue
230
+ Log.exception $!
231
+ raise $!
232
+ end
233
+ end
234
+ else
235
+ err_thread = Open.consume_stream(serr, true)
236
+ end
237
+
238
+ sout.threads = [in_thread, err_thread, wait_thr].compact
239
+
240
+ sout
241
+ else
242
+
243
+ if bar
244
+ err = ""
245
+ err_thread = Thread.new do
246
+ while not serr.eof?
247
+ line = serr.gets
248
+ bar.process(line)
249
+ err << line if Integer === stderr and log
250
+ end
251
+ serr.close
252
+ end
253
+ elsif log and Integer === stderr
254
+ err = ""
255
+ err_thread = Thread.new do
256
+ while not serr.eof?
257
+ err << serr.gets
258
+ end
259
+ serr.close
260
+ end
261
+ else
262
+ Open.consume_stream(serr, true)
263
+ #serr.close
264
+ err_thread = nil
265
+ err = ""
266
+ end
267
+
268
+ ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
269
+
270
+ begin
271
+ out = StringIO.new sout.read
272
+ sout.close unless sout.closed?
273
+
274
+ status = wait_thr.value
275
+ if not status.success? and not no_fail
276
+ if !err.empty?
277
+ raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
278
+ else
279
+ raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
280
+ end
281
+ else
282
+ Log.log err, stderr if Integer === stderr and log
283
+ end
284
+ out
285
+ ensure
286
+ post.call if post
287
+ end
288
+ end
289
+ end
290
+
291
+ def self.cmd_pid(*args)
292
+ all_args = *args
293
+
294
+ bar = all_args.last[:progress_bar] if Hash === all_args.last
295
+
296
+ all_args << {} unless Hash === all_args.last
297
+
298
+ level = all_args.last[:log] || 0
299
+ level = 0 if TrueClass === level
300
+ level = 10 if FalseClass === level
301
+ level = level.to_i
302
+
303
+ all_args.last[:log] = true
304
+ all_args.last[:pipe] = true
305
+
306
+ io = cmd(*all_args)
307
+ pid = io.pids.first
308
+
309
+ line = "" if bar
310
+ starting = true
311
+ while c = io.getc
312
+ if starting
313
+ if pid
314
+ Log.logn "STDOUT [#{pid}]: ", level
315
+ else
316
+ Log.logn "STDOUT: ", level
317
+ end
318
+ starting = false
319
+ end
320
+ STDERR << c if Log.severity <= level
321
+ line << c if bar
322
+ if c == "\n"
323
+ bar.process(line) if bar
324
+ starting = true
325
+ line = "" if bar
326
+ end
327
+ end
328
+ begin
329
+ io.join
330
+ bar.remove if bar
331
+ rescue
332
+ bar.remove(true) if bar
333
+ raise $!
334
+ end
335
+
336
+ nil
337
+ end
338
+
339
+ def self.cmd_log(*args)
340
+ cmd_pid(*args)
341
+ nil
342
+ end
343
+
344
+ end
@@ -0,0 +1,259 @@
1
+ require_relative 'indiferent_hash'
2
+
3
+ module AbortedStream
4
+ attr_accessor :exception
5
+ def self.setup(obj, exception = nil)
6
+ obj.extend AbortedStream
7
+ obj.exception = exception
8
+ end
9
+ end
10
+
11
+ module ConcurrentStream
12
+ attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
13
+
14
+ def self.setup(stream, options = {}, &block)
15
+
16
+ threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
17
+ stream.extend ConcurrentStream unless ConcurrentStream === stream
18
+
19
+ stream.threads ||= []
20
+ stream.pids ||= []
21
+ stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
22
+ stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
23
+ stream.autojoin = autojoin unless autojoin.nil?
24
+ stream.no_fail = no_fail unless no_fail.nil?
25
+ stream.std_err = ""
26
+
27
+ stream.pair = pair unless pair.nil?
28
+
29
+ callback = block if block_given?
30
+ if callback
31
+ if stream.callback
32
+ old_callback = stream.callback
33
+ stream.callback = Proc.new do
34
+ old_callback.call
35
+ callback.call
36
+ end
37
+ else
38
+ stream.callback = callback
39
+ end
40
+ end
41
+
42
+ if abort_callback
43
+ if stream.abort_callback
44
+ old_abort_callback = stream.abort_callback
45
+ stream.abort_callback = Proc.new do
46
+ old_abort_callback.call
47
+ abort_callback.call
48
+ end
49
+ else
50
+ stream.abort_callback = abort_callback
51
+ end
52
+ end
53
+
54
+ stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
55
+
56
+ stream.lockfile = lockfile unless lockfile.nil?
57
+
58
+ stream.aborted = false
59
+
60
+ stream
61
+ end
62
+
63
+ def annotate(stream)
64
+ ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
65
+ stream
66
+ end
67
+
68
+ def clear
69
+ @threads = @pids = @callback = @abort_callback = @joined = nil
70
+ end
71
+
72
+ def joined?
73
+ @joined
74
+ end
75
+
76
+ def aborted?
77
+ @aborted
78
+ end
79
+
80
+ def join_threads
81
+ if @threads
82
+ @threads.each do |t|
83
+ next if t == Thread.current
84
+ begin
85
+ t.join
86
+ if Process::Status === t.value
87
+ if ! (t.value.success? || no_fail)
88
+
89
+ if log
90
+ msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
91
+ else
92
+ msg = "Error joining #{self.filename || self.inspect}"
93
+ end
94
+
95
+ raise ConcurrentStreamProcessFailed.new t.pid, msg, self
96
+ end
97
+ end
98
+ rescue Exception
99
+ if no_fail
100
+ Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
101
+ else
102
+ Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
103
+ stream_raise_exception $!
104
+ end
105
+ end
106
+ end
107
+ end
108
+ @threads = []
109
+ end
110
+
111
+ def join_pids
112
+ if @pids and @pids.any?
113
+ @pids.each do |pid|
114
+ begin
115
+ Process.waitpid(pid, Process::WUNTRACED)
116
+ stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
117
+ rescue Errno::ECHILD
118
+ end
119
+ end
120
+ @pids = []
121
+ end
122
+ end
123
+
124
+ def join_callback
125
+ if @callback and not joined?
126
+ begin
127
+ @callback.call
128
+ ensure
129
+ @callback = nil
130
+ end
131
+ end
132
+ end
133
+
134
+ def join
135
+ begin
136
+ join_threads
137
+ join_pids
138
+ join_callback
139
+ close unless closed?
140
+ ensure
141
+ @joined = true
142
+ lockfile.unlock if lockfile and lockfile.locked?
143
+ raise stream_exception if stream_exception
144
+ end
145
+ end
146
+
147
+ def abort_threads(exception = nil)
148
+ return unless @threads and @threads.any?
149
+ name = Log.fingerprint(Thread.current)
150
+ name += " - file:#{filename}" if filename
151
+ Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
152
+
153
+ @threads.each do |t|
154
+ next if t == Thread.current
155
+ Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
156
+ t.raise((exception.nil? ? Aborted.new : exception))
157
+ end
158
+
159
+ @threads.each do |t|
160
+ next if t == Thread.current
161
+ begin
162
+ t.join unless t == Thread.current
163
+ rescue Aborted
164
+ rescue Exception
165
+ Log.debug "Thread (#{name}) exception: #{$!.message}"
166
+ end
167
+ end
168
+ end
169
+
170
+ def abort_pids
171
+ @pids.each do |pid|
172
+ begin
173
+ Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
174
+ Process.kill :INT, pid
175
+ rescue Errno::ESRCH
176
+ end
177
+ end if @pids
178
+ @pids = []
179
+ end
180
+
181
+ def abort(exception = nil)
182
+ self.stream_exception ||= exception
183
+ if @aborted
184
+ Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
185
+ return
186
+ else
187
+ Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
188
+ end
189
+ AbortedStream.setup(self, exception)
190
+ @aborted = true
191
+ begin
192
+ @abort_callback.call exception if @abort_callback
193
+
194
+ abort_threads(exception)
195
+ abort_pids
196
+
197
+ @callback = nil
198
+ @abort_callback = nil
199
+
200
+ if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
201
+ Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
202
+ @pair.abort exception
203
+ end
204
+ ensure
205
+ close unless closed?
206
+
207
+ if lockfile and lockfile.locked?
208
+ lockfile.unlock
209
+ end
210
+ end
211
+ end
212
+
213
+ def close(*args)
214
+ if autojoin
215
+ begin
216
+ super(*args)
217
+ rescue
218
+ Log.exception $!
219
+ self.abort
220
+ self.join
221
+ stream_raise_exception $!
222
+ ensure
223
+ self.join if self.closed? or self.eof?
224
+ end
225
+ else
226
+ super(*args)
227
+ end
228
+ end
229
+
230
+ def read(*args)
231
+ begin
232
+ super(*args)
233
+ ensure
234
+ begin
235
+ close unless closed?
236
+ rescue Exception
237
+ raise $! if ConcurrentStreamProcessFailed === $!
238
+ end if autojoin && ! closed? && eof?
239
+ end
240
+ end
241
+
242
+ def add_callback(&block)
243
+ old_callback = callback
244
+ @callback = Proc.new do
245
+ old_callback.call if old_callback
246
+ block.call
247
+ end
248
+ end
249
+
250
+ def stream_raise_exception(exception)
251
+ threads.each do |thread|
252
+ thread.raise exception
253
+ end
254
+ self.stream_exception = exception
255
+
256
+ self.abort
257
+ end
258
+
259
+ end
@@ -13,6 +13,12 @@ end
13
13
 
14
14
  class Aborted < StandardError; end
15
15
 
16
+ class ParameterException < ScoutException; end
17
+ class MissingParameterException < ParameterException
18
+ def initialize(parameter)
19
+ super("Missing parameter '#{parameter}'")
20
+ end
21
+ end
16
22
  class ProcessFailed < StandardError;
17
23
  attr_accessor :pid, :msg
18
24
  def initialize(pid = Process.pid, msg = nil)
@@ -41,7 +47,13 @@ end
41
47
 
42
48
  class OpenURLError < StandardError; end
43
49
 
44
- class DontClose < Exception; end
50
+ class DontClose < Exception
51
+ attr_accessor :payload
52
+ def initialize(payload = nil)
53
+ @payload = payload
54
+ end
55
+ end
56
+
45
57
 
46
58
  class KeepLocked < Exception
47
59
  attr_accessor :payload
@@ -57,13 +69,9 @@ class KeepBar < Exception
57
69
  end
58
70
  end
59
71
 
60
- #class ParameterException < ScoutException; end
72
+ class LockInterrupted < TryAgain; end
73
+
61
74
  #
62
- #class MissingParameterException < ParameterException
63
- # def initialize(parameter)
64
- # super("Missing parameter '#{parameter}'")
65
- # end
66
- #end
67
75
  #class ClosedStream < StandardError; end
68
76
  #class OpenGzipError < StandardError; end
69
77
  #
@@ -76,7 +84,6 @@ end
76
84
  #end
77
85
  #
78
86
  #class SemaphoreInterrupted < TryAgain; end
79
- #class LockInterrupted < TryAgain; end
80
87
  #
81
88
  #class RemoteServerError < StandardError; end
82
89
  #
@@ -1,32 +1,24 @@
1
1
  module IndiferentHash
2
2
  def self.add_defaults(options, defaults = {})
3
- options ||= {}
3
+ options = string2hash options if String === options
4
4
  IndiferentHash.setup(options)
5
- case
6
- when Hash === options
7
- new_options = options.dup
8
- when String === options
9
- new_options = string2hash options
10
- else
11
- raise "Format of '#{options.inspect}' not understood. It should be a hash"
12
- end
5
+
6
+ defaults = string2hash defaults if String === defaults
13
7
 
14
8
  defaults.each do |key, value|
15
9
  next if options.include? key
16
10
 
17
- new_options[key] = value
11
+ options[key] = value
18
12
  end
19
13
 
20
- new_options
21
-
22
- options.replace new_options
14
+ options
23
15
  end
24
16
 
25
17
  def self.process_options(hash, *keys)
26
18
  IndiferentHash.setup(hash)
27
19
 
28
20
  defaults = keys.pop if Hash === keys.last
29
- hahs = IndiferentHash.add_defaults hash, defaults if defaults
21
+ hash = IndiferentHash.add_defaults hash, defaults if defaults
30
22
 
31
23
  if keys.length == 1
32
24
  hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s)
@@ -102,7 +94,7 @@ module IndiferentHash
102
94
  options = {}
103
95
 
104
96
  string.split('#').each do |str|
105
- key, sep, value = str.partition "="
97
+ key, _, value = str.partition "="
106
98
 
107
99
  key = key[1..-1].to_sym if key[0] == ":"
108
100
 
@@ -114,17 +106,7 @@ module IndiferentHash
114
106
  options[key] = value.to_f and next if value =~ /^\d*\.\d+$/
115
107
  options[key] = true and next if value == "true"
116
108
  options[key] = false and next if value == "false"
117
- options[key] = value and next
118
-
119
- options[key] = begin
120
- saved_safe = $SAFE
121
- $SAFE = 0
122
- eval(value)
123
- rescue Exception
124
- value
125
- ensure
126
- $SAFE = saved_safe
127
- end
109
+ options[key] = value
128
110
  end
129
111
 
130
112
  IndiferentHash.setup(options)