rbbt-util 5.10.2 → 5.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,9 +28,11 @@ class RbbtProcessQueue
28
28
 
29
29
  rescue ClosedStream
30
30
  rescue Aborted
31
- Log.exception $!
31
+ Log.error "Worker #{Process.pid} aborted"
32
+ Kernel.exit! -1
32
33
  rescue Exception
33
34
  @callback_queue.push($!) if @callback_queue
35
+ Kernel.exit! -1
34
36
  ensure
35
37
  @callback_queue.close_write if @callback_queue
36
38
  end
data/lib/rbbt/util/log.rb CHANGED
@@ -72,6 +72,7 @@ module Log
72
72
  end
73
73
  end
74
74
 
75
+ LOG_MUTEX = Mutex.new
75
76
  def self.log(message = nil, severity = MEDIUM, &block)
76
77
  return if severity < self.severity
77
78
  message ||= block.call if block_given?
@@ -85,8 +86,10 @@ module Log
85
86
  message = "" << highlight << message << color(0) if severity >= INFO
86
87
  str = prefix << " " << message
87
88
 
88
- STDERR.puts str
89
- logfile.puts str unless logfile.nil?
89
+ LOG_MUTEX.synchronize do
90
+ STDERR.puts str
91
+ logfile.puts str unless logfile.nil?
92
+ end
90
93
  end
91
94
 
92
95
  def self.debug(message = nil, &block)
@@ -194,14 +197,6 @@ def iii(message, file = $stdout)
194
197
  Log.info{""}
195
198
  end
196
199
 
197
- def www(message, file = $stdout)
198
- stack = caller
199
- Log.warn{"#{Log.color :cyan, "INFO:"} " << stack.first}
200
- Log.warn{""}
201
- Log.warn{"=> " << message.inspect}
202
- Log.warn{""}
203
- end
204
-
205
200
  def eee(message, file = $stdout)
206
201
  stack = caller
207
202
  Log.error{"#{Log.color :cyan, "INFO:"} " << stack.first}
@@ -21,6 +21,12 @@ class FieldNotFoundError < Exception;end
21
21
  class Aborted < Exception; end
22
22
  class TryAgain < Exception; end
23
23
  class ClosedStream < Exception; end
24
+ class KeepLocked < Exception
25
+ attr_accessor :payload
26
+ def initialize(payload)
27
+ @payload = payload
28
+ end
29
+ end
24
30
 
25
31
  module LaterString
26
32
  def to_s
@@ -31,27 +37,35 @@ end
31
37
  module ConcurrentStream
32
38
  attr_accessor :threads, :pids, :callback, :filename
33
39
 
34
- def consume
35
- Thread.pass while IO.select([self], nil, nil).nil? #if IO === self
36
- while block = self.read(2048)
37
- Thread.pass while IO.select([self], nil, nil).nil? # if IO === self
40
+ def join
41
+
42
+ if @threads and @threads.any?
43
+ @threads.each do |t|
44
+ t.join
45
+ end
46
+ @threads = []
47
+ end
48
+
49
+ if @pids and @pids.any?
50
+ @pids.each do |pid|
51
+ begin
52
+ Process.waitpid(pid, Process::WUNTRACED)
53
+ raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
54
+ rescue Errno::ECHILD
55
+ end
56
+ end
57
+ @pids = []
38
58
  end
39
- end
40
59
 
41
- def join
42
- filename = self.respond_to?(:filename)? self.filename : :none
43
60
  if @callback
44
61
  @callback.call
62
+ @callback = nil
45
63
  end
46
- @threads.each{|t| t.join } if @threads
47
-
48
- @pids.each do |pid|
49
- begin
50
- Process.waitpid(pid, Process::WUNTRACED)
51
- raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
52
- rescue Errno::ECHILD
53
- end
54
- end if @pids
64
+ end
65
+
66
+ def abort
67
+ @threads.each{|t| t.raise Aborted.new } if @threads
68
+ @pids.each{|pid| Process.kill :INT, pid } if @pids
55
69
  end
56
70
 
57
71
  def self.setup(stream, options = {}, &block)
@@ -116,9 +130,11 @@ module Misc
116
130
  end
117
131
  end
118
132
 
119
- def self.open_pipe(do_fork = false, other_stream = nil)
133
+ def self.open_pipe(do_fork = false, close = true)
120
134
  raise "No block given" unless block_given?
135
+
121
136
  sout, sin = Misc.pipe
137
+
122
138
  if do_fork
123
139
  parent_pid = Process.pid
124
140
  pid = Process.fork {
@@ -131,22 +147,23 @@ module Misc
131
147
  Process.kill :INT, parent_pid
132
148
  Kernel.exit! -1
133
149
  ensure
134
- Misc.release_pipes(sin)
150
+ sin.close unless sin.closed?
135
151
  end
136
152
  Kernel.exit! 0
137
153
  }
138
- Misc.release_pipes(sin)
154
+ sin.close if close
155
+ ConcurrentStream.setup sout, :pids => [pid]
139
156
  else
140
157
  thread = Thread.new(Thread.current) do |parent|
141
158
  begin
142
159
  yield sin
143
160
  rescue
144
- Log.exception $!
145
161
  parent.raise $!
146
162
  ensure
147
- Misc.release_pipes(sin)
163
+ sin.close if close
148
164
  end
149
165
  end
166
+ ConcurrentStream.setup sout, :threads => [thread]
150
167
  end
151
168
  sout
152
169
  end
@@ -161,20 +178,24 @@ module Misc
161
178
  stream_out2.close
162
179
  begin
163
180
  filename = stream.respond_to?(:filename)? stream.filename : nil
181
+ skip1 = skip2 = false
164
182
  while block = stream.read(2048)
165
- begin stream_in1.write block; rescue Exception; Log.exception $! end
166
- begin stream_in2.write block; rescue Exception; Log.exception $! end
183
+ begin stream_in1.write block; rescue Exception; Log.exception $!; skip1 = true end unless skip1
184
+ begin stream_in2.write block; rescue Exception; Log.exception $!; skip2 = true end unless skip2
167
185
  end
186
+ raise "Error writing in stream_in2" if skip2
187
+ raise "Error writing in stream_in2" if skip2
188
+ rescue Aborted
189
+ stream.abort if stream.respond_to? :abort
190
+ raise $!
168
191
  rescue IOError
169
192
  Log.exception $!
170
193
  rescue Exception
171
194
  Log.exception $!
172
195
  ensure
173
- if stream.respond_to? :join
174
- stream.join
175
- end
176
- Misc.release_pipes(stream_in1)
177
- Misc.release_pipes(stream_in2)
196
+ stream_in1.close
197
+ stream_in2.close
198
+ stream.join if stream.respond_to? :join
178
199
  end
179
200
  end
180
201
  stream.close
@@ -195,21 +216,23 @@ module Misc
195
216
  splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
196
217
  begin
197
218
  filename = stream.respond_to?(:filename)? stream.filename : nil
219
+ skip1 = skip2 = false
198
220
  while block = stream.read(2048)
199
- begin stream_in1.write block; rescue Exception; Log.exception $! end
200
- begin stream_in2.write block; rescue Exception; Log.exception $! end
221
+ begin stream_in1.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip1 = true end unless skip1
222
+ begin stream_in2.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip2 = true end unless skip2
201
223
  end
224
+ rescue Aborted
225
+ stream.abort if stream.respond_to? :abort
226
+ raise $!
202
227
  rescue IOError
203
228
  Log.exception $!
204
229
  rescue Exception
205
230
  Log.exception $!
206
231
  parent.raise $!
207
232
  ensure
208
- if stream.respond_to? :join
209
- stream.join
210
- end
211
- Misc.release_pipes(stream_in1)
212
- Misc.release_pipes(stream_in2)
233
+ stream_in1.close
234
+ stream_in2.close
235
+ stream.join if stream.respond_to? :join
213
236
  end
214
237
  end
215
238
 
@@ -220,7 +243,7 @@ module Misc
220
243
  end
221
244
 
222
245
  class << self
223
- alias tee_stream tee_stream_fork
246
+ alias tee_stream tee_stream_thread
224
247
  end
225
248
 
226
249
  def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
@@ -494,6 +517,8 @@ module Misc
494
517
  case obj
495
518
  when nil
496
519
  "nil"
520
+ when (defined? Step and Step)
521
+ obj.path || Misc.fingerprint([obj.task.name, obj.inputs])
497
522
  when TrueClass
498
523
  "true"
499
524
  when FalseClass
@@ -1276,8 +1301,8 @@ end
1276
1301
  @hostanem ||= `hostname`.strip
1277
1302
  end
1278
1303
 
1279
- def self.lock(file, *args)
1280
- return yield file, *args if file.nil?
1304
+ def self.lock(file, unlock = true)
1305
+ return yield if file.nil?
1281
1306
  FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists? File.dirname(File.expand_path(file))
1282
1307
 
1283
1308
  res = nil
@@ -1287,27 +1312,34 @@ end
1287
1312
 
1288
1313
  begin
1289
1314
  Misc.insist 3 do
1290
- if File.exists? lockfile and
1291
- Misc.hostname == (info = Open.open(lockfile){|f| YAML.load(f) })["host"] and
1315
+ if File.exists? lock_path and
1316
+ Misc.hostname == (info = Open.open(lock_path){|f| YAML.load(f) })["host"] and
1292
1317
  info["pid"] and not Misc.pid_exists?(info["pid"])
1293
1318
 
1294
- Log.info("Removing lockfile: #{lockfile}. This pid #{Process.pid}. Content: #{info.inspect}")
1295
- FileUtils.rm lockfile
1319
+ Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
1320
+ FileUtils.rm lock_path
1296
1321
  end
1297
1322
  end
1298
1323
  rescue
1299
- Log.warn("Error checking lockfile #{lockfile}: #{$!.message}. Removing. Content: #{begin Open.read(lockfile) rescue "Could not open file" end}")
1300
- FileUtils.rm lockfile if File.exists?(lockfile)
1324
+ Log.warn("Error checking lockfile #{lock_path}: #{$!.message}. Removing. Content: #{begin Open.read(lock_path) rescue "Could not open file" end}")
1325
+ FileUtils.rm lock_path if File.exists?(lock_path)
1301
1326
  lockfile = Lockfile.new(lock_path)
1302
1327
  retry
1303
1328
  end
1304
1329
 
1305
1330
  begin
1306
- lockfile.lock do
1307
- res = yield file, *args
1331
+ lockfile.lock
1332
+ res = yield lockfile
1333
+ rescue Lockfile::StolenLockError
1334
+ unlock = false
1335
+ rescue KeepLocked
1336
+ unlock = false
1337
+ res = $!.payload
1338
+ ensure
1339
+ if unlock and lockfile.locked?
1340
+ lockfile.unlock
1341
+ FileUtils.rm lock_path if File.exists? lock_path
1308
1342
  end
1309
- rescue Interrupt
1310
- raise $!
1311
1343
  end
1312
1344
 
1313
1345
  res
@@ -1315,7 +1347,6 @@ end
1315
1347
 
1316
1348
 
1317
1349
  LOCK_REPO_SERIALIZER=Marshal
1318
-
1319
1350
  def self.lock_in_repo(repo, key, *args)
1320
1351
  return yield file, *args if repo.nil? or key.nil?
1321
1352
 
@@ -1402,17 +1433,17 @@ end
1402
1433
 
1403
1434
  def self.sensiblewrite(path, content = nil, &block)
1404
1435
  return if File.exists? path
1405
- Misc.lock path + '.sensible_write' do
1436
+ tmp_path = path + '.sensible_write'
1437
+ Misc.lock tmp_path do
1406
1438
  if not File.exists? path
1439
+ FileUtils.rm_f tmp_path if File.exists? tmp_path
1407
1440
  begin
1408
- tmp_path = path + '.sensible_write'
1409
1441
  case
1410
1442
  when block_given?
1411
1443
  File.open(tmp_path, 'w', &block)
1412
1444
  when String === content
1413
1445
  File.open(tmp_path, 'w') do |f| f.write content end
1414
1446
  when (IO === content or StringIO === content or File === content)
1415
- #Thread.pass while IO.select([content], nil, nil, 1) if IO === content
1416
1447
  File.open(tmp_path, 'w') do |f|
1417
1448
  while block = content.read(2048);
1418
1449
  f.write block
@@ -1488,6 +1519,8 @@ end
1488
1519
  str << remove_long_items(v)
1489
1520
  when Array === v
1490
1521
  str << k.to_s << "=>[" << v * "," << "]"
1522
+ when File === v
1523
+ str << k.to_s << "=>[File:" << v.path << "]"
1491
1524
  else
1492
1525
  v_ins = v.inspect
1493
1526
 
data/lib/rbbt/workflow.rb CHANGED
@@ -138,7 +138,8 @@ module Workflow
138
138
  if wf_name =~ /::\w+$/
139
139
  clean_name = wf_name.sub(/::.*/,'')
140
140
  Log.info{"Looking for '#{wf_name}' in '#{clean_name}'"}
141
- wf_name = clean_name
141
+ require_workflow clean_name
142
+ return Misc.string2const Misc.camel_case(wf_name)
142
143
  end
143
144
 
144
145
  Log.info{"Loading workflow #{wf_name}"}
@@ -24,9 +24,14 @@ class Step
24
24
  else
25
25
  [dependencies]
26
26
  end
27
+ @mutex = Mutex.new
27
28
  @inputs = inputs || []
28
29
  end
29
30
 
31
+ def task_name
32
+ @task.name
33
+ end
34
+
30
35
  def path
31
36
  @path = Misc.sanitize_filename(Path.setup(@path.call)) if Proc === @path
32
37
  @path
@@ -43,10 +48,9 @@ class Step
43
48
  attr_accessor :relay_step
44
49
  alias original_log log
45
50
  def log(status, message = nil)
46
- Log.with_severity 10 do
47
- original_log(status, message)
48
- end
49
- relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : [task.name.to_s, message] * ">")
51
+ self.status = status
52
+ message message
53
+ relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message )
50
54
  end
51
55
  end
52
56
  end
@@ -70,7 +74,10 @@ class Step
70
74
  else
71
75
  value.read
72
76
  end
77
+ rescue Exception
78
+ value.abort if value.respond_to? :abort
73
79
  ensure
80
+ value.close unless value.closed?
74
81
  value.join if value.respond_to? :join
75
82
  end
76
83
  when (not defined? Entity or description.nil? or not Entity.formats.include? description)
@@ -88,6 +95,17 @@ class Step
88
95
  end
89
96
  end
90
97
 
98
+ def get_stream
99
+ @mutex.synchronize do
100
+ begin
101
+ res = @result
102
+ IO === res ? res : nil
103
+ ensure
104
+ @result = nil
105
+ end
106
+ end
107
+ end
108
+
91
109
  def _exec
92
110
  @exec = true if @exec.nil?
93
111
  @task.exec_in((bindings ? bindings : self), *@inputs)
@@ -100,13 +118,18 @@ class Step
100
118
  end
101
119
 
102
120
  def join
103
- case @result
104
- when IO
105
- while @result.read 2048; Thread.pass end unless @result.closed? or @result.eof?
106
- @result.join if @result.respond_to? :join
107
- @result = nil
121
+ stream = get_stream
122
+ begin
123
+ stream.read if stream
124
+ rescue
125
+ stream.abort if stream.respond_to? :abort
126
+ raise $!
127
+ ensure
128
+ stream.join if stream.respond_to? :join
108
129
  end
109
130
 
131
+ dependencies.each{|dep| dep.join }
132
+
110
133
  if @pid.nil?
111
134
  self
112
135
  else
@@ -140,12 +163,15 @@ class Step
140
163
 
141
164
  set_info :dependencies, dependencies.collect{|dep| [dep.task.name, dep.name]}
142
165
  log(:preparing, "Preparing job")
166
+ seen_deps = []
143
167
  dependencies.each{|dependency|
144
168
  Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task.name.to_s || ""}"
145
169
  begin
170
+ next if seen_deps.include? dependency.path
146
171
  dependency.relay_log self
147
172
  dependency.clean if not dependency.done? and dependency.error?
148
- dependency.run true
173
+ dependency.run true unless dependency.done?
174
+ seen_deps.concat dependency.rec_dependencies.collect{|d| d.path}
149
175
  rescue Exception
150
176
  backtrace = $!.backtrace
151
177
  set_info :backtrace, backtrace
@@ -157,7 +183,7 @@ class Step
157
183
  set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
158
184
 
159
185
  set_info :started, (start_time = Time.now)
160
- log :started, "#{Log.color :magenta, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
186
+ log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
161
187
 
162
188
  begin
163
189
  result = _exec
@@ -191,26 +217,33 @@ class Step
191
217
  end
192
218
 
193
219
  case result
194
- when IO, StringIO
220
+ when IO
195
221
  log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
196
222
  ConcurrentStream.setup result do
197
- eee 1
198
- set_info :done, (done_time = Time.now)
199
- set_info :time_elapsed, (time_elapsed = done_time - start_time)
200
- log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
223
+ begin
224
+ set_info :done, (done_time = Time.now)
225
+ set_info :time_elapsed, (time_elapsed = done_time - start_time)
226
+ log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
227
+ rescue
228
+ Log.exception $!
229
+ end
201
230
  end
202
231
  when TSV::Dumper
203
232
  log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
204
233
  ConcurrentStream.setup result.stream do
205
- set_info :done, (done_time = Time.now)
206
- set_info :done, (done_time = Time.now)
207
- set_info :time_elapsed, (time_elapsed = done_time - start_time)
208
- log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
234
+ begin
235
+ set_info :done, (done_time = Time.now)
236
+ set_info :done, (done_time = Time.now)
237
+ set_info :time_elapsed, (time_elapsed = done_time - start_time)
238
+ log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
239
+ rescue
240
+ Log.exception $!
241
+ end
209
242
  end
210
243
  else
211
244
  set_info :done, (done_time = Time.now)
212
245
  set_info :time_elapsed, (time_elapsed = done_time - start_time)
213
- log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
246
+ log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
214
247
  end
215
248
 
216
249
  result
@@ -232,9 +265,13 @@ class Step
232
265
  FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
233
266
  begin
234
267
  res = run(true)
235
- io = res.result if IO === res.result
236
- io = res.result.stream if TSV::Dumper === res.result
237
- while not io.eof?; io.read(2048); end if io
268
+ io = get_stream
269
+ #io = res.result if IO === res.result
270
+ #io = res.result.stream if TSV::Dumper === res.result
271
+ if IO === io
272
+ io.read
273
+ io.join if io.respond_to? :join
274
+ end
238
275
  rescue Aborted
239
276
  Log.debug{"Forked process aborted: #{path}"}
240
277
  log :aborted, "Aborted"
@@ -275,13 +312,20 @@ class Step
275
312
 
276
313
  def abort
277
314
  @pid ||= info[:pid]
278
- if @pid.nil? and info[:forked]
315
+
316
+ return true unless info[:forked]
317
+
318
+ case @pid
319
+ when nil
279
320
  Log.medium "Could not abort #{path}: no pid"
280
321
  false
322
+ when Process.pid
323
+ Log.medium "Could not abort #{path}: same process"
324
+ false
281
325
  else
282
326
  Log.medium "Aborting #{path}: #{ @pid }"
283
327
  begin
284
- Process.kill("KILL", @pid) unless Process.pid == @pid
328
+ Process.kill("KILL", @pid)
285
329
  Process.waitpid @pid
286
330
  rescue Exception
287
331
  Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
@@ -289,6 +333,7 @@ class Step
289
333
  log(:aborted, "Job aborted by user")
290
334
  true
291
335
  end
336
+ log(:aborted, "Job aborted by user")
292
337
  end
293
338
 
294
339
  def child(&block)
@@ -335,9 +380,12 @@ class Step
335
380
  # placed. In that case, do not consider its dependencies
336
381
  return [] if self.done? and not Open.exists? self.info_file
337
382
 
338
- dependencies.collect{|step|
383
+ return [] if dependencies.nil? or dependencies.empty?
384
+ new_dependencies = dependencies.collect{|step|
339
385
  step.rec_dependencies
340
- }.flatten.concat dependencies
386
+ }.flatten
387
+
388
+ dependencies + new_dependencies
341
389
  end
342
390
 
343
391
  def recursive_clean
@@ -350,6 +398,8 @@ class Step
350
398
  end
351
399
 
352
400
  def step(name)
353
- rec_dependencies.select{|step| step.task.name.to_sym == name.to_sym}.first
401
+ rec_dependencies.select do |step|
402
+ step.task_name.to_sym == name.to_sym
403
+ end.first
354
404
  end
355
405
  end