rbbt-util 5.11.5 → 5.11.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ec4bc56a7d0d260d93be40ee61d23c08f58d5a8
-  data.tar.gz: 6f51a747302e936ed1e5953e929e5120fa5caa89
+  metadata.gz: 2230976fc29ab61ab320c1c007521196acaa9b5f
+  data.tar.gz: 60ea71f47e8afa071cdc84f7de115487ec0b1ac5
 SHA512:
-  metadata.gz: db5a97b2787f182444de536059beb027d038573c0c3ac3196ec44ce18d2561fa3204a38229d3054e5c92594309fc4a488cc6e9116e370c4f2d383b28a8ed9e71
-  data.tar.gz: 8a4fc523ae9266ceea9964f26f85834a8f5c0e0899a523bb938e2e813da191651215ec356432ee7b467f39b470328b32dda911d1b2cd9bd757fa467ad14d54ce
+  metadata.gz: 8f7bbea5678b31d6f49ce13f94dd0b7d3271ae6e187ea837265e202ee90e95c3cb8af27ae5ba8c05b3dd8fa5d0d6eca14ae7870685990ae391fdfb1273ddbac8
+  data.tar.gz: d4746fa8d4393a9f7c399ae766b1881bc41568793ed50722a4b4f101a54962146fead34931b393c8ca44b3ea17057f59b21f505d7336939b777ede2983f9ddc9

data/lib/rbbt/persist.rb

@@ -292,12 +292,14 @@ module Persist
 
     return (persist_options[:repo] || Persist::MEMORY)[persist_options[:file]] ||= yield if type ==:memory and persist_options[:file] and persist_options[:persist] and persist_options[:persist] != :update
 
-    if FalseClass != persist_options[:persist]
+    if FalseClass == persist_options[:persist]
+      yield
+    else
       other_options = Misc.process_options persist_options, :other
       path = persistence_path(name, persist_options, other_options || {})
 
       case 
-      when type.to_sym === :memory
+      when type.to_sym == :memory
         repo = persist_options[:repo] || Persist::MEMORY
         repo[path] ||= yield
 
@@ -410,7 +412,14 @@ module Persist
                 res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
                 ConcurrentStream.setup res do
                   begin
-                    lockfile.unlock
+                    lockfile.unlock if lockfile.locked?
+                  rescue
+                    Log.warn "Lockfile exception: " << $!.message
+                  end
+                end
+                res.abort_callback = Proc.new do
+                  begin
+                    lockfile.unlock if lockfile.locked?
                   rescue
                     Log.warn "Lockfile exception: " << $!.message
                   end
@@ -425,6 +434,13 @@ module Persist
                     Log.warn "Lockfile exception: " << $!.message
                   end
                 end
+                res.abort_callback = Proc.new do
+                  begin
+                    lockfile.unlock
+                  rescue
+                    Log.warn "Lockfile exception: " << $!.message
+                  end
+                end
                 raise KeepLocked.new res 
               end
             end
@@ -440,21 +456,19 @@ module Persist
                       else
                         res.read
                       end
+                res.join if res.respond_to? :join
               rescue
                 res.abort if res.respond_to? :abort
                 raise $!
-              ensure
-                res.join if res.respond_to? :join
               end
             when TSV::Dumper
               begin
                 io = res.stream
                 res = TSV.open(io)
+                io.join if io.respond_to? :join
               rescue
                 io.abort if io.respond_to? :abort
                 raise $!
-              ensure
-                io.join if io.respond_to? :join
               end
             end
 
@@ -472,8 +486,6 @@ module Persist
         end
       end
 
-    else
-      yield
     end
   end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -83,13 +83,18 @@ module TSV
   def self.traverse_io(io, options = {}, &block)
     filename = io.filename if io.respond_to? :filename
     callback = Misc.process_options options, :callback
-    if callback
-      TSV::Parser.traverse(io, options) do |k,v|
-        res = yield k, v
-        callback.call res
+    begin
+      if callback
+        TSV::Parser.traverse(io, options) do |k,v|
+          res = yield k, v
+          callback.call res
+        end
+      else
+        TSV::Parser.traverse(io, options, &block)
       end
-    else
-      TSV::Parser.traverse(io, options, &block)
+    rescue
+      Log.error "Traverse IO error"
+      raise $!
     end
   end
 
@@ -305,6 +310,8 @@ module TSV
       rescue Exception
         Log.exception $!
         parent.raise $!
+        stream = obj_stream(into)
+        stream.abort if stream and stream.respond_to? :abort
       end
     end
     ConcurrentStream.setup(obj_stream(into), :threads => thread)

data/lib/rbbt/util/misc.rb

@@ -546,37 +546,6 @@ end
     res
   end
 
-  def self.sensiblewrite(path, content = nil, &block)
-    return if File.exists? path
-    tmp_path = path + '.sensible_write'
-    Misc.lock tmp_path  do
-      if not File.exists? path
-        FileUtils.rm_f tmp_path if File.exists? tmp_path
-        begin
-          case
-          when block_given?
-            File.open(tmp_path, 'w', &block)
-          when String === content
-            File.open(tmp_path, 'w') do |f| f.write content end
-          when (IO === content or StringIO === content or File === content)
-            File.open(tmp_path, 'w') do |f|  
-              while block = content.read(2048); 
-                f.write block
-              end  
-            end
-          else
-            File.open(tmp_path, 'w') do |f|  end
-          end
-          FileUtils.mv tmp_path, path
-        rescue Exception
-          Log.error "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
-          FileUtils.rm_f tmp_path if File.exists? tmp_path
-          FileUtils.rm_f path if File.exists? path
-          raise $!
-        end
-      end
-    end
-  end
 
   def self.add_defaults(options, defaults = {})
     case

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -1,19 +1,48 @@
 module ConcurrentStream
   attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
 
+  def self.setup(stream, options = {}, &block)
+    threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
+    stream.extend ConcurrentStream unless ConcurrentStream === stream
+
+    stream.threads ||= []
+    stream.pids ||= []
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
+
+    callback = block if block_given?
+    if stream.callback and callback
+      old_callback = stream.callback
+      stream.callback = Proc.new do
+        old_callback.call
+        callback.call
+      end
+    else
+      stream.callback = callback
+    end
+
+    stream.filename = filename unless filename.nil?
+
+    stream
+  end
+
   def joined?
     @joined
   end
 
-  def join
-
+  def join_threads
     if @threads and @threads.any?
       @threads.each do |t| 
-        t.join unless t == Thread.current
+        begin
+        ensure
+          t.join unless t == Thread.current
+        end
       end
       @threads = []
     end
+  end
 
+  def join_pids
     if @pids and @pids.any?
       @pids.each do |pid| 
         begin
@@ -24,46 +53,38 @@ module ConcurrentStream
       end 
       @pids = []
     end
+  end
 
+  def join_callback
     if @callback and not joined?
       @callback.call
       @callback = nil
     end
+  end
+
+  def join
+    join_threads
+    join_pids
+
+    join_callback
 
     @joined = true
   end
 
-  def abort
-    @threads.each{|t| t.raise Aborted.new unless t = Thread.current } if @threads
-    @threads.each{|t| t.join unless t = Thread.current } if @threads
+  def abort_threads
+    @threads.each{|t| t.raise Aborted.new unless t == Thread.current } if @threads
+  end
+
+  def abort_pids
     @pids.each{|pid| Process.kill :INT, pid } if @pids
-    @pids.each{|pid| Process.waitpid pid } if @pids
+  end
+
+  def abort
+    abort_threads
+    abort_pids
     @abort_callback.call if @abort_callback
     @abort_callback = nil
+    @callback = nil
   end
 
-  def self.setup(stream, options = {}, &block)
-    threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
-    stream.extend ConcurrentStream unless ConcurrentStream === stream
-
-    stream.threads ||= []
-    stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
-    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
-
-    callback = block if block_given?
-    if stream.callback and callback
-      old_callback = stream.callback
-      stream.callback = Proc.new do
-        old_callback.call
-        callback.call
-      end
-    else
-      stream.callback = callback
-    end
-
-    stream.filename = filename unless filename.nil?
-
-    stream
-  end
 end

data/lib/rbbt/util/misc/lock.rb

@@ -3,14 +3,14 @@ Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
 module Misc
 
   LOCK_MUTEX = Mutex.new
-  def self.lock(file, unlock = true)
+  def self.lock(file, unlock = true, options = {})
     return yield if file.nil?
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
 
     res = nil
 
     lock_path = File.expand_path(file + '.lock')
-    lockfile = Lockfile.new(lock_path)
+    lockfile = Lockfile.new(lock_path, options)
 
     hostname = Misc.hostname
     LOCK_MUTEX.synchronize do
@@ -28,7 +28,7 @@ module Misc
             end
           rescue Exception
             FileUtils.rm lock_path if File.exists? lock_path
-            lockfile = Lockfile.new(lock_path) unless File.exists? lock_path
+            lockfile = Lockfile.new(lock_path, options) unless File.exists? lock_path
             raise $!
           end
         end
@@ -43,6 +43,9 @@ module Misc
     rescue KeepLocked
       unlock = false
       res = $!.payload
+    rescue Exception
+      lockfile.unlock if lockfile.locked?
+      raise $!
     ensure
       if unlock and lockfile.locked?
         lockfile.unlock

data/lib/rbbt/util/misc/pipes.rb

@@ -1,5 +1,12 @@
 module Misc
 
+  class << self
+    attr_accessor :sensiblewrite_dir
+    def sensiblewrite_dir
+      @sensiblewrite_dir = Rbbt.tmp.sensiblewrite
+    end
+  end
+
   PIPE_MUTEX = Mutex.new
 
   OPEN_PIPE_IN = []
@@ -21,7 +28,6 @@ module Misc
     end
   end
 
-
   def self.purge_pipes(*save)
     PIPE_MUTEX.synchronize do
       OPEN_PIPE_IN.each do |pipe|
@@ -43,12 +49,11 @@ module Misc
         sout.close
         begin
           yield sin
+          sin.close if close and not sin.closed? 
         rescue
           Log.exception $!
           Process.kill :INT, parent_pid
           Kernel.exit! -1
-        ensure
-          sin.close if close and not sin.closed? 
         end
         Kernel.exit! 0
       }
@@ -58,10 +63,9 @@ module Misc
       thread = Thread.new(Thread.current) do |parent|
         begin
           yield sin
+          sin.close if close and not sin.closed?
         rescue
           parent.raise $!
-        ensure
-          sin.close if close and not sin.closed?
         end
       end
       ConcurrentStream.setup sout, :threads => [thread]
@@ -84,8 +88,11 @@ module Misc
           begin stream_in1.write block; rescue Exception;  Log.exception $!; skip1 = true end unless skip1 
           begin stream_in2.write block; rescue Exception;  Log.exception $!; skip2 = true end unless skip2 
         end
+        raise "Error writing in stream_in1" if skip1
         raise "Error writing in stream_in2" if skip2
-        raise "Error writing in stream_in2" if skip2
+        stream.join if stream.respond_to? :join
+        stream_in1.close 
+        stream_in2.close 
       rescue Aborted
         stream.abort if stream.respond_to? :abort
         raise $!
@@ -93,16 +100,11 @@ module Misc
         Log.exception $!
       rescue Exception
         Log.exception $!
-      ensure
-        stream_in1.close 
-        stream_in2.close 
-        stream.join if stream.respond_to? :join
       end
     end
     stream.close
     stream_in1.close
     stream_in2.close
-    #stream.join if stream.respond_to? :join
 
     ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
     ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
@@ -122,18 +124,17 @@ module Misc
           begin stream_in1.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip1 = true end unless skip1 
           begin stream_in2.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip2 = true end unless skip2 
         end
+        stream_in1.close 
+        stream_in2.close 
+        stream.join if stream.respond_to? :join
       rescue Aborted
         stream.abort if stream.respond_to? :abort
-        raise $!
+        parent.raise $!
       rescue IOError
         Log.exception $!
       rescue Exception
         Log.exception $!
         parent.raise $!
-      ensure
-        stream_in1.close 
-        stream_in2.close 
-        stream.join if stream.respond_to? :join
       end
     end
 
@@ -153,11 +154,9 @@ module Misc
       while block = io.read(2048)
         str << block
       end
+      io.join if io.respond_to? :join
     rescue
       io.abort if io.respond_to? :abort
-    ensure
-      io.join if io.respond_to? :join
-      io.close if io.respond_to? :close
     end
     str
   end
@@ -167,12 +166,10 @@ module Misc
       while block = io.read(2048)
         return if io.eof?
         Thread.pass 
-     end
+      end
+      io.join if io.respond_to? :join
     rescue
       io.abort if io.respond_to? :abort
-    ensure
-      io.join if io.respond_to? :join
-      io.close if io.respond_to? :close
     end
   end
 
@@ -213,12 +210,38 @@ module Misc
     end
     str
   end
-  def self._read_stream(stream, size)
-    str = ""
-    while (len=str.length) < size
-      str << (stream.read(size-len) or break)
+
+  def self.sensiblewrite(path, content = nil, &block)
+    return if File.exists? path
+    #tmp_path = path + '.sensible_write'
+    tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
+    Misc.lock tmp_path do
+      if not File.exists? path
+        FileUtils.rm_f tmp_path if File.exists? tmp_path
+        begin
+          case
+          when block_given?
+            File.open(tmp_path, 'w', &block)
+          when String === content
+            File.open(tmp_path, 'w') do |f| f.write content end
+          when (IO === content or StringIO === content or File === content)
+            File.open(tmp_path, 'w') do |f|  
+              while block = content.read(2048); 
+                f.write block
+              end  
+            end
+          else
+            File.open(tmp_path, 'w') do |f|  end
+          end
+          FileUtils.mv tmp_path, path
+        rescue Exception
+          Log.error "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
+          FileUtils.rm_f path if File.exists? path
+          raise $!
+        ensure
+          FileUtils.rm_f tmp_path if File.exists? tmp_path
+        end
+      end
     end
-    str
   end
-
 end

data/lib/rbbt/util/open.rb

@@ -236,13 +236,13 @@ module Open
     end
   end
 
-  def self.lock(file, &block)
+  def self.lock(file, options = {}, &block)
     if (dir_sub_path = find_repo_dir(file))
       dir, sub_path = dir_sub_path
       repo = get_repo_from_dir(dir)
       Misc.lock_in_repo(repo, sub_path, &block)
     else
-      Misc.lock(file, &block)
+      Misc.lock(file, true, options, &block)
     end
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -73,7 +73,7 @@ class Step
     return nil if @exec or info_file.nil?
     value = Annotated.purge value if defined? Annotated
     lock_filename = Persist.persistence_path(info_file, {:dir => Step.lock_dir})
-    Open.lock(info_file) do
+    Open.lock(info_file, :refresh => false) do
       i = info
       i[key] = value 
       @info_cache = i

data/lib/rbbt/workflow/step.rb

@@ -3,7 +3,7 @@ require 'rbbt/persist/tsv'
 require 'rbbt/util/log'
 require 'rbbt/util/semaphore'
 require 'rbbt/workflow/accessor'
-
+require 'rbbt/workflow/step/run'
 
 class Step
   attr_accessor :path, :task, :inputs, :dependencies, :bindings
@@ -49,357 +49,63 @@ class Step
 
   class << self
   attr_accessor :log_relay_step
-end
-
-def relay_log(step)
-  return self unless Task === self.task and not self.task.name.nil?
-  if not self.respond_to? :original_log
-    class << self
-      attr_accessor :relay_step
-      alias original_log log 
-      def log(status, message = nil)
-        self.status = status
-        message Log.uncolor message
-        relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
-      end
-    end
   end
-  @relay_step = step
-  self
-end
 
-def prepare_result(value, description = nil, info = {})
-  case 
-  when IO === value
-    begin
-      case @task.result_type
-      when :array
-        array = []
-        while line = value.gets
-          array << line.strip
+  def relay_log(step)
+    return self unless Task === self.task and not self.task.name.nil?
+    if not self.respond_to? :original_log
+      class << self
+        attr_accessor :relay_step
+        alias original_log log 
+        def log(status, message = nil)
+          self.status = status
+          message Log.uncolor message
+          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
         end
-        array
-      when :tsv
-        TSV.open(value)
-      else
-        value.read
       end
-    rescue Exception
-      value.abort if value.respond_to? :abort
-    ensure
-      value.join if value.respond_to? :join
-      value.close unless value.closed?
     end
-  when (not defined? Entity or description.nil? or not Entity.formats.include? description)
-    value
-  when (Annotated === value and info.empty?)
-    value
-  when Annotated === value
-    annotations = value.annotations
-    info.each do |k,v|
-      value.send("#{h}=", v) if annotations.include? k
-    end
-    value
-  else
-    Entity.formats[description].setup(value, info.merge(:format => description))
-  end
-end
-
-def get_stream
-  @mutex.synchronize do
-    begin
-      IO === @result ? @result : nil
-    ensure
-      @result = nil
-    end
-  end
-end
-
-def _exec
-  @exec = true if @exec.nil?
-  @task.exec_in((bindings ? bindings : self), *@inputs)
-end
-
-def exec(no_load=false)
-  dependencies.each{|dependency| dependency.exec(no_load) }
-  @result = _exec
-  @result = @result.stream if TSV::Dumper === @result
-  no_load ? @result : prepare_result(@result, @task.result_description)
-end
-
-def join
-  stream = get_stream if @result
-  begin
-    Misc.consume_stream stream if stream
-  rescue
-    stream.abort if stream.respond_to? :abort
-    raise $!
-  ensure
-    stream.join if stream.respond_to? :join and not stream.joined?
-  end
-
-  return if not Open.exists? info_file
-  @pid ||= info[:pid]
-
-  #while not done?
-  #  Misc.insist 2, 0.5 do
-  #    raise "Job error while joining: #{info[:messages].last}" if error?
-  #    raise "Job aborted  while joining: #{info[:messages].last}" if aborted?
-  #    raise "Job vanished  while joining: #{@pid}" if @pid and not Misc.pid_exists? @pid
-  #  end
-  #end
-
-  Misc.insist [0.1, 0.2, 0.5, 1] do
-    @pid ||= info[:pid]
-  end
-
-  if @pid.nil?
-    dependencies.each{|dep| dep.join }
-    self
-  else
-    begin
-      Log.debug{"Waiting for pid: #{@pid}"}
-      Process.waitpid @pid 
-    rescue Errno::ECHILD
-      Log.debug{"Process #{ @pid } already finished: #{ path }"}
-    end if Misc.pid_exists? @pid
-    @pid = nil
-    dependencies.each{|dep| dep.join }
+    @relay_step = step
     self
   end
-  self
-end
-
-def checks
-  rec_dependencies.collect{|dependency| dependency.path }.uniq
-end
 
-def kill_children
-  children_pids = info[:children_pids]
-  if children_pids and children_pids.any?
-    Log.medium("Killing children: #{ children_pids * ", " }")
-    children_pids.each do |pid|
-      Log.medium("Killing child #{ pid }")
+  def prepare_result(value, description = nil, info = {})
+    case 
+    when IO === value
       begin
-        Process.kill "INT", pid
-      rescue Exception
-        Log.medium("Exception killing child #{ pid }: #{$!.message}")
-      end
-    end
-  end
-end
-
-def run_dependencies(seen = [])
-  seen << self.path
-  dependencies.uniq.each{|dependency| 
-    next if seen.include? dependency.path
-    Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} -- #{Log.color :blue, dependency.path}"
-    begin
-      dependency.relay_log self
-      dependency.clean if not dependency.done? and (dependency.error? or dependency.aborted?)
-      dependency.clean if dependency.streaming? and not dependency.running?
-      #dependency.run_dependencies(seen)
-      dependency.run(ENV["RBBT_NO_STREAM"] != 'true') unless dependency.result or dependency.done?
-      seen << dependency.path
-      seen.concat dependency.rec_dependencies.collect{|d| d.path} 
-    rescue Exception
-      backtrace = $!.backtrace
-      set_info :backtrace, backtrace 
-      log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
-      raise $!
-    end
-  }
-end
-
-def run(no_load = false)
-
-  result = nil
-  begin
-    @mutex.synchronize do
-      no_load = no_load ? :stream : false
-      result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
-        if Step === Step.log_relay_step and not self == Step.log_relay_step
-          relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
-        end
-        @exec = false
-
-        Open.rm info_file if Open.exists? info_file
-
-        set_info :pid, Process.pid
-        set_info :issued, Time.now
-
-        log(:preparing, "Preparing job: #{Misc.fingerprint dependencies}")
-        set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name]}
-
-        run_dependencies
-
-        set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
-
-        set_info :started, (start_time = Time.now)
-        log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-
-        begin
-          result = _exec
-        rescue Aborted
-          log(:error, "Aborted")
-
-          kill_children
-          raise $!
-        rescue Exception
-          backtrace = $!.backtrace
-
-          # HACK: This fixes an strange behaviour in 1.9.3 where some
-          # backtrace strings are coded in ASCII-8BIT
-          kill_children
-          set_info :backtrace, backtrace 
-          log(:error, "#{$!.class}: #{$!.message}")
-          backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
-          raise $!
-        end
-
-        result = prepare_result result, @task.description, info if IO === result and ENV["RBBT_NO_STREAM"]
-        result = prepare_result result.stream, @task.description, info if TSV::Dumper === result and ENV["RBBT_NO_STREAM"]
-
-        case result
-        when IO
-          result = Misc.read_stream(result) if ENV["RBBT_NO_STREAM"]
-
-          log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-          ConcurrentStream.setup result do
-            begin
-              set_info :done, (done_time = Time.now)
-              set_info :time_elapsed, (time_elapsed = done_time - start_time)
-              log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
-            rescue
-              Log.exception $!
-            ensure
-              join
-            end
-          end
-          result.abort_callback = Proc.new do
-            begin
-              log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
-            rescue
-              Log.exception $!
-            ensure
-              join
-            end
-          end
-        when TSV::Dumper
-          log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-          ConcurrentStream.setup result.stream do
-            begin
-              set_info :done, (done_time = Time.now)
-              set_info :done, (done_time = Time.now)
-              set_info :time_elapsed, (time_elapsed = done_time - start_time)
-              log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
-            rescue
-              Log.exception $!
-            ensure
-              join
-            end
-          end
-          result.stream.abort_callback = Proc.new do
-            begin
-              log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
-            rescue
-              Log.exception $!
-            end
+        case @task.result_type
+        when :array
+          array = []
+          while line = value.gets
+            array << line.strip
           end
+          array
+        when :tsv
+          TSV.open(value)
         else
-          set_info :done, (done_time = Time.now)
-          set_info :time_elapsed, (time_elapsed = done_time - start_time)
-          log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+          value.read
         end
-
-        result
-      end
-
-      if no_load
-        @result ||= result
-        self
-      else
-        @result = prepare_result result, @task.result_description
-      end
-    end
-  ensure
-    join unless no_load
-  end
-  end
-
-  def fork(semaphore = nil)
-    raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid
-    @pid = Process.fork do
-      begin
-        RbbtSemaphore.wait_semaphore(semaphore) if semaphore
-        FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
-        begin
-          res = run
-        rescue Aborted
-          Log.debug{"Forked process aborted: #{path}"}
-          log :aborted, "Aborted"
-          raise $!
-        rescue Exception
-          Log.debug("Exception '#{$!.message}' caught on forked process: #{path}")
-          raise $!
-        ensure
-          join
-        end
-
-        begin
-          children_pids = info[:children_pids]
-          if children_pids
-            children_pids.each do |pid|
-              if Misc.pid_exists? pid
-                begin
-                  Process.waitpid pid
-                rescue Errno::ECHILD
-                  Log.low "Waiting on #{ pid } failed: #{$!.message}"
-                end
-              end
-            end
-            set_info :children_done, Time.now
-          end
-        rescue Exception
-          Log.debug("Exception waiting for children: #{$!.message}")
-          exit -1
-        end
-        set_info :pid, nil
-        exit 0
+      rescue Exception
+        value.abort if value.respond_to? :abort
       ensure
-        RbbtSemaphore.post_semaphore(semaphore) if semaphore
+        value.join if value.respond_to? :join
+        value.close unless value.closed?
       end
-    end
-    set_info :forked, true
-    Process.detach(@pid)
-    self
-  end
-
-  def abort
-    @pid ||= info[:pid]
-
-    #return true unless info[:forked]
-
-    case @pid
-    when nil
-      Log.medium "Could not abort #{path}: no pid"
-      false
-    when Process.pid
-      Log.medium "Could not abort #{path}: same process"
-      false
-    else
-      Log.medium "Aborting #{path}: #{ @pid }"
-      begin
-        Process.kill("KILL", @pid)
-        Process.waitpid @pid
-      rescue Exception
-        Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
+    when (not defined? Entity or description.nil? or not Entity.formats.include? description)
+      value
+    when (Annotated === value and info.empty?)
+      value
+    when Annotated === value
+      annotations = value.annotations
+      info.each do |k,v|
+        value.send("#{h}=", v) if annotations.include? k
       end
-      log(:aborted, "Job aborted")
-      true
+      value
+    else
+      Entity.formats[description].setup(value, info.merge(:format => description))
     end
   end
 
+
   def child(&block)
     child_pid = Process.fork &block
     children_pids = info[:children_pids]

data/lib/rbbt/workflow/step/run.rb

@@ -0,0 +1,325 @@
+class Step
+
+  attr_reader :stream
+
+  def get_stream
+    @mutex.synchronize do
+      @stream = begin
+        IO === @result ? @result : nil
+      ensure
+        @result = nil
+      end
+    end
+  end
+
+  def _exec
+    @exec = true if @exec.nil?
+    @task.exec_in((bindings ? bindings : self), *@inputs)
+  end
+
+  def exec(no_load=false)
+    dependencies.each{|dependency| dependency.exec(no_load) }
+    @result = _exec
+    @result = @result.stream if TSV::Dumper === @result
+    no_load ? @result : prepare_result(@result, @task.result_description)
+  end
+
+  def checks
+    rec_dependencies.collect{|dependency| dependency.path }.uniq
+  end
+
+  def kill_children
+    children_pids = info[:children_pids]
+    if children_pids and children_pids.any?
+      Log.medium("Killing children: #{ children_pids * ", " }")
+      children_pids.each do |pid|
+        Log.medium("Killing child #{ pid }")
+        begin
+          Process.kill "INT", pid
+        rescue Exception
+          Log.medium("Exception killing child #{ pid }: #{$!.message}")
+        end
+      end
+    end
+  end
+
+  def run_dependencies(seen = [])
+    seen << self.path
+    dependencies.uniq.each{|dependency| 
+      next if seen.include? dependency.path
+      Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} -- #{Log.color :blue, dependency.path}"
+      begin
+        dependency.relay_log self
+        dependency.clean if not dependency.done? and (dependency.error? or dependency.aborted?)
+        dependency.clean if dependency.streaming? and not dependency.running?
+        #dependency.run_dependencies(seen)
+        dependency.run(ENV["RBBT_NO_STREAM"] != 'true') unless dependency.result or dependency.done?
+        seen << dependency.path
+        seen.concat dependency.rec_dependencies.collect{|d| d.path} 
+      rescue Exception
+        backtrace = $!.backtrace
+        set_info :backtrace, backtrace 
+        log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
+        raise $!
+      end
+    }
+  end
+
+  def run(no_load = false)
+
+    result = nil
+    begin
+      @mutex.synchronize do
+        no_load = no_load ? :stream : false
+        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
+          if Step === Step.log_relay_step and not self == Step.log_relay_step
+            relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
+          end
+          @exec = false
+
+          Open.rm info_file if Open.exists? info_file
+
+          set_info :pid, Process.pid
+          set_info :issued, Time.now
+
+          log(:preparing, "Preparing job: #{Misc.fingerprint dependencies}")
+          set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name]}
+
+          run_dependencies
+
+          set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
+
+          set_info :started, (start_time = Time.now)
+          log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+
+          begin
+            result = _exec
+          rescue Aborted
+            log(:error, "Aborted")
+
+            kill_children
+            raise $!
+          rescue Exception
+            backtrace = $!.backtrace
+
+            # HACK: This fixes an strange behaviour in 1.9.3 where some
+            # backtrace strings are coded in ASCII-8BIT
+            kill_children
+            set_info :backtrace, backtrace 
+            log(:error, "#{$!.class}: #{$!.message}")
+            backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
+            raise $!
+          end
+
+          result = prepare_result result, @task.description, info if IO === result and ENV["RBBT_NO_STREAM"]
+          result = prepare_result result.stream, @task.description, info if TSV::Dumper === result and ENV["RBBT_NO_STREAM"]
+
+          case result
+          when IO
+            result = Misc.read_stream(result) if ENV["RBBT_NO_STREAM"]
+
+            log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+            ConcurrentStream.setup result do
+              begin
+                set_info :done, (done_time = Time.now)
+                set_info :time_elapsed, (time_elapsed = done_time - start_time)
+                log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
+              rescue
+                Log.exception $!
+              ensure
+                join
+              end
+            end
+            result.abort_callback = Proc.new do
+              begin
+                log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}" if status == :streaming
+                stop_dependencies
+                abort_stream
+              rescue
+                Log.exception $!
+              ensure
+                join
+              end
+            end
+          when TSV::Dumper
+            log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+            ConcurrentStream.setup result.stream do
+              begin
+                set_info :done, (done_time = Time.now)
+                set_info :done, (done_time = Time.now)
+                set_info :time_elapsed, (time_elapsed = done_time - start_time)
+                log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
+              rescue
+                Log.exception $!
+              ensure
+                join
+              end
+            end
+            result.stream.abort_callback = Proc.new do
+              begin
+                log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"  if status == :streaming
+                stop_dependencies
+                abort_stream
+              rescue
+                Log.exception $!
+              ensure
+                join
+              end
+            end
+          else
+            set_info :done, (done_time = Time.now)
+            set_info :time_elapsed, (time_elapsed = done_time - start_time)
+            log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+          end
+
+          result
+        end
+
+        if no_load
+          @result ||= result
+          self
+        else
+          @result = prepare_result result, @task.result_description
+        end
+      end
+    ensure
+      join unless no_load
+    end
+  end
+
+  def fork(semaphore = nil)
+    raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid
+    @pid = Process.fork do
+      begin
+        RbbtSemaphore.wait_semaphore(semaphore) if semaphore
+        FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
+        begin
+          res = run
+        rescue Aborted
+          Log.debug{"Forked process aborted: #{path}"}
+          log :aborted, "Aborted"
+          raise $!
+        rescue Exception
+          Log.debug("Exception '#{$!.message}' caught on forked process: #{path}")
+          raise $!
+        ensure
+          join
+        end
+
+        begin
+          children_pids = info[:children_pids]
+          if children_pids
+            children_pids.each do |pid|
+              if Misc.pid_exists? pid
+                begin
+                  Process.waitpid pid
+                rescue Errno::ECHILD
+                  Log.low "Waiting on #{ pid } failed: #{$!.message}"
+                end
+              end
+            end
+            set_info :children_done, Time.now
+          end
+        rescue Exception
+          Log.debug("Exception waiting for children: #{$!.message}")
+          exit -1
+        end
+        set_info :pid, nil
+        exit 0
+      ensure
+        RbbtSemaphore.post_semaphore(semaphore) if semaphore
+      end
+    end
+    set_info :forked, true
+    Process.detach(@pid)
+    self
+  end
+
+  def stop_dependencies
+    dependencies.each do |dep|
+      dep.abort unless dep.done?
+    end
+  end
+
+  def abort_pid
+    @pid ||= info[:pid]
+
+    case @pid
+    when nil
+      Log.medium "Could not abort #{path}: no pid"
+      false
+    when Process.pid
+      Log.medium "Could not abort #{path}: same process"
+      false
+    else
+      Log.medium "Aborting #{path}: #{ @pid }"
+      begin
+        Process.kill("KILL", @pid)
+        Process.waitpid @pid
+      rescue Exception
+        Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
+      end
+      true
+    end
+  end
+
+  def abort_stream
+    stream = get_stream if @result
+    stream ||= @stream 
+    if stream
+      stream.abort if stream.respond_to? :abort
+    end
+  end
+
+  def abort
+    begin
+      abort_pid
+      stop_dependencies
+      abort_stream
+    ensure
+      log(:aborted, "Job aborted")
+    end
+  end
+
+  def join_stream
+    stream = get_stream if @result
+    if stream
+      begin
+        Misc.consume_stream stream
+        stream.join if stream.respond_to? :join # and not stream.joined?
+      rescue Exception
+        stream.abort if stream.respond_to? :abort 
+        self.abort
+        raise $!
+      end
+    end
+  end
+
+  def join
+
+    join_stream
+
+    return if not Open.exists? info_file
+    @pid ||= info[:pid]
+
+    Misc.insist [0.1, 0.2, 0.5, 1] do
+      @pid ||= info[:pid]
+    end
+
+    if @pid.nil?
+      dependencies.each{|dep| dep.join }
+      self
+    else
+      begin
+        Log.debug{"Waiting for pid: #{@pid}"}
+        Process.waitpid @pid 
+      rescue Errno::ECHILD
+        Log.debug{"Process #{ @pid } already finished: #{ path }"}
+      end if Misc.pid_exists? @pid
+      @pid = nil
+      dependencies.each{|dep| dep.join }
+      self
+    end
+    self
+  end
+end

data/share/rbbt_commands/workflow/task

@@ -413,9 +413,13 @@ when Step
       end unless io.closed? 
       io.join if io.respond_to? :join
     rescue Exception
-      io.abort if io.respond_to? :abort
-    ensure
-      io.join if io.respond_to? :join
+      Log.exception $!
+      begin
+        io.abort if io.respond_to? :abort
+        io.join
+      ensure
+        exit -1
+      end
     end
   else
     res.join

data/test/rbbt/test_persist.rb

@@ -75,6 +75,7 @@ class TestPersist < Test::Unit::TestCase
       end
 
       assert_equal 10, stream.read.split("\n").length
+      stream.join
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.11.5
+  version: 5.11.6
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -205,6 +205,7 @@ files:
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
+- lib/rbbt/workflow/step/run.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - share/Rlib/util.R