rbbt-util 5.11.1 → 5.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee7693bddd82af53509c8ed2e5414f1a86ac51ce
-  data.tar.gz: 5122a7d8c8404d7a045e571310cfb34c3e808bbe
+  metadata.gz: 06c3a98f22b69a5738d6c9b478a2cee9a4e52e66
+  data.tar.gz: d60e07e6371c2c6c88359d6b028e7c7fdcbd0c64
 SHA512:
-  metadata.gz: e75a716cf8fd069b161d3f167217b0a329eac88d1458e6133fdbc0b3e51cfcd647e6c53cf2e4cbe9a28ab5499e2822138797688eb75239fffe458975195b6ad3
-  data.tar.gz: ae968a716ff89c35ba2a07300ac56cba42452692a75e2b811051d360fd64ce51c4f95aba36a9d974234450cdfdab846ddd31a367383a41ac80b1b49f177ac303
+  metadata.gz: dab5cbee06c6169ab384555b53e6ea3407cb2c3f18b0b8d99f9aac6d6209d4c8656a6baeba4d46ede3c87c8eb141a870723d7a8336f7f58c85535d6d2f3f54a6
+  data.tar.gz: e3268aebe76ca88cfbc948b34367992188e38289fe966fc90ffc07609e2f6b355d751327008b691fbe4e94a1d7b9085e32fd0f73734ee538b041b9bd982486b0

data/lib/rbbt/persist.rb

@@ -172,59 +172,118 @@ module Persist
     end
   end
 
-  def self.tee_stream_fork(stream, path, type, callback = nil)
+  #def self.tee_stream_fork(stream, path, type, callback = nil)
+  #  file, out = Misc.tee_stream(stream)
+
+  #  saver_pid = Process.fork do
+  #    out.close
+  #    stream.close
+  #    Misc.purge_pipes
+  #    begin
+  #      Misc.lock(path) do
+  #        save_file(path, type, file)
+  #      end
+  #    rescue Aborted
+  #      stream.abort if stream.respond_to? :abort
+  #      raise $!
+  #    rescue Exception
+  #      Log.exception $!
+  #      Kernel.exit! -1
+  #    end
+  #    Kernel.exit! 0
+  #  end
+  #  file.close
+  #  ConcurrentStream.setup(out, :pids => [saver_pid], :filename => path)
+  #end
+
+  def self.tee_stream_thread(stream, path, type, callback = nil)
     file, out = Misc.tee_stream(stream)
 
-    saver_pid = Process.fork do
-      out.close
-      stream.close
-      Misc.purge_pipes
+    saver_thread = Thread.new(Thread.current, path, file) do |parent,path,file|
       begin
+        Thread.current["name"] = "file saver: " + path
         Misc.lock(path) do
           save_file(path, type, file)
         end
       rescue Aborted
+        Log.error "Persist stream thread aborted: #{ Log.color :blue, path }"
         stream.abort if stream.respond_to? :abort
-        raise $!
       rescue Exception
+        Log.error "Persist stream thread exception: #{ Log.color :blue, path }"
         Log.exception $!
-        Kernel.exit! -1
+        stream.abort if stream.respond_to? :abort
+        parent.raise $!
       end
-      Kernel.exit! 0
     end
-    file.close
-    ConcurrentStream.setup(out, :pids => [saver_pid], :filename => path)
+    ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
   end
 
-  def self.tee_stream_thread(stream, path, type, callback = nil)
-    file, out = Misc.tee_stream(stream)
-
-    saver_thread = Thread.new(Thread.current, path, file) do |parent,path,file|
+  def self.tee_stream_pipe(stream, path, type, callback = nil)
+    parent_pid = Process.pid
+    out = Misc.open_pipe true, false do |sin|
       begin
-        Thread.current["name"] = "file saver: " + path
-        Misc.lock(path) do
-          save_file(path, type, file)
+        file, out = Misc.tee_stream(stream)
+
+        saver_th = Thread.new(Thread.current, path, file) do |parent,path,file|
+          begin
+            Misc.lock(path) do
+              save_file(path, type, file)
+            end
+            Log.high "Stream pipe saved: #{path}"
+          rescue Aborted
+            Log.error "Persist stream pipe exception: #{ Log.color :blue, path }"
+            stream.abort if stream.respond_to? :abort
+          rescue Exception
+            Log.error "Persist stream pipe exception: #{ Log.color :blue, path }"
+            Log.exception $!
+            stream.abort if stream.respond_to? :abort
+            parent.raise $!
+          end
+        end
+
+        tee_th = Thread.new(Thread.current) do |parent|
+          begin
+            while block = out.read(2028)
+              sin.write block
+            end
+          rescue Aborted
+            Log.error "Tee stream thread aborted"
+            stream.abort if stream.respond_to? :abort
+          rescue Exception
+            stream.abort if stream.respond_to? :abort
+            Log.exception $!
+            parent.raise $!
+          ensure
+            sin.close unless sin.closed?
+          end
         end
+        saver_th.join
+        tee_th.join
       rescue Aborted
-        Log.error "Tee stream thread aborted"
-        stream.abort if stream.respond_to? :abort
+        tee_th.raise Aborted.new if tee_th and tee_th.alive?
+        saver_th.raise Aborted.new if saver_th and saver_th.alive?
+        Kernel.exit! -1
       rescue Exception
-        stream.abort if stream.respond_to? :abort
+        tee_th.raise Aborted.new if tee_th and tee_th.alive?
+        saver_th.raise Aborted.new if saver_th and saver_th.alive?
         Log.exception $!
-        parent.raise $!
+        Process.kill :INT, parent_pid
+        Kernel.exit! -1
       end
     end
-    ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
+    stream.close
+    out
   end
 
   class << self
     alias tee_stream tee_stream_thread 
   end
 
+
   def self.persist(name, type = nil, persist_options = {})
     type ||= :marshal
 
-    return (persist_options[:repo] || Persist::MEMORY)[persist_options[:file]] ||= yield if type ==:memory and persist_options[:file]
+    return (persist_options[:repo] || Persist::MEMORY)[persist_options[:file]] ||= yield if type ==:memory and persist_options[:file] and persist_options[:persist] and persist_options[:persist] != :update
 
     if FalseClass != persist_options[:persist]
       other_options = Misc.process_options persist_options, :other
@@ -327,6 +386,7 @@ module Persist
 
           lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
           Misc.lock lock_filename  do |lockfile|
+
             if is_persisted?(path, persist_options)
               Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
               return path if persist_options[:no_load]
@@ -342,13 +402,21 @@ module Persist
               when IO
                 res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
                 ConcurrentStream.setup res do
-                  lockfile.unlock
+                  begin
+                    lockfile.unlock
+                  rescue
+                    Log.warn "Lockfile exception: " << $!.message
+                  end
                 end
                 raise KeepLocked.new res 
               when TSV::Dumper
                 res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
                 ConcurrentStream.setup res do
-                  lockfile.unlock
+                  begin
+                    lockfile.unlock
+                  rescue
+                    Log.warn "Lockfile exception: " << $!.message
+                  end
                 end
                 raise KeepLocked.new res 
               end
@@ -367,6 +435,7 @@ module Persist
                       end
               rescue
                 res.abort if res.respond_to? :abort
+                raise $!
               ensure
                 res.join if res.respond_to? :join
               end
@@ -376,6 +445,7 @@ module Persist
                 res = TSV.open(io)
               rescue
                 io.abort if io.respond_to? :abort
+                raise $!
               ensure
                 io.join if io.respond_to? :join
               end
@@ -389,7 +459,7 @@ module Persist
           end
 
         rescue
-          Log.high "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
+          Log.error "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
           FileUtils.rm path if Open.exists? path 
           raise $!
         end
@@ -410,7 +480,7 @@ module Persist
       file = name
       repo = options.delete :repo if options and options.any?
       file << "_" << (options[:key] ? options[:key] : Misc.hash2md5(options)) if options and options.any?
-      persist name, :memory, options.merge(:repo => repo, :persist => true, :file => file), &block
+      persist name, :memory, {:repo => repo, :persist => true, :file => file}.merge(options), &block
     end
   end
 end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -115,7 +115,7 @@ module TSV
       else
         obj.traverse(options, &block)
       end
-    when IO, File, StringIO
+    when IO, File
       begin
         if options[:type] == :array
           traverse_io_array(obj, options, &block)
@@ -123,7 +123,8 @@ module TSV
           traverse_io(obj, options, &block)
         end
       rescue Exception
-        raise Aborted
+        obj.abort if obj.respond_to? :abort
+        raise $!
       ensure
         begin
           obj.close if obj.respond_to? :close and not obj.closed?
@@ -150,7 +151,7 @@ module TSV
     when Array
       traverse_array(obj, options, &block)
     when nil
-      raise "Can not traverse nil object"
+      raise "Can not traverse nil object into #{stream_name(options)}"
     else
       raise "Unknown object for traversal: #{Misc.fingerprint obj }"
     end
@@ -191,18 +192,24 @@ module TSV
       q.callback &callback
       q.init &block
 
-      traverse_obj(obj, options) do |*p|
-        q.process *p
+      pid = Process.fork do 
+        Misc.purge_pipes(q.queue.swrite)
+        traverse_obj(obj, options) do |*p|
+          q.process *p
+        end
       end
 
-      into = options[:into]
+      #stream = obj_stream(obj)
+      #stream.close if stream
+
+      Process.waitpid pid
     rescue Exception
+      Log.exception $!
       q.abort
       raise $!
     ensure
       q.join
     end
-
   end
 
   def self.store_into(store, value)
@@ -243,11 +250,13 @@ module TSV
     when TSV::Dumper
       close_streams << obj.result.in_stream
     when (defined? Step and Step)
-      case obj.result
-      when IO
-        close_streams << obj.result
-      when TSV::Dumper
-        close_streams << obj.result.in_stream
+      obj.mutex.synchronize do
+        case obj.result
+        when IO
+          close_streams << obj.result
+        when TSV::Dumper
+          close_streams << obj.result.in_stream
+        end
       end
       obj.inputs.each do |input|
         close_streams = get_streams_to_close(input) + close_streams
@@ -294,7 +303,6 @@ module TSV
         parent.raise $!
       end
     end
-    thread.wakeup
     ConcurrentStream.setup(obj_stream(into), :threads => thread)
   end
 
@@ -311,7 +319,8 @@ module TSV
         begin
           traverse(obj, options.merge(:into => sin), &block)                                                                                                                                  
         rescue Exception
-          sin.abort if sin.respond_to? :abort
+          sout.abort if sout.respond_to? :abort
+          sout.join if sout.respond_to? :join
         end
       end                                                                                                                                                                                   
       return sout
@@ -328,11 +337,10 @@ module TSV
       end
 
       case into
-      when TSV::Dumper, IO, StringIO
+      when TSV::Dumper, IO
         traverse_stream(obj, threads, cpus, options, &block)
       else
         traverse_run(obj, threads, cpus, options, &block)
-        into.join if into.respond_to? :join
         into.close if into.respond_to? :close
       end
 

data/lib/rbbt/util/concurrency/processes.rb

@@ -31,7 +31,7 @@ class RbbtProcessQueue
           Log.error "Callback thread aborted"
         rescue ClosedStream
         rescue Exception
-          Log.exception $!
+          Log.error "Callback thread exception"
           parent.raise $!
         ensure
           @callback_queue.sread.close unless @callback_queue.sread.closed?
@@ -68,7 +68,6 @@ class RbbtProcessQueue
 
   def close_callback
     @callback_queue.push ClosedStream.new if  @callback_thread.alive?
-    @callback_queue.swrite.close 
     @callback_thread.join 
   end
 
@@ -81,6 +80,7 @@ class RbbtProcessQueue
       close_callback if @callback
     rescue Exception
       Log.exception $!
+      raise $!
     ensure
       @queue.swrite.close
     end
@@ -100,7 +100,7 @@ class RbbtProcessQueue
   end
 
   def abort
-    @process_monitor.raise Aborted.new if @process_monitor.alive?
+    @process_monitor.raise Aborted.new if @process_monitor and @process_monitor.alive?
     @callback_thread.raise Aborted.new if @callback_thread and @callback_thread.alive?
   end
 

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -37,7 +37,7 @@ class RbbtProcessQueue
       end
 
       write_length = str.length
-      IO.select(nil, [stream])
+      #IO.select(nil, [stream])
       wrote = stream.write(str) 
       while wrote < write_length
         wrote += stream.write(str[wrote..-1]) 

data/lib/rbbt/util/log.rb

@@ -4,13 +4,29 @@ require 'rbbt/util/color'
 module Log
   extend Term::ANSIColor
 
-  DEBUG    = 0
-  LOW      = 1
-  MEDIUM   = 2
-  HIGH     = 3
-  INFO     = 4
-  WARN     = 5
-  ERROR    = 6
+
+  SEVERITY_NAMES ||= begin
+                     names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR ) 
+                     names.each_with_index do |name,i|
+                       eval "#{ name } = #{ i }" 
+                     end
+                     names
+                   end
+
+  def self.get_level(level)
+    case level
+    when Fixnum
+      level
+    when String
+      begin
+        Log.const_get(level.upcase)
+      rescue
+        Log.exception $!
+      end
+    when Symbol
+      get_level(level.to_s)
+    end || 0
+  end
 
   class << self
     attr_accessor :logfile, :severity, :nocolor, :tty_size
@@ -42,11 +58,15 @@ module Log
     Term::ANSIColor.uncolor(str)
   end
 
-  def self.color(severity, str = nil)
+  def self.reset_color
+    reset
+  end
+
+  def self.color(severity, str = nil, reset = false)
     return str || "" if nocolor 
-    color = SEVERITY_COLOR[severity] if Fixnum === severity
-    color = Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 
-    color ||= ""
+    color = reset ? Term::ANSIColor.reset : ""
+    color << SEVERITY_COLOR[severity] if Fixnum === severity
+    color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 
     if str.nil?
       color
     else
@@ -92,6 +112,40 @@ module Log
     end
   end
 
+  def self.log_obj_inspect(obj, level, file = $stdout)
+    stack = caller
+
+    line = nil
+    while line.nil? or line =~ /util\/log\.rb/ and stack.any?
+      line = stack.shift 
+    end
+    line ||= caller.first
+
+    level = Log.get_level level
+    name = Log::SEVERITY_NAMES[level] + ": "
+    Log.log Log.color(level, name, true) << line, level
+    Log.log "", level
+    Log.log Log.color(level, "=> ", true) << obj.inspect, level
+    Log.log "", level
+  end
+
+  def self.log_obj_fingerprint(obj, level, file = $stdout)
+    stack = caller
+
+    line = nil
+    while line.nil? or line =~ /util\/log\.rb/ and stack.any?
+      line = stack.shift 
+    end
+    line ||= caller.first
+
+    level = Log.get_level level
+    name = Log::SEVERITY_NAMES[level] + ": "
+    Log.log Log.color(level, name, true) << line, level
+    Log.log "", level
+    Log.log Log.color(level, "=> ", true) << Misc.fingerprint(obj), level
+    Log.log "", level
+  end
+
   def self.debug(message = nil, &block)
     log(message, DEBUG, &block)
   end
@@ -157,52 +211,52 @@ def fff(object)
   Log.debug{""}
 end
 
-def ddd(message, file = $stdout)
-  stack = caller
-  Log.debug{"#{Log.color :cyan, "DEBUG:"} " << stack.first}
-  Log.debug{""}
-  Log.debug{"=> " << message.inspect}
-  Log.debug{""}
+def ddd(obj, file = $stdout)
+  Log.log_obj_inspect(obj, :debug, file)
 end
 
-def lll(message, file = $stdout)
-  stack = caller
-  Log.low{"#{Log.color :cyan, "LOW:"} " << stack.first}
-  Log.low{""}
-  Log.low{"=> " << message.inspect}
-  Log.low{""}
+def lll(obj, file = $stdout)
+  Log.log_obj_inspect(obj, :low, file)
 end
 
-def mmm(message, file = $stdout)
-  stack = caller
-  Log.low{"#{Log.color :cyan, "MEDIUM:"} " << stack.first}
-  Log.low{""}
-  Log.low{"=> " << message.inspect}
-  Log.low{""}
+def mmm(obj, file = $stdout)
+  Log.log_obj_inspect(obj, :medium, file)
 end
 
-def hhh(message, file = $stdout)
-  stack = caller
-  Log.high{"#{Log.color :cyan, "HIGH:"} " << stack.first}
-  Log.high{""}
-  Log.high{"=> " << message.inspect}
-  Log.high{""}
+def iii(obj, file = $stdout)
+  Log.log_obj_inspect(obj, :info, file)
 end
 
-def iii(message, file = $stdout)
-  stack = caller
-  Log.info{"#{Log.color :cyan, "INFO:"} " << stack.first}
-  Log.info{""}
-  Log.info{"=> " << message.inspect}
-  Log.info{""}
+def wwww(obj, file = $stdout)
+  Log.log_obj_inspect(obj, :warn, file)
 end
 
-def eee(message, file = $stdout)
-  stack = caller
-  Log.error{"#{Log.color :cyan, "INFO:"} " << stack.first}
-  Log.error{""}
-  Log.error{"=> " << message.inspect}
-  Log.error{""}
+def eee(obj, file = $stdout)
+  Log.log_obj_inspect(obj, :error, file)
+end
+
+def ddf(obj, file = $stdout)
+  Log.log_obj_fingerprint(obj, :debug, file)
+end
+
+def llf(obj, file = $stdout)
+  Log.log_obj_fingerprint(obj, :low, file)
+end
+
+def mmf(obj, file = $stdout)
+  Log.log_obj_fingerprint(obj, :medium, file)
+end
+
+def iif(obj, file = $stdout)
+  Log.log_obj_fingerprint(obj, :info, file)
+end
+
+def wwwf(obj, file = $stdout)
+  Log.log_obj_fingerprint(obj, :warn, file)
+end
+
+def eef(obj, file = $stdout)
+  Log.log_obj_fingerprint(obj, :error, file)
 end
 
 if __FILE__ == $0
@@ -211,4 +265,8 @@ if __FILE__ == $0
   (0..6).each do |level|
     Log.log("Level #{level}", level)
   end
+
+  require 'rbbt/util/misc'
+  eee [1,2,3]
+  eef [1,2,3]
 end

data/lib/rbbt/util/misc.rb

@@ -35,7 +35,11 @@ module LaterString
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :filename
+  attr_accessor :threads, :pids, :callback, :filename, :joined
+
+  def joined?
+    @joined
+  end
 
   def join
 
@@ -57,10 +61,12 @@ module ConcurrentStream
       @pids = []
     end
 
-    if @callback
+    if @callback and not joined?
       @callback.call
       @callback = nil
     end
+
+    @joined = true
   end
 
   def abort
@@ -147,11 +153,11 @@ module Misc
           Process.kill :INT, parent_pid
           Kernel.exit! -1
         ensure
-          sin.close unless sin.closed? 
+          sin.close if close and not sin.closed? 
         end
         Kernel.exit! 0
       }
-      sin.close if close
+      sin.close #if close
       ConcurrentStream.setup sout, :pids => [pid]
     else
       thread = Thread.new(Thread.current) do |parent|
@@ -160,7 +166,7 @@ module Misc
         rescue
           parent.raise $!
         ensure
-          sin.close if close 
+          sin.close if close and not sin.closed?
         end
       end
       ConcurrentStream.setup sout, :threads => [thread]
@@ -246,6 +252,10 @@ module Misc
     alias tee_stream tee_stream_thread 
   end
 
+  def self.consume_stream(io)
+    Thread.pass while block = io.read(2048)
+  end
+
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
@@ -320,6 +330,14 @@ module Misc
     str
   end
 
+  def self.read_stream(stream, size)
+    str = ""
+    while (len=str.length) < size
+      str << stream.read(size-len)
+    end
+    str
+  end
+
   def self.parse_cmd_params(str)
     return str if Array === str
     str.scan(/
@@ -531,9 +549,9 @@ module Misc
       else 
         "'" << obj << "'"
       end
-    when AnnotatedArray
+    when (defined? AnnotatedArray and AnnotatedArray)
       "<A: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
-    when TSV::Parser
+    when (defined? TSV and TSV::Parser)
       "<TSVStream:" + obj.filename + "--" << Misc.fingerprint(obj.options) << ">"
     when IO
       "<IO:" + (obj.respond_to?(:filename) ? obj.filename : obj.inspect) + ">"
@@ -545,7 +563,7 @@ module Misc
       else
         "[" << (obj.collect{|e| fingerprint(e) } * ",") << "]"
       end
-    when TSV
+    when (defined? TSV and TSV)
       obj.with_unnamed do
         "TSV:{"<< fingerprint(obj.all_fields|| []).inspect << ";" << fingerprint(obj.keys).inspect << "}"
       end
@@ -1301,6 +1319,7 @@ end
     @hostanem ||= `hostname`.strip
   end
 
+  LOCK_MUTEX = Mutex.new
   def self.lock(file, unlock = true)
     return yield if file.nil?
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
@@ -1310,24 +1329,49 @@ end
     lock_path = File.expand_path(file + '.lock')
     lockfile = Lockfile.new(lock_path)
 
-    begin
-      Misc.insist 3 do
-        if File.exists? lock_path and
-          Misc.hostname == (info = Open.open(lock_path){|f| YAML.load(f) })["host"] and 
-          info["pid"] and not Misc.pid_exists?(info["pid"])
+    hostname = Misc.hostname
+    LOCK_MUTEX.synchronize do
+      Misc.insist 3, 0.1 do
+        begin
+          if File.exists? lock_path
+            info = Open.open(lock_path){|f| YAML.load(f) }
+            raise "No info" unless info
 
-          Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
-          FileUtils.rm lock_path 
+            if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
+              Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
+              FileUtils.rm lock_path
+            end
+          end
+        rescue Exception
+          FileUtils.rm lock_path if File.exists? lock_path
+          raise $!
+        ensure
+          lockfile = Lockfile.new(lock_path) unless File.exists? lock_path
         end
       end
-    rescue
-      Log.warn("Error checking lockfile #{lock_path}: #{$!.message}. Removing. Content: #{begin Open.read(lock_path) rescue "Could not open file" end}")
-      FileUtils.rm lock_path if File.exists?(lock_path)
-      lockfile = Lockfile.new(lock_path)
-      retry
     end
 
+    #begin
+    #  Misc.insist 3 do
+    #    LOCK_MUTEX.synchronize do
+    #      if File.exists? lock_path and
+    #        Misc.hostname == (info = Open.open(lock_path){|f| YAML.load(f) })["host"] and 
+    #        info["pid"] and not Misc.pid_exists?(info["pid"])
+
+    #        Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
+    #        FileUtils.rm lock_path 
+    #      end
+    #    end
+    #  end
+    #rescue
+    #  Log.warn("Error checking lockfile #{lock_path}: #{$!.message}. Removing. Content: #{begin Open.read(lock_path) rescue "Could not open file" end}")
+    #  FileUtils.rm lock_path if File.exists?(lock_path)
+    #  lockfile = Lockfile.new(lock_path)
+    #  retry
+    #end
+
     begin
+      
       lockfile.lock 
       res = yield lockfile
     rescue Lockfile::StolenLockError
@@ -1338,7 +1382,6 @@ end
     ensure
       if unlock and lockfile.locked?
         lockfile.unlock
-        FileUtils.rm lock_path if File.exists? lock_path
       end
     end
 

data/lib/rbbt/workflow.rb

@@ -159,6 +159,7 @@ module Workflow
   attr_accessor :helpers, :tasks
   attr_accessor :task_dependencies, :task_description, :last_task 
   attr_accessor :asynchronous_exports, :synchronous_exports, :exec_exports
+  attr_accessor :step_cache
 
   #{{{ ATTR DEFAULTS
   
@@ -180,6 +181,10 @@ module Workflow
     @libdir = Path.caller_lib_dir if @libdir.nil?
     @libdir 
   end
+
+  def step_cache
+    @step_cache ||= {}
+  end
   
 
   def helpers
@@ -226,6 +231,28 @@ module Workflow
     end 
   end
 
+  def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
+    step_path = step_path.call if Proc === step_path
+    persist = input_values.nil? ? false : true
+    step = Persist.memory("Step", :key => step_path, :repo => step_cache, :persist => persist ) do
+      step = Step.new step_path, task, input_values, dependencies
+
+      helpers.each do |name, block|
+        (class << step; self; end).instance_eval do
+          define_method name, &block
+        end
+      end
+      step
+
+    end
+
+    step.task ||= task
+    step.inputs ||= input_values
+    step.dependencies = dependencies if dependencies and (step.dependencies.nil? or step.dependencies.length < dependencies.length)
+
+    step
+  end
+
   def job(taskname, jobname = nil, inputs = {})
     taskname = taskname.to_sym
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
@@ -247,20 +274,12 @@ module Workflow
       step_path = step_path taskname, jobname, input_values, dependencies, task.extension
     end
 
-    step = Step.new step_path, task, input_values, dependencies
-
-    helpers.each do |name, block|
-      (class << step; self; end).instance_eval do
-        define_method name, &block
-      end
-    end
-
-    step
+    get_job_step step_path, task, input_values, dependencies
   end
 
   def load_step(path)
     task = task_for path
-    Step.new path, tasks[task.to_sym]
+    get_job_step path, tasks[task.to_sym]
   end
 
   def load_id(id)
@@ -279,7 +298,7 @@ module Workflow
   def load_name(task, name)
     task = tasks[task.to_sym] if String === task or Symbol === task
     path = step_path task.name, name, [], [], task.extension
-    Step.new path, task
+    get_job_step path, task
   end
 
   def jobs(taskname, query = nil)

data/lib/rbbt/workflow/accessor.rb

@@ -42,24 +42,29 @@ class Step
   def info
     return {} if info_file.nil? or not Open.exists? info_file
     begin
-      return @info_cache if @info_cache and File.mtime(info_file) < @info_cache_time
-    rescue Exception
-    end
-    begin
-      @info_cache = Misc.insist(3, 5, info_file) do
-        Misc.insist(2, 2, info_file) do
-          Misc.insist(2, 0.5, info_file) do
-            Open.open(info_file) do |file|
-              INFO_SERIALIAZER.load(file) || {}
+      @info_mutex.synchronize do
+        begin
+          return @info_cache if @info_cache and File.mtime(info_file) < @info_cache_time
+        rescue Exception
+        end
+        begin
+          @info_cache = Misc.insist(2, 1, info_file) do
+            Misc.insist(2, 0.5, info_file) do
+              Misc.insist(3, 0.1, info_file) do
+                Open.open(info_file) do |file|
+                  INFO_SERIALIAZER.load(file) || {}
+                end
+              end
             end
           end
+          @info_cache_time = Time.now
+          @info_cache
         end
       end
-      @info_cache_time = Time.now
-      @info_cache
     rescue Exception
       Log.debug{"Error loading info file: " + info_file}
       Open.write(info_file, INFO_SERIALIAZER.dump({:status => :error, :messages => ["Info file lost"]}))
+      self.abort
       raise $!
     end
   end
@@ -69,7 +74,7 @@ class Step
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file) do
       i = info
-      i[key] = value #File === value ? value.filename : value
+      i[key] = value 
       @info_cache = i
       Open.write(info_file, INFO_SERIALIAZER.dump(i))
       @info_cache_time = Time.now
@@ -143,6 +148,10 @@ class Step
     path and path.exists?
   end
 
+  def streaming?
+    IO === @result or status == :streaming
+  end
+
   def running?
     return nil if not Open.exists? info_file
     return nil if info[:pid].nil?
@@ -239,6 +248,14 @@ class Step
     end
     {:inputs => info[:inputs], :provenance => provenance}
   end
+
+  def provenance_paths
+    provenance = {}
+    dependencies.each do |dep|
+      provenance[dep.path] = dep.provenance_paths if File.exists? dep.path
+    end
+    provenance
+  end
 end
 
 module Workflow

data/lib/rbbt/workflow/step.rb

@@ -9,10 +9,11 @@ class Step
   attr_accessor :path, :task, :inputs, :dependencies, :bindings
   attr_accessor :pid
   attr_accessor :exec
-  attr_accessor :result
+  attr_accessor :result, :mutex
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
+    pat = path.call if Proc === path
     @path = path
     @task = task
     @bindings = bindings
@@ -25,6 +26,7 @@ class Step
                       [dependencies]
                     end
     @mutex = Mutex.new
+    @info_mutex = Mutex.new
     @inputs = inputs || []
   end
 
@@ -59,6 +61,7 @@ class Step
   end
 
   def prepare_result(value, description = nil, info = {})
+    #info = self.info
     case 
     when IO === value
       begin
@@ -98,8 +101,7 @@ class Step
   def get_stream
     @mutex.synchronize do
       begin
-        res = @result
-        IO === res ? res : nil
+        IO === @result ? @result : nil
       ensure
         @result = nil
       end
@@ -112,6 +114,7 @@ class Step
   end
 
   def exec(no_load=false)
+    dependencies.each{|dependency| dependency.exec(no_load) }
     @result = _exec
     @result = @result.stream if TSV::Dumper === @result
     no_load ? @result : prepare_result(@result, @task.result_description)
@@ -120,17 +123,17 @@ class Step
   def join
     stream = get_stream
     begin
-      stream.read if stream
+      Misc.consume_stream stream if stream
     rescue
       stream.abort if stream.respond_to? :abort
       raise $!
     ensure
-      stream.join if stream.respond_to? :join
+      stream.join if stream.respond_to? :join and not stream.joined?
     end
 
-    dependencies.each{|dep| dep.join }
 
     if @pid.nil?
+      dependencies.each{|dep| dep.join }
       self
     else
       begin
@@ -140,8 +143,9 @@ class Step
         Log.debug{"Process #{ @pid } already finished: #{ path }"}
       end if Misc.pid_exists? @pid
       @pid = nil
+      dependencies.each{|dep| dep.join }
+      self
     end
-    self
   end
 
   def checks
@@ -150,110 +154,113 @@ class Step
 
   def run(no_load = false)
 
-    result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load ? :stream : false do
-      if Step === Step.log_relay_step and not self == Step.log_relay_step
-        relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
-      end
-      @exec = false
+    @mutex.synchronize do
+      result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load ? :stream : false do
+        if Step === Step.log_relay_step and not self == Step.log_relay_step
+          relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
+        end
+        @exec = false
+
+        Open.rm info_file if Open.exists? info_file
+
+        set_info :pid, Process.pid
+        set_info :issued, Time.now
+
+        log(:preparing, "Preparing job: #{Misc.fingerprint dependencies}")
+        set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name]}
+        seen_deps = []
+        dependencies.uniq.each{|dependency| 
+          Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""}"
+          begin
+            next if seen_deps.include? dependency.path
+            dependency.relay_log self
+            dependency.clean if not dependency.done? and dependency.error?
+            dependency.clean if dependency.streaming? and not dependency.running?
+            dependency.run true unless dependency.result 
+            seen_deps.concat dependency.rec_dependencies.collect{|d| d.path} 
+          rescue Exception
+            backtrace = $!.backtrace
+            set_info :backtrace, backtrace 
+            log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
+            raise $!
+          end
+        }
 
-      Open.rm info_file if Open.exists? info_file
+        set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
-      set_info :pid, Process.pid
-      set_info :issued, Time.now
+        set_info :started, (start_time = Time.now)
+        log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
 
-      set_info :dependencies, dependencies.collect{|dep| [dep.task.name, dep.name]}
-      log(:preparing, "Preparing job")
-      seen_deps = []
-      dependencies.each{|dependency| 
-        Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task.name.to_s || ""}"
         begin
-          next if seen_deps.include? dependency.path
-          dependency.relay_log self
-          dependency.clean if not dependency.done? and dependency.error?
-          dependency.run true unless dependency.done?
-          seen_deps.concat dependency.rec_dependencies.collect{|d| d.path} 
+          result = _exec
+        rescue Aborted
+          log(:error, "Aborted")
+
+          children_pids = info[:children_pids]
+          if children_pids and children_pids.any?
+            Log.medium("Killing children: #{ children_pids * ", " }")
+            children_pids.each do |pid|
+              Log.medium("Killing child #{ pid }")
+              begin
+                Process.kill "INT", pid
+              rescue Exception
+                Log.medium("Exception killing child #{ pid }: #{$!.message}")
+              end
+            end
+          end
+
+          raise $!
         rescue Exception
           backtrace = $!.backtrace
+
+          # HACK: This fixes an strange behaviour in 1.9.3 where some
+          # backtrace strings are coded in ASCII-8BIT
+          backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
+
           set_info :backtrace, backtrace 
-          log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
+          log(:error, "#{$!.class}: #{$!.message}")
           raise $!
         end
-      }
-
-      set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
-      set_info :started, (start_time = Time.now)
-      log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-
-      begin
-        result = _exec
-      rescue Aborted
-        log(:error, "Aborted")
-
-        children_pids = info[:children_pids]
-        if children_pids and children_pids.any?
-          Log.medium("Killing children: #{ children_pids * ", " }")
-          children_pids.each do |pid|
-            Log.medium("Killing child #{ pid }")
+        case result
+        when IO
+          log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+          ConcurrentStream.setup result do
+            begin
+              set_info :done, (done_time = Time.now)
+              set_info :time_elapsed, (time_elapsed = done_time - start_time)
+              log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
+            rescue
+              Log.exception $!
+            end
+          end
+        when TSV::Dumper
+          log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+          ConcurrentStream.setup result.stream do
             begin
-              Process.kill "INT", pid
-            rescue Exception
-              Log.medium("Exception killing child #{ pid }: #{$!.message}")
+              set_info :done, (done_time = Time.now)
+              set_info :done, (done_time = Time.now)
+              set_info :time_elapsed, (time_elapsed = done_time - start_time)
+              log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
+            rescue
+              Log.exception $!
             end
           end
+        else
+          set_info :done, (done_time = Time.now)
+          set_info :time_elapsed, (time_elapsed = done_time - start_time)
+          log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
         end
 
-        raise $!
-      rescue Exception
-        backtrace = $!.backtrace
-
-        # HACK: This fixes an strange behaviour in 1.9.3 where some
-        # backtrace strings are coded in ASCII-8BIT
-        backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
-
-        set_info :backtrace, backtrace 
-        log(:error, "#{$!.class}: #{$!.message}")
-        raise $!
+        result
       end
 
-      case result
-      when IO
-        log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-        ConcurrentStream.setup result do
-          begin
-            set_info :done, (done_time = Time.now)
-            set_info :time_elapsed, (time_elapsed = done_time - start_time)
-            log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
-          rescue
-            Log.exception $!
-          end
-        end
-      when TSV::Dumper
-        log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
-        ConcurrentStream.setup result.stream do
-          begin
-            set_info :done, (done_time = Time.now)
-            set_info :done, (done_time = Time.now)
-            set_info :time_elapsed, (time_elapsed = done_time - start_time)
-            log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
-          rescue
-            Log.exception $!
-          end
-        end
+      if no_load
+        @result ||= result
+        self
       else
-        set_info :done, (done_time = Time.now)
-        set_info :time_elapsed, (time_elapsed = done_time - start_time)
-        log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+        @result = prepare_result result, @task.result_description
       end
-
-      result
-    end
-
-    if no_load
-      @result = result
-      self
-    else
-      @result = prepare_result result, @task.result_description, info
     end
   end
 
@@ -266,11 +273,10 @@ class Step
         begin
           res = run(true)
           io = get_stream
-          #io = res.result  if IO === res.result
-          #io = res.result.stream  if TSV::Dumper === res.result
           if IO === io
-            io.read 
-            io.join if io.respond_to? :join
+            Misc.consume_stream(io)
+            io.close unless io.closed?
+            io.join if io.respond_to? :join and not io.joined?
           end
         rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
@@ -363,6 +369,9 @@ class Step
       rescue Exception
       end
 
+      @result = nil
+      @pid = nil
+
       begin
         Open.rm info_file if Open.exists? info_file
         Open.rm info_file + '.lock' if Open.exists? info_file + '.lock'
@@ -378,14 +387,17 @@ class Step
 
     # A step result with no info_file means that it was manually
     # placed. In that case, do not consider its dependencies
-    return [] if self.done? and not Open.exists? self.info_file
+    return [] if Open.exists?(self.path.to_s) and not Open.exists? self.info_file
 
     return [] if dependencies.nil? or dependencies.empty?
     new_dependencies = dependencies.collect{|step| 
       step.rec_dependencies 
-    }.flatten
+    }.flatten.uniq.compact
 
-    dependencies + new_dependencies
+    dependencies = self.dependencies ? self.dependencies + new_dependencies : new_dependencies
+    dependencies.flatten!
+    dependencies.uniq!
+    dependencies
   end
 
   def recursive_clean
@@ -398,7 +410,8 @@ class Step
   end
 
   def step(name)
-    rec_dependencies.select do |step| 
+    @steps ||= {}
+    @steps[name] ||= rec_dependencies.select do |step| 
       step.task_name.to_sym == name.to_sym
     end.first
   end

data/share/rbbt_commands/workflow/task

@@ -81,7 +81,7 @@ def fix_options(workflow, task, job_options)
   job_options.each do |name, value|
     value = case input_types[name].to_sym
             when :boolean
-              TrueClass == value or %w(true TRUE T yes).include? value
+              TrueClass === value or %w(true TRUE T yes).include? value
             when :float
               value.to_f
             when :integer
@@ -322,7 +322,7 @@ begin
 
   if options.delete(:provenance)
     job.join
-    pp job.provenance
+    pp job.provenance_paths
     exit 0
   end
 

data/test/rbbt/util/test_log.rb

@@ -2,6 +2,14 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
 require 'rbbt/util/log'
 
 class TestLog < Test::Unit::TestCase
+  def test_get_level
+    assert_equal 0, Log.get_level(:debug)
+    assert_equal 1, Log.get_level(:low)
+    assert_equal 1, Log.get_level("LOW")
+    assert_equal 1, Log.get_level(1)
+    assert_equal 0, Log.get_level(nil)
+  end
+
   def test_color
     assert Log.color(:green, "green")
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.11.1
+  version: 5.11.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-05 00:00:00.000000000 Z
+date: 2014-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake