rbbt-util 5.10.2 → 5.11.1

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -28,9 +28,11 @@ class RbbtProcessQueue
 
         rescue ClosedStream
         rescue Aborted
-          Log.exception $!
+          Log.error "Worker #{Process.pid} aborted"
+          Kernel.exit! -1
         rescue Exception
           @callback_queue.push($!) if @callback_queue
+          Kernel.exit! -1
         ensure
           @callback_queue.close_write if @callback_queue 
         end

data/lib/rbbt/util/log.rb

@@ -72,6 +72,7 @@ module Log
     end
   end
 
+  LOG_MUTEX = Mutex.new
   def self.log(message = nil, severity = MEDIUM, &block)
     return if severity < self.severity 
     message ||= block.call if block_given?
@@ -85,8 +86,10 @@ module Log
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message
 
-    STDERR.puts str
-    logfile.puts str unless logfile.nil?
+    LOG_MUTEX.synchronize do
+      STDERR.puts str
+      logfile.puts str unless logfile.nil?
+    end
   end
 
   def self.debug(message = nil, &block)
@@ -194,14 +197,6 @@ def iii(message, file = $stdout)
   Log.info{""}
 end
 
-def www(message, file = $stdout)
-  stack = caller
-  Log.warn{"#{Log.color :cyan, "INFO:"} " << stack.first}
-  Log.warn{""}
-  Log.warn{"=> " << message.inspect}
-  Log.warn{""}
-end
-
 def eee(message, file = $stdout)
   stack = caller
   Log.error{"#{Log.color :cyan, "INFO:"} " << stack.first}

data/lib/rbbt/util/misc.rb

@@ -21,6 +21,12 @@ class FieldNotFoundError < Exception;end
 class Aborted < Exception; end
 class TryAgain < Exception; end
 class ClosedStream < Exception; end
+class KeepLocked < Exception
+  attr_accessor :payload
+  def initialize(payload)
+    @payload = payload
+  end
+end
 
 module LaterString
   def to_s
@@ -31,27 +37,35 @@ end
 module ConcurrentStream
   attr_accessor :threads, :pids, :callback, :filename
 
-  def consume
-    Thread.pass while IO.select([self], nil, nil).nil? #if IO === self
-    while block = self.read(2048)
-      Thread.pass while IO.select([self], nil, nil).nil? # if IO === self
+  def join
+
+    if @threads and @threads.any?
+      @threads.each do |t| 
+        t.join 
+      end
+      @threads = []
+    end
+
+    if @pids and @pids.any?
+      @pids.each do |pid| 
+        begin
+          Process.waitpid(pid, Process::WUNTRACED)
+          raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
+        rescue Errno::ECHILD
+        end
+      end 
+      @pids = []
     end
-  end
 
-  def join
-    filename = self.respond_to?(:filename)? self.filename : :none
     if @callback
       @callback.call
+      @callback = nil
     end
-    @threads.each{|t| t.join } if @threads
-    
-    @pids.each do |pid| 
-      begin
-        Process.waitpid(pid, Process::WUNTRACED)
-        raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
-      rescue Errno::ECHILD
-      end
-    end if @pids
+  end
+
+  def abort
+    @threads.each{|t| t.raise Aborted.new } if @threads
+    @pids.each{|pid| Process.kill :INT, pid } if @pids
   end
 
   def self.setup(stream, options = {}, &block)
@@ -116,9 +130,11 @@ module Misc
     end
   end
 
-  def self.open_pipe(do_fork = false, other_stream = nil)
+  def self.open_pipe(do_fork = false, close = true)
     raise "No block given" unless block_given?
+
     sout, sin = Misc.pipe
+
     if do_fork
       parent_pid = Process.pid
       pid = Process.fork {
@@ -131,22 +147,23 @@ module Misc
           Process.kill :INT, parent_pid
           Kernel.exit! -1
         ensure
-          Misc.release_pipes(sin)
+          sin.close unless sin.closed? 
         end
         Kernel.exit! 0
       }
-      Misc.release_pipes(sin)
+      sin.close if close
+      ConcurrentStream.setup sout, :pids => [pid]
     else
       thread = Thread.new(Thread.current) do |parent|
         begin
           yield sin
         rescue
-          Log.exception $!
           parent.raise $!
         ensure
-          Misc.release_pipes(sin)
+          sin.close if close 
         end
       end
+      ConcurrentStream.setup sout, :threads => [thread]
     end
     sout
   end
@@ -161,20 +178,24 @@ module Misc
       stream_out2.close
       begin
         filename = stream.respond_to?(:filename)? stream.filename : nil
+        skip1 = skip2 = false
         while block = stream.read(2048)
-          begin stream_in1.write block; rescue Exception;  Log.exception $! end 
-          begin stream_in2.write block; rescue Exception;  Log.exception $! end 
+          begin stream_in1.write block; rescue Exception;  Log.exception $!; skip1 = true end unless skip1 
+          begin stream_in2.write block; rescue Exception;  Log.exception $!; skip2 = true end unless skip2 
         end
+        raise "Error writing in stream_in2" if skip2
+        raise "Error writing in stream_in2" if skip2
+      rescue Aborted
+        stream.abort if stream.respond_to? :abort
+        raise $!
       rescue IOError
         Log.exception $!
       rescue Exception
         Log.exception $!
       ensure
-        if stream.respond_to? :join
-          stream.join
-        end
-        Misc.release_pipes(stream_in1)
-        Misc.release_pipes(stream_in2)
+        stream_in1.close 
+        stream_in2.close 
+        stream.join if stream.respond_to? :join
       end
     end
     stream.close
@@ -195,21 +216,23 @@ module Misc
     splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
       begin
         filename = stream.respond_to?(:filename)? stream.filename : nil
+        skip1 = skip2 = false
         while block = stream.read(2048)
-          begin stream_in1.write block; rescue Exception;  Log.exception $! end 
-          begin stream_in2.write block; rescue Exception;  Log.exception $! end 
+          begin stream_in1.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip1 = true end unless skip1 
+          begin stream_in2.write block; rescue Exception; Aborted === $! ? raise($!): Log.exception($!); skip2 = true end unless skip2 
         end
+      rescue Aborted
+        stream.abort if stream.respond_to? :abort
+        raise $!
       rescue IOError
         Log.exception $!
       rescue Exception
         Log.exception $!
         parent.raise $!
       ensure
-        if stream.respond_to? :join
-          stream.join
-        end
-        Misc.release_pipes(stream_in1)
-        Misc.release_pipes(stream_in2)
+        stream_in1.close 
+        stream_in2.close 
+        stream.join if stream.respond_to? :join
       end
     end
 
@@ -220,7 +243,7 @@ module Misc
   end
 
   class << self
-    alias tee_stream tee_stream_fork 
+    alias tee_stream tee_stream_thread 
   end
 
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
@@ -494,6 +517,8 @@ module Misc
     case obj
     when nil
       "nil"
+    when (defined? Step and Step)
+      obj.path || Misc.fingerprint([obj.task.name, obj.inputs])
     when TrueClass
       "true"
     when FalseClass
@@ -1276,8 +1301,8 @@ end
     @hostanem ||= `hostname`.strip
   end
 
-  def self.lock(file, *args)
-    return yield file, *args if file.nil?
+  def self.lock(file, unlock = true)
+    return yield if file.nil?
     FileUtils.mkdir_p File.dirname(File.expand_path(file)) unless File.exists?  File.dirname(File.expand_path(file))
 
     res = nil
@@ -1287,27 +1312,34 @@ end
 
     begin
       Misc.insist 3 do
-        if File.exists? lockfile and
-          Misc.hostname == (info = Open.open(lockfile){|f| YAML.load(f) })["host"] and 
+        if File.exists? lock_path and
+          Misc.hostname == (info = Open.open(lock_path){|f| YAML.load(f) })["host"] and 
           info["pid"] and not Misc.pid_exists?(info["pid"])
 
-          Log.info("Removing lockfile: #{lockfile}. This pid #{Process.pid}. Content: #{info.inspect}")
-          FileUtils.rm lockfile 
+          Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
+          FileUtils.rm lock_path 
         end
       end
     rescue
-      Log.warn("Error checking lockfile #{lockfile}: #{$!.message}. Removing. Content: #{begin Open.read(lockfile) rescue "Could not open file" end}")
-      FileUtils.rm lockfile if File.exists?(lockfile)
+      Log.warn("Error checking lockfile #{lock_path}: #{$!.message}. Removing. Content: #{begin Open.read(lock_path) rescue "Could not open file" end}")
+      FileUtils.rm lock_path if File.exists?(lock_path)
       lockfile = Lockfile.new(lock_path)
       retry
     end
 
     begin
-      lockfile.lock do 
-        res = yield file, *args
+      lockfile.lock 
+      res = yield lockfile
+    rescue Lockfile::StolenLockError
+      unlock = false
+    rescue KeepLocked
+      unlock = false
+      res = $!.payload
+    ensure
+      if unlock and lockfile.locked?
+        lockfile.unlock
+        FileUtils.rm lock_path if File.exists? lock_path
       end
-    rescue Interrupt
-      raise $!
     end
 
     res
@@ -1315,7 +1347,6 @@ end
   
 
   LOCK_REPO_SERIALIZER=Marshal
-
   def self.lock_in_repo(repo, key, *args)
     return yield file, *args if repo.nil? or key.nil?
 
@@ -1402,17 +1433,17 @@ end
 
   def self.sensiblewrite(path, content = nil, &block)
     return if File.exists? path
-    Misc.lock path + '.sensible_write' do
+    tmp_path = path + '.sensible_write'
+    Misc.lock tmp_path  do
       if not File.exists? path
+        FileUtils.rm_f tmp_path if File.exists? tmp_path
         begin
-          tmp_path = path + '.sensible_write'
           case
           when block_given?
             File.open(tmp_path, 'w', &block)
           when String === content
             File.open(tmp_path, 'w') do |f| f.write content end
           when (IO === content or StringIO === content or File === content)
-            #Thread.pass while IO.select([content], nil, nil, 1) if IO === content
             File.open(tmp_path, 'w') do |f|  
               while block = content.read(2048); 
                 f.write block
@@ -1488,6 +1519,8 @@ end
         str << remove_long_items(v)
       when Array === v
         str << k.to_s << "=>[" << v * "," << "]"
+      when File === v
+        str << k.to_s << "=>[File:" << v.path << "]"
       else
         v_ins = v.inspect
 

data/lib/rbbt/workflow.rb

@@ -138,7 +138,8 @@ module Workflow
     if wf_name =~ /::\w+$/
       clean_name = wf_name.sub(/::.*/,'')  
       Log.info{"Looking for '#{wf_name}' in '#{clean_name}'"}
-      wf_name = clean_name
+      require_workflow clean_name
+      return Misc.string2const Misc.camel_case(wf_name)
     end
 
     Log.info{"Loading workflow #{wf_name}"}

data/lib/rbbt/workflow/step.rb

@@ -24,9 +24,14 @@ class Step
                     else
                       [dependencies]
                     end
+    @mutex = Mutex.new
     @inputs = inputs || []
   end
 
+  def task_name
+    @task.name
+  end
+
   def path
     @path = Misc.sanitize_filename(Path.setup(@path.call)) if Proc === @path
     @path
@@ -43,10 +48,9 @@ class Step
         attr_accessor :relay_step
         alias original_log log 
         def log(status, message = nil)
-          Log.with_severity 10 do
-            original_log(status, message)
-          end
-          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : [task.name.to_s, message] * ">")
+          self.status = status
+          message message
+          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message )
         end
       end
     end
@@ -70,7 +74,10 @@ class Step
         else
           value.read
         end
+      rescue Exception
+        value.abort if value.respond_to? :abort
       ensure
+        value.close unless value.closed?
         value.join if value.respond_to? :join
       end
     when (not defined? Entity or description.nil? or not Entity.formats.include? description)
@@ -88,6 +95,17 @@ class Step
     end
   end
 
+  def get_stream
+    @mutex.synchronize do
+      begin
+        res = @result
+        IO === res ? res : nil
+      ensure
+        @result = nil
+      end
+    end
+  end
+
   def _exec
     @exec = true if @exec.nil?
     @task.exec_in((bindings ? bindings : self), *@inputs)
@@ -100,13 +118,18 @@ class Step
   end
 
   def join
-    case @result
-    when IO
-      while @result.read 2048; Thread.pass end unless @result.closed? or @result.eof?
-      @result.join if @result.respond_to? :join
-      @result = nil
+    stream = get_stream
+    begin
+      stream.read if stream
+    rescue
+      stream.abort if stream.respond_to? :abort
+      raise $!
+    ensure
+      stream.join if stream.respond_to? :join
     end
 
+    dependencies.each{|dep| dep.join }
+
     if @pid.nil?
       self
     else
@@ -140,12 +163,15 @@ class Step
 
       set_info :dependencies, dependencies.collect{|dep| [dep.task.name, dep.name]}
       log(:preparing, "Preparing job")
+      seen_deps = []
       dependencies.each{|dependency| 
         Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task.name.to_s || ""}"
         begin
+          next if seen_deps.include? dependency.path
           dependency.relay_log self
           dependency.clean if not dependency.done? and dependency.error?
-          dependency.run true
+          dependency.run true unless dependency.done?
+          seen_deps.concat dependency.rec_dependencies.collect{|d| d.path} 
         rescue Exception
           backtrace = $!.backtrace
           set_info :backtrace, backtrace 
@@ -157,7 +183,7 @@ class Step
       set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
       set_info :started, (start_time = Time.now)
-      log :started, "#{Log.color :magenta, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
+      log :started, "#{Log.color :green, "Starting task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
 
       begin
         result = _exec
@@ -191,26 +217,33 @@ class Step
       end
 
       case result
-      when IO, StringIO
+      when IO
         log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
         ConcurrentStream.setup result do
-          eee 1
-          set_info :done, (done_time = Time.now)
-          set_info :time_elapsed, (time_elapsed = done_time - start_time)
-          log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+          begin
+            set_info :done, (done_time = Time.now)
+            set_info :time_elapsed, (time_elapsed = done_time - start_time)
+            log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+          rescue
+            Log.exception $!
+          end
         end
       when TSV::Dumper
         log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
         ConcurrentStream.setup result.stream do
-          set_info :done, (done_time = Time.now)
-          set_info :done, (done_time = Time.now)
-          set_info :time_elapsed, (time_elapsed = done_time - start_time)
-          log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+          begin
+            set_info :done, (done_time = Time.now)
+            set_info :done, (done_time = Time.now)
+            set_info :time_elapsed, (time_elapsed = done_time - start_time)
+            log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+          rescue
+            Log.exception $!
+          end
         end
       else
         set_info :done, (done_time = Time.now)
         set_info :time_elapsed, (time_elapsed = done_time - start_time)
-        log :done, "#{Log.color :magenta, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
+        log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i}"
       end
 
       result
@@ -232,9 +265,13 @@ class Step
         FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
         begin
           res = run(true)
-          io = res.result  if IO === res.result
-          io = res.result.stream  if TSV::Dumper === res.result
-          while not io.eof?; io.read(2048); end if io
+          io = get_stream
+          #io = res.result  if IO === res.result
+          #io = res.result.stream  if TSV::Dumper === res.result
+          if IO === io
+            io.read 
+            io.join if io.respond_to? :join
+          end
         rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Aborted"
@@ -275,13 +312,20 @@ class Step
 
   def abort
     @pid ||= info[:pid]
-    if @pid.nil? and info[:forked] 
+
+    return true unless info[:forked]
+
+    case @pid
+    when nil
       Log.medium "Could not abort #{path}: no pid"
       false
+    when Process.pid
+      Log.medium "Could not abort #{path}: same process"
+      false
     else
       Log.medium "Aborting #{path}: #{ @pid }"
       begin
-        Process.kill("KILL", @pid) unless Process.pid == @pid
+        Process.kill("KILL", @pid)
         Process.waitpid @pid
       rescue Exception
         Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
@@ -289,6 +333,7 @@ class Step
       log(:aborted, "Job aborted by user")
       true
     end
+    log(:aborted, "Job aborted by user")
   end
 
   def child(&block)
@@ -335,9 +380,12 @@ class Step
     # placed. In that case, do not consider its dependencies
     return [] if self.done? and not Open.exists? self.info_file
 
-    dependencies.collect{|step| 
+    return [] if dependencies.nil? or dependencies.empty?
+    new_dependencies = dependencies.collect{|step| 
       step.rec_dependencies 
-    }.flatten.concat  dependencies
+    }.flatten
+
+    dependencies + new_dependencies
   end
 
   def recursive_clean
@@ -350,6 +398,8 @@ class Step
   end
 
   def step(name)
-    rec_dependencies.select{|step| step.task.name.to_sym == name.to_sym}.first
+    rec_dependencies.select do |step| 
+      step.task_name.to_sym == name.to_sym
+    end.first
   end
 end