rbbt-util 5.11.3 → 5.11.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: add772f9d207913045a9042098c44b2c5549215c
-  data.tar.gz: 5932efc5c957bc4cf93de0228d4e654a47a788ce
+  metadata.gz: 1b14c182984b18f946f50ba4d5e07fafec05f8b5
+  data.tar.gz: f6cdb39396d9bd4871e74ab0eb63015373e415d9
 SHA512:
-  metadata.gz: 864893d125d7ca6e03736829a5c77d11e936229dfd3569139c22b1acd10d8430156603b8a23f3ecf5c9b0602040278b6fff1482ccddde3cc9ca3686f40c16791
-  data.tar.gz: 71da4b77c4fad1c83fcaa068118150bd6cfbc0e6e029a2bd3ffc2867293fa585cba4a4638eff98df9d4120fe2a95847c290ae7f6b07bfad2699ba15566f1a99c
+  metadata.gz: a51d7e12fb6fa32385092372830f5f3349fd697c0263f561c287cf17860b604c9544254444d0148c214381639f76e4de396b6d32c1d4a3cbf0e8bd631b45f3c2
+  data.tar.gz: d56ed3b1b3a048d5a68f1ad5b17f1548ed993387ff5fc7313b79716faeb31a96e5e7aa289b3a0d7821aeaa60ffbe2767ac771f9aba57c31b990b94186b1a77dc

data/lib/rbbt/persist.rb

@@ -404,7 +404,7 @@ module Persist
 
             res = yield
 
-            if persist_options[:no_load] == :stream
+            if persist_options[:no_load] == :stream 
               case res
               when IO
                 res = tee_stream(res, path, type, res.respond_to?(:callback)? res.callback : nil)
@@ -420,7 +420,6 @@ module Persist
                 res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
                 ConcurrentStream.setup res do
                   begin
-                    iii [:unlock, lockfile.path]
                     lockfile.unlock
                   rescue
                     Log.warn "Lockfile exception: " << $!.message

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -126,11 +126,8 @@ module TSV
         obj.abort if obj.respond_to? :abort
         raise $!
       ensure
-        begin
-          obj.close if obj.respond_to? :close and not obj.closed?
-        ensure
-          obj.join if obj.respond_to? :join
-        end
+        obj.close if obj.respond_to? :close and not obj.closed?
+        obj.join if obj.respond_to? :join
       end
     when Path
       obj.open do |stream|
@@ -193,15 +190,24 @@ module TSV
       q.init &block
 
       pid = Process.fork do 
+      #thread = Thread.new do 
         Misc.purge_pipes(q.queue.swrite)
         traverse_obj(obj, options) do |*p|
           q.process *p
         end
       end
 
+      #thread.join
       Process.waitpid pid
+      raise "Traversal process ended with error status" unless $?.success?
     rescue Exception
-      Log.error "Exception traversin in cpus: #{$!.message}"
+      Log.error "Exception traversing in cpus: #{$!.message}"
+      Log.exception $!
+
+      stream = obj_stream(obj)
+      stream.abort if stream.respond_to? :abort
+      stream = obj_stream(options[:into])
+      stream.abort if stream.respond_to? :abort
       q.abort
       raise $!
     ensure

data/lib/rbbt/util/concurrency/processes.rb

@@ -60,15 +60,19 @@ class RbbtProcessQueue
       rescue Exception
         Log.error "Process monitor exception: #{$!.message}"
         @processes.each{|p| p.abort }
-        @callback_thread.raise Aborted.new if @callback_thread
+        @callback_thread.raise $! if @callback_thread
         parent.raise $!
       end
     end
   end
 
   def close_callback
-    @callback_queue.push ClosedStream.new if  @callback_thread.alive?
-    @callback_thread.join 
+    begin
+      @callback_queue.push ClosedStream.new if @callback_thread.alive?
+    rescue
+      Log.error "Error closing callback: #{$!.message}"
+    end
+    @callback_thread.join  if @callback_thread.alive?
   end
 
   def join

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -20,6 +20,7 @@ class RbbtProcessQueue
           Signal.trap(:INT){ raise Aborted; }
           loop do
             p = @queue.pop
+            next if p.nil?
             raise p if Exception === p
             raise p.first if Exception === p.first
             res = @block.call *p

data/lib/rbbt/util/misc.rb

@@ -71,8 +71,11 @@ module ConcurrentStream
 
   def abort
     @threads.each{|t| t.raise Aborted.new } if @threads
+    @threads.each{|t| t.join } if @threads
     @pids.each{|pid| Process.kill :INT, pid } if @pids
+    @pids.each{|pid| Process.waitpid pid } if @pids
     @abort_callback.call if @abort_callback
+    @abort_callback = nil
   end
 
   def self.setup(stream, options = {}, &block)
@@ -253,8 +256,30 @@ module Misc
     alias tee_stream tee_stream_thread 
   end
 
+  def self.read_full_stream(io)
+    str = ""
+    begin
+      while block = io.read(2048)
+        str << block
+      end
+    rescue
+      io.abort if io.respond_to? :abort
+    ensure
+      io.join if io.respond_to? :join
+      io.close if io.respond_to? :close
+    end
+    str
+  end
+
   def self.consume_stream(io)
-    Thread.pass while block = io.read(2048)
+    begin
+      Thread.pass while block = io.read(2048)
+    rescue
+      io.abort if io.respond_to? :abort
+    ensure
+      io.join if io.respond_to? :join
+      io.close if io.respond_to? :close
+    end
   end
 
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
@@ -332,9 +357,27 @@ module Misc
   end
 
   def self.read_stream(stream, size)
+    str = nil
+    Thread.pass while IO.select([stream],nil,nil,1).nil?
+    while not str = stream.read(size)
+      IO.select([stream],nil,nil,1) 
+      Thread.pass
+      raise ClosedStream if stream.eof?
+    end
+
+    while str.length < size
+      raise ClosedStream if stream.eof?
+      IO.select([stream],nil,nil,1)
+      if new = stream.read(size-str.length)
+        str << new
+      end
+    end
+    str
+  end
+  def self._read_stream(stream, size)
     str = ""
     while (len=str.length) < size
-      str << stream.read(size-len)
+      str << (stream.read(size-len) or break)
     end
     str
   end
@@ -1195,6 +1238,11 @@ end
   end
 
   def self.insist(times = 3, sleep = nil, msg = nil)
+    if Array === times
+      sleep_array = times
+      times = sleep_array.length
+      sleep = sleep_array.shift
+    end
     try = 0
     begin
       yield
@@ -1206,6 +1254,7 @@ end
       end
       if sleep and try > 0
         sleep sleep
+        sleep = sleep_array.shift if sleep_array
       else
         Thread.pass
       end
@@ -1332,47 +1381,28 @@ end
 
     hostname = Misc.hostname
     LOCK_MUTEX.synchronize do
-      Misc.insist 3, 0.1 do
-        begin
-          if File.exists? lock_path
-            info = Open.open(lock_path){|f| YAML.load(f) }
-            raise "No info" unless info
-
-            if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
-              Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
-              FileUtils.rm lock_path
+      Misc.insist 2, 0.1 do
+        Misc.insist 3, 0.1 do
+          begin
+            if File.exists? lock_path
+              info = Open.open(lock_path){|f| YAML.load(f) }
+              raise "No info" unless info
+
+              if hostname == info["host"] and not Misc.pid_exists?(info["pid"])
+                Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
+                FileUtils.rm lock_path
+              end
             end
+          rescue Exception
+            FileUtils.rm lock_path if File.exists? lock_path
+            lockfile = Lockfile.new(lock_path) unless File.exists? lock_path
+            raise $!
           end
-        rescue Exception
-          FileUtils.rm lock_path if File.exists? lock_path
-          raise $!
-        ensure
-          lockfile = Lockfile.new(lock_path) unless File.exists? lock_path
         end
       end
     end
 
-    #begin
-    #  Misc.insist 3 do
-    #    LOCK_MUTEX.synchronize do
-    #      if File.exists? lock_path and
-    #        Misc.hostname == (info = Open.open(lock_path){|f| YAML.load(f) })["host"] and 
-    #        info["pid"] and not Misc.pid_exists?(info["pid"])
-
-    #        Log.info("Removing lockfile: #{lock_path}. This pid #{Process.pid}. Content: #{info.inspect}")
-    #        FileUtils.rm lock_path 
-    #      end
-    #    end
-    #  end
-    #rescue
-    #  Log.warn("Error checking lockfile #{lock_path}: #{$!.message}. Removing. Content: #{begin Open.read(lock_path) rescue "Could not open file" end}")
-    #  FileUtils.rm lock_path if File.exists?(lock_path)
-    #  lockfile = Lockfile.new(lock_path)
-    #  retry
-    #end
-
     begin
-      
       lockfile.lock 
       res = yield lockfile
     rescue Lockfile::StolenLockError
@@ -1498,6 +1528,7 @@ end
           end
           FileUtils.mv tmp_path, path
         rescue Exception
+          Log.error "Exception in sensiblewrite: #{$!.message} -- #{ Log.color :blue, path }"
           FileUtils.rm_f tmp_path if File.exists? tmp_path
           FileUtils.rm_f path if File.exists? path
           raise $!

data/lib/rbbt/workflow/accessor.rb

@@ -167,7 +167,7 @@ class Step
   end
 
   def done?
-    path and path.exists?
+    path and File.exists? path
   end
 
   def streaming?

data/lib/rbbt/workflow/step.rb

@@ -40,135 +40,173 @@ class Step
   end
 
   class << self
-    attr_accessor :log_relay_step
-  end
+  attr_accessor :log_relay_step
+end
 
-  def relay_log(step)
-    return self unless Task === self.task and not self.task.name.nil?
-    if not self.respond_to? :original_log
-      class << self
-        attr_accessor :relay_step
-        alias original_log log 
-        def log(status, message = nil)
-          self.status = status
-          message message
-          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless relay_step.done? or relay_step.error? or relay_step.aborted?
-        end
+def relay_log(step)
+  return self unless Task === self.task and not self.task.name.nil?
+  if not self.respond_to? :original_log
+    class << self
+      attr_accessor :relay_step
+      alias original_log log 
+      def log(status, message = nil)
+        self.status = status
+        message Log.uncolor message
+        relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
       end
     end
-    @relay_step = step
-    self
   end
+  @relay_step = step
+  self
+end
 
-  def prepare_result(value, description = nil, info = {})
-    #info = self.info
-    case 
-    when IO === value
-      begin
-        case @task.result_type
-        when :array
-          array = []
-          while line = value.gets
-            array << line
-          end
-          array
-        when :tsv
-          TSV.open(value)
-        else
-          value.read
+def prepare_result(value, description = nil, info = {})
+  #info = self.info
+  case 
+  when IO === value
+    begin
+      case @task.result_type
+      when :array
+        array = []
+        while line = value.gets
+          array << line
         end
-      rescue Exception
-        value.abort if value.respond_to? :abort
-      ensure
-        value.join if value.respond_to? :join
-        value.close unless value.closed?
-      end
-    when (not defined? Entity or description.nil? or not Entity.formats.include? description)
-      value
-    when (Annotated === value and info.empty?)
-      value
-    when Annotated === value
-      annotations = value.annotations
-      info.each do |k,v|
-        value.send("#{h}=", v) if annotations.include? k
+        array
+      when :tsv
+        TSV.open(value)
+      else
+        value.read
       end
-      value
-    else
-      Entity.formats[description].setup(value, info.merge(:format => description))
+    rescue Exception
+      value.abort if value.respond_to? :abort
+    ensure
+      value.join if value.respond_to? :join
+      value.close unless value.closed?
+    end
+  when (not defined? Entity or description.nil? or not Entity.formats.include? description)
+    value
+  when (Annotated === value and info.empty?)
+    value
+  when Annotated === value
+    annotations = value.annotations
+    info.each do |k,v|
+      value.send("#{h}=", v) if annotations.include? k
     end
+    value
+  else
+    Entity.formats[description].setup(value, info.merge(:format => description))
   end
+end
 
-  def get_stream
-    @mutex.synchronize do
-      begin
-        IO === @result ? @result : nil
-      ensure
-        @result = nil
-      end
+def get_stream
+  @mutex.synchronize do
+    begin
+      IO === @result ? @result : nil
+    ensure
+      @result = nil
     end
   end
+end
 
-  def _exec
-    @exec = true if @exec.nil?
-    @task.exec_in((bindings ? bindings : self), *@inputs)
-  end
+def _exec
+  @exec = true if @exec.nil?
+  @task.exec_in((bindings ? bindings : self), *@inputs)
+end
 
-  def exec(no_load=false)
-    dependencies.each{|dependency| dependency.exec(no_load) }
-    @result = _exec
-    @result = @result.stream if TSV::Dumper === @result
-    no_load ? @result : prepare_result(@result, @task.result_description)
+def exec(no_load=false)
+  dependencies.each{|dependency| dependency.exec(no_load) }
+  @result = _exec
+  @result = @result.stream if TSV::Dumper === @result
+  no_load ? @result : prepare_result(@result, @task.result_description)
+end
+
+def join
+  stream = get_stream if @result
+  begin
+    Misc.consume_stream stream if stream
+  rescue
+    stream.abort if stream.respond_to? :abort
+    raise $!
+  ensure
+    stream.join if stream.respond_to? :join and not stream.joined?
   end
 
-  def join
-    stream = get_stream
-    begin
-      Misc.consume_stream stream if stream
-    rescue
-      stream.abort if stream.respond_to? :abort
-      raise $!
-    ensure
-      stream.join if stream.respond_to? :join and not stream.joined?
-    end
+  return if not Open.exists? info_file
+  @pid ||= info[:pid]
 
+  #while not done?
+  #  Misc.insist 2, 0.5 do
+  #    raise "Job error while joining: #{info[:messages].last}" if error?
+  #    raise "Job aborted  while joining: #{info[:messages].last}" if aborted?
+  #    raise "Job vanished  while joining: #{@pid}" if @pid and not Misc.pid_exists? @pid
+  #  end
+  #end
 
-    if @pid.nil?
-      dependencies.each{|dep| dep.join }
-      self
-    else
-      begin
-        Log.debug{"Waiting for pid: #{@pid}"}
-        Process.waitpid @pid 
-      rescue Errno::ECHILD
-        Log.debug{"Process #{ @pid } already finished: #{ path }"}
-      end if Misc.pid_exists? @pid
-      @pid = nil
-      dependencies.each{|dep| dep.join }
-      self
-    end
+  Misc.insist [0.1, 0.2, 0.5, 1] do
+    @pid ||= info[:pid]
   end
 
-  def checks
-    rec_dependencies.collect{|dependency| dependency.path }.uniq
+  if @pid.nil?
+    dependencies.each{|dep| dep.join }
+    self
+  else
+    begin
+      Log.debug{"Waiting for pid: #{@pid}"}
+      Process.waitpid @pid 
+    rescue Errno::ECHILD
+      Log.debug{"Process #{ @pid } already finished: #{ path }"}
+    end if Misc.pid_exists? @pid
+    @pid = nil
+    dependencies.each{|dep| dep.join }
+    self
   end
+end
 
-  def kill_children
-    children_pids = info[:children_pids]
-    if children_pids and children_pids.any?
-      Log.medium("Killing children: #{ children_pids * ", " }")
-      children_pids.each do |pid|
-        Log.medium("Killing child #{ pid }")
-        begin
-          Process.kill "INT", pid
-        rescue Exception
-          Log.medium("Exception killing child #{ pid }: #{$!.message}")
-        end
+def checks
+  rec_dependencies.collect{|dependency| dependency.path }.uniq
+end
+
+def kill_children
+  children_pids = info[:children_pids]
+  if children_pids and children_pids.any?
+    Log.medium("Killing children: #{ children_pids * ", " }")
+    children_pids.each do |pid|
+      Log.medium("Killing child #{ pid }")
+      begin
+        Process.kill "INT", pid
+      rescue Exception
+        Log.medium("Exception killing child #{ pid }: #{$!.message}")
       end
     end
   end
+end
 
-  def run(no_load = false)
+def run_dependencies(seen = [])
+  seen << self.path
+  dependencies.uniq.each{|dependency| 
+    Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""}"
+    begin
+      next if seen.include? dependency.path
+      dependency.relay_log self
+      dependency.clean if not dependency.done? and dependency.error? or dependency.aborted?
+      dependency.clean if dependency.streaming? and not dependency.running?
+      dependency.run_dependencies(seen)
+      dependency.run true unless dependency.result or dependency.done?
+      seen << dependency.path
+      seen.concat dependency.rec_dependencies.collect{|d| d.path} 
+    rescue Exception
+      backtrace = $!.backtrace
+      set_info :backtrace, backtrace 
+      log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
+      raise $!
+    end
+  }
+end
 
+def run(no_load = false)
+
+  result = nil
+  begin
     @mutex.synchronize do
       no_load = no_load ? :stream : false
       result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
@@ -184,24 +222,8 @@ class Step
 
         log(:preparing, "Preparing job: #{Misc.fingerprint dependencies}")
         set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name]}
-        seen_deps = []
-        dependencies.uniq.each{|dependency| 
-          Log.info "#{Log.color :magenta, "Checking dependency"} #{Log.color :yellow, task.name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""}"
-          begin
-            next if seen_deps.include? dependency.path
-            dependency.relay_log self
-            dependency.clean if not dependency.done? and dependency.error?
-            dependency.clean if dependency.streaming? and not dependency.running?
-            dependency.run true unless dependency.result or dependency.done?
-            seen_deps << dependency.path
-            seen_deps.concat dependency.rec_dependencies.collect{|d| d.path} 
-          rescue Exception
-            backtrace = $!.backtrace
-            set_info :backtrace, backtrace 
-            log(:error, "Exception processing dependency #{Log.color :yellow, dependency.task.name.to_s} -- #{$!.class}: #{$!.message}")
-            raise $!
-          end
-        }
+
+        run_dependencies
 
         set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
 
@@ -213,19 +235,6 @@ class Step
         rescue Aborted
           log(:error, "Aborted")
 
-          #          children_pids = info[:children_pids]
-          #          if children_pids and children_pids.any?
-          #            Log.medium("Killing children: #{ children_pids * ", " }")
-          #            children_pids.each do |pid|
-          #              Log.medium("Killing child #{ pid }")
-          #              begin
-          #                Process.kill "INT", pid
-          #              rescue Exception
-          #                Log.medium("Exception killing child #{ pid }: #{$!.message}")
-          #              end
-          #            end
-          #          end
-          
           kill_children
           raise $!
         rescue Exception
@@ -240,8 +249,13 @@ class Step
           raise $!
         end
 
+        result = prepare_result result, @task.description, info if IO === result and ENV["RBBT_NO_STREAM"]
+        result = prepare_result result.stream, @task.description, info if TSV::Dumper === result and ENV["RBBT_NO_STREAM"]
+
         case result
         when IO
+          result = Misc.read_stream(result) if ENV["RBBT_NO_STREAM"]
+
           log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
           ConcurrentStream.setup result do
             begin
@@ -250,6 +264,8 @@ class Step
               log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
             rescue
               Log.exception $!
+            ensure
+              join
             end
           end
           result.abort_callback = Proc.new do
@@ -257,6 +273,8 @@ class Step
               log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
             rescue
               Log.exception $!
+            ensure
+              join
             end
           end
         when TSV::Dumper
@@ -269,6 +287,8 @@ class Step
               log :done, "#{Log.color :red, "Completed task"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] +#{time_elapsed.to_i} -- #{path}"
             rescue
               Log.exception $!
+            ensure
+              join
             end
           end
           result.stream.abort_callback = Proc.new do
@@ -294,22 +314,21 @@ class Step
         @result = prepare_result result, @task.result_description
       end
     end
+  ensure
+    join unless no_load
+  end
   end
 
   def fork(semaphore = nil)
-    raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil?
+    raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid
+    iii :forking
     @pid = Process.fork do
       begin
+        iii :forked
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
         begin
-          res = run(true)
-          io = get_stream
-          if IO === io
-            Misc.consume_stream(io)
-            io.close unless io.closed?
-            io.join if io.respond_to? :join and not io.joined?
-          end
+          res = run
         rescue Aborted
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Aborted"
@@ -317,6 +336,8 @@ class Step
         rescue Exception
           Log.debug("Exception '#{$!.message}' caught on forked process: #{path}")
           raise $!
+        ensure
+          join
         end
 
         begin
@@ -351,7 +372,7 @@ class Step
   def abort
     @pid ||= info[:pid]
 
-    return true unless info[:forked]
+    #return true unless info[:forked]
 
     case @pid
     when nil
@@ -368,10 +389,9 @@ class Step
       rescue Exception
         Log.debug("Aborted job #{@pid} was not killed: #{$!.message}")
       end
-      log(:aborted, "Job aborted by user")
+      log(:aborted, "Job aborted")
       true
     end
-    log(:aborted, "Job aborted by user")
   end
 
   def child(&block)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.11.3
+  version: 5.11.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-06 00:00:00.000000000 Z
+date: 2014-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake