rbbt-util 5.23.27 → 5.23.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 727fd7ea14a972a4bbd10a93b3d5ff270f049d7d
-  data.tar.gz: 8f60c11ea14eca70d231e8a4e55208a51207d989
+  metadata.gz: 2275f8b0ef9e0fd2bf6a6b964aa0bb8541f7bd35
+  data.tar.gz: 3ddc1b829d1990a27fbbfec708cc41e89a7b4926
 SHA512:
-  metadata.gz: 4bfcfd6f510295128945a4f3a759b755777063776f5b9478e3c94865854ecbc755714d61a9bb783b5beb18fd88f85ce03b4e87aa26f4bf794e40acee282f3972
-  data.tar.gz: 9c7d620f5acd24e129c210fe35476b8f055bb5170ce5698217b90406fc314b3b9a02d2d80142a5c744f5817f0c84ccdae6a1dd4a57b34703aa47a4a6b70ba8c9
+  metadata.gz: fd83f6e4487f0f36f33720ded3a875cb123c4bb1398b69ab49affb229af0b1b5e63a464ea74bf1a4946929a68bf334087163de762cc1edc6b7ae85c4cd908947
+  data.tar.gz: 19109b2be3083248f31b1fb9cad1596b201f4801566d7c3a608fe24536557e44611f5a224f9cdd1e7a76aab17c7e3834c84037243e2a16254b80b7903319c6d6

data/lib/rbbt/tsv/excel.rb

@@ -105,7 +105,7 @@ module TSV
       workbook = Spreadsheet.open Open.open(file)
 
       if sheet && sheet.to_s =~ /^\d+$/
-        sheet = workbook.worksheets.collect{|s| s.sheet_name }[sheet.to_i]
+        sheet = workbook.worksheets.collect{|s| s.name }[sheet.to_i]
       end
       sheet_name = sheet
       Log.debug "Opening LSX #{file} sheet #{ sheet_name }"

data/lib/rbbt/util/cmd.rb

@@ -1,5 +1,6 @@
 require 'rbbt/util/log'
 require 'stringio'
+require 'open3'
 
 module CMD
 
@@ -53,50 +54,58 @@ module CMD
 
     in_content = StringIO.new in_content if String === in_content
 
-    sout, serr, sin = Misc.pipe, Misc.pipe, Misc.pipe
+    #sout, serr, sin = Misc.pipe, Misc.pipe, Misc.pipe
 
-    pid = fork {
-      begin
-        Misc.purge_pipes(sin.last,sout.last,serr.last)
+    #pid = fork {
+    #  begin
+    #    Misc.purge_pipes(sin.last,sout.last,serr.last)
 
-        sin.last.close
-        sout.first.close
-        serr.first.close
+    #    sin.last.close
+    #    sout.first.close
+    #    serr.first.close
 
-        if IO === in_content
-          in_content.close if in_content.respond_to?(:close) and not in_content.closed?
-        end
+    #    if IO === in_content
+    #      in_content.close if in_content.respond_to?(:close) and not in_content.closed?
+    #    end
 
-        STDERR.reopen serr.last
-        serr.last.close
+    #    STDERR.reopen serr.last
+    #    serr.last.close
 
-        STDIN.reopen sin.first
-        sin.first.close
+    #    STDIN.reopen sin.first
+    #    sin.first.close
 
-        STDOUT.reopen sout.last
-        sout.last.close
+    #    STDOUT.reopen sout.last
+    #    sout.last.close
 
-        STDOUT.sync = STDERR.sync = true
+    #    STDOUT.sync = STDERR.sync = true
 
-        exec(ENV, cmd)
+    #    exec(ENV, cmd)
 
-        exit(-1)
-      rescue Exception
-        Log.debug{ "ProcessFailed: #{$!.message}" } if log
-        Log.debug{ "Backtrace: \n" + $!.backtrace * "\n" } if log
-        raise ProcessFailed, $!.message
-      end
-    }
+    #    exit(-1)
+    #  rescue Exception
+    #    Log.debug{ "ProcessFailed: #{$!.message}" } if log
+    #    Log.debug{ "Backtrace: \n" + $!.backtrace * "\n" } if log
+    #    raise ProcessFailed, $!.message
+    #  end
+    #}
 
-    sin.first.close
-    sout.last.close
-    serr.last.close
+    #sin.first.close
+    #sout.last.close
+    #serr.last.close
 
 
-    sin = sin.last
-    sout = sout.first
-    serr = serr.first
+    #sin = sin.last
+    #sout = sout.first
+    #serr = serr.first
 
+    sin, sout, serr, wait_thr = begin
+                                  Open3.popen3(ENV, cmd)
+                                rescue
+                                  Log.warn $!.message
+                                  raise ProcessFailed, cmd unless no_fail
+                                  return
+                                end
+    pid = wait_thr.pid
 
     Log.debug{"CMD: [#{pid}] #{cmd}" if log}
 
@@ -160,10 +169,9 @@ module CMD
       out = StringIO.new sout.read
       sout.close unless sout.closed?
 
-      Process.waitpid pid
-
-      if not $?.success? and not no_fail
-        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
+      status = wait_thr.value
+      if not status.success? and not no_fail
+        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
       else
         Log.log err, stderr if Integer === stderr and log
       end

data/lib/rbbt/util/concurrency/processes.rb

@@ -126,7 +126,7 @@ class RbbtProcessQueue
                   @count += 1
                   @total -= 1
                   next unless processes.length > 1
-                  last = processes.last
+                  last = processes.reject{|p| p.stopping }.last
                   last.stop
                   Log.warn "Removed process #{last.pid} from #{Process.pid} (#{processes.length})"
                 end

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -1,7 +1,7 @@
 require 'rbbt/util/concurrency/processes/socket'
 class RbbtProcessQueue
   class RbbtProcessQueueWorker
-    attr_reader :pid, :queue, :callback_queue, :cleanup, :block
+    attr_reader :pid, :queue, :callback_queue, :cleanup, :block, :stopping
 
     class Respawn < Exception
       attr_accessor :payload 
@@ -277,6 +277,7 @@ class RbbtProcessQueue
 
     def stop
       begin
+        @stoping = true
         Process.kill :USR2, @pid
       rescue Errno::ESRCH 
       rescue Exception

data/lib/rbbt/util/config.rb

@@ -84,13 +84,14 @@ module Rbbt::Config
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
         l =~ /rbbt\/resource\/path\.rb/ or
         l =~ /rbbt\/util\/misc\.rb/ or
+        l =~ /accessor\.rb/ or
         l =~ /progress-monitor\.rb/ 
     }.first.partition(":")
 
     File.expand_path(file)
 
     tokens << ("file:" << file)
-    tokens << ("line:" << file << ":" << line)
+    tokens << ("line:" << file << ":" << line.sub(/:in \`.*/,''))
 
     entries = CACHE[key.to_s]
     priorities = {}
@@ -105,7 +106,9 @@ module Rbbt::Config
 
     return nil if priorities.empty?
 
-    priorities.sort_by{|p,v| p}.first.last.first
+    value = priorities.sort_by{|p,v| p}.first.last.first
+    Log.debug "Value '#{value}' for config key '#{ key }': #{tokens * ", "}"
+    value
   end
 
   self.load_config

data/lib/rbbt/util/misc/pipes.rb

@@ -252,6 +252,7 @@ module Misc
     end
     tee1, *rest = Misc.tee_stream stream_dup, num + 1
     stream.reopen(tee1)
+    tee1.annotate(stream)
     rest
   end
 

data/lib/rbbt/util/open.rb

@@ -426,7 +426,7 @@ module Open
       exists_in_repo(*dir_sub_path)
     else
       file = file.find if Path === file
-      File.exist?(file) || File.symlink?(file)
+      File.exist?(file) #|| File.symlink?(file)
     end
   end
   class << self
@@ -711,8 +711,8 @@ module Open
       get_time_from_repo(*dir_sub_path)
     else
       file = file.find if Path === file
-      file = Pathname.new(file).realpath.to_s if File.symlink?(file)
       begin
+        file = Pathname.new(file).realpath.to_s if File.symlink?(file)
         File.mtime(file)
       rescue
         nil

data/lib/rbbt/workflow.rb

@@ -315,7 +315,7 @@ module Workflow
     missing_inputs = []
     task.required_inputs.each do |input|
       missing_inputs << input if inputs[input].nil?
-    end
+    end if task.required_inputs
 
     if missing_inputs.length == 1
       raise ParameterException, "Input #{missing_inputs.first} is required but was not provided or is nil"

data/lib/rbbt/workflow/step/run.rb

@@ -436,7 +436,7 @@ class Step
         else
           e = get_exception
           if e
-            Log.error "Raising exception in produced job #{job.path}: #{e.message}" 
+            Log.error "Raising exception in produced job #{self.path}: #{e.message}" 
             raise e
           else
             raise "Error in job: #{self.path}"

data/share/rbbt_commands/system/clean

@@ -97,10 +97,12 @@ TSV.traverse jobs, :_bar => "Checking job status" do |file,i|
     pid = info[:pid]
 
     status = info[:status].to_s
-    status = :missing if status == "done" and not File.exist? file
-    status = :nopid if status != "done" and pid.nil?
-    status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
-    status = :sync if status != "done" and File.exist? file
+    if status != "noinfo"
+      status = :missing if status == "done" and not File.exist? file
+      status = :nopid if status != "done" and pid.nil?
+      status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
+      status = :sync if status != "done" and File.exist? file
+    end
 
     status = :dirty if info[:status].to_s == "done" and dirty and Workflow.load_step(file).dirty?
     if info[:status] == :error

data/share/rbbt_commands/workflow/prov

@@ -120,6 +120,7 @@ def report(step, offset = 0, task = nil)
     path = dep.path
     new = ! $seen.include?(path)
     if new
+      $seen << path
       str << report(dep, offset + 1, task)
     else
       str << Log.color(:blue, Log.uncolor(report(dep, offset+1, task)))

data/test/rbbt/util/test_cmd.rb

@@ -35,8 +35,8 @@ class TestCmd < Test::Unit::TestCase
     assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true) end
     assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true) end
  
-    assert_nothing_raised ProcessFailed do CMD.cmd('fake-command', :stderr => false, :pipe => true) end
-    assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :stderr => false, :pipe => true) end
+    assert_nothing_raised ProcessFailed do CMD.cmd('fake-command', :no_fail => true, :pipe => true) end
+    assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :no_fail => true, :pipe => true) end
  
     assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true, :pipe => true).join end
     assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
@@ -63,4 +63,16 @@ line33
     end
   end
 
+  def test_STDIN_close
+    TmpFile.with_file("Hello") do |file|
+      STDIN.close
+      Open.open(file) do |f|
+        io = CMD.cmd("tr 'e' 'E'", :in => f, :pipe => true)
+        txt = io.read
+        io.join
+        assert_equal "HEllo", txt
+      end
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.27
+  version: 5.23.28
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-24 00:00:00.000000000 Z
+date: 2018-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake