rbbt-util 5.26.85 → 5.26.86

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b150d8b85d7565e2ec820343f002c8bfc80e46302d5a35d83b10f132d2ebfc94
-  data.tar.gz: 66359a6ec5533d41ab1bb2344bf6795844e0f4ca46332523647ba13319a7d6e8
+  metadata.gz: fc6102e3c95948e874ce0ef40256083bc5bbb61c27700dbab1e864a3ec14ebe3
+  data.tar.gz: 5267bbf2d76c3a1ec0920fa5ecb03acf711ae541198d220055afae0965fa9630
 SHA512:
-  metadata.gz: 1cf80ad89a6007b291518458b7eb995b2a589ce34230a8fea9b03e83cf196df787f7c46376dfb80d0c8a4d3fdbb83cc906a8a1240ac26bbee25c8bebc67c606c
-  data.tar.gz: 6a9fc1eee7bf113ccefa6aef14d704406fbe68a07a24f0c603102e2cfe3e9863ec4b88a01a3029885b2e98754ad85c215956e2a267b71d894659c041da62ae43
+  metadata.gz: 8b5fa3392feae0691db1b06276294be43f6f8e0c8dd2d553967cf4993277b2cee9284ccb10b4c1b27848c0fc6beb09c5e9b65f5b3235f97180eaad2d2279df05
+  data.tar.gz: 7cc7b7c0207712ff2541d9ced370131c431359f28de3f9aaee4d1ce840f59506c6e05776021f04fe47d8fe461362f7fbfa867cc2982afe760dc451178342ab08

data/bin/rbbt

@@ -55,11 +55,13 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --update_tsv            #{Log.color :yellow, "Update persisted TSV files also if source has been updated"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
 --dump_mem*             #{Log.color :yellow, "Dump strings in memory each second into file"}
--nolock--no_lock_id            #{Log.color :yellow, "Do not track lockfiles with ids (prevent stale file handlers for high-througput and high-concurrency)"}
+-nolock--no_lock_id     #{Log.color :yellow, "Do not track lockfiles with ids (prevent stale file handlers for high-througput and high-concurrency)"}
 -ji--jobname_show_inputs  #{Log.color :yellow, "Show inputs as part of the jobname in workflows instead of digesting them"}
 -ck--config_keys*       #{Log.color :yellow, "Override some config keys"}
+-srand--random_seed*    #{Log.color :yellow, "Set the random seed"}
 EOF
 
+srand(options[:random_seed].to_i) if options[:random_seed]
 if options[:jobname_show_inputs]
   ENV["RBBT_INPUT_JOBNAME"] = "true"
 end

data/lib/rbbt/hpc.rb

@@ -15,6 +15,7 @@ module Marenostrum
     def self.template(args, options = {})
 
       development      = options.delete :drbbt
+      singularity      = options.delete :singularity
       contain          = options.delete :contain
       sync             = options.delete :sync
       contain_and_sync = options.delete :contain_and_sync
@@ -38,7 +39,7 @@ module Marenostrum
 
       contain = File.expand_path(contain) if contain
 
-      singularity = true if contain || ! development
+      singularity = true if contain 
 
 
       name = options[:name] ||= Misc.obj2digest({:options => options.collect{|k,v| [k,v]}.sort_by{|k,v| k.to_s }, :args => args})
@@ -228,7 +229,11 @@ EOF
 
         exec_cmd = singularity_exec + " " + exec_cmd
       else
-        exec_cmd << %(~/git/rbbt-util/bin/rbbt --dev=~/git/)
+        if development
+          exec_cmd << " " << %(~/git/rbbt-util/bin/rbbt --dev=~/git/)
+        else
+          exec_cmd << " " << 'rbbt'
+        end
       end
 
 

data/lib/rbbt/util/log.rb

@@ -34,6 +34,70 @@ module Log
     line.gsub('`', "'")
   end
 
+
+  def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
+    sout, sin = Misc.pipe
+    soute, sine = Misc.pipe
+    backup_stderr = STDERR.dup
+    backup_stdout = STDOUT.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    severity_err ||= severity
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    th_loge = Thread.new do
+      while line = soute.gets
+        Log.logn "#{msge}: " + line, severity_err
+      end
+    end
+
+    begin
+      STDOUT.reopen(sin)
+      STDERR.reopen(sine)
+      yield
+    ensure
+      STDERR.reopen backup_stderr
+      STDOUT.reopen backup_stdout
+      sin.close
+      sine.close
+      th_log.join
+      th_loge.join
+      backup_stdout.close
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+  def self.trap_stderr(msg = "STDERR", severity = 0)
+    sout, sin = Misc.pipe
+    backup_stderr = STDERR.dup
+    old_logfile = Log.logfile
+    Log.logfile(backup_stderr)
+
+    th_log = Thread.new do
+      while line = sout.gets
+        Log.logn "#{msg}: " + line, severity
+      end
+    end
+
+    begin
+      STDERR.reopen(sin)
+      yield
+      sin.close
+    ensure
+      STDERR.reopen backup_stderr
+      th_log.join
+      backup_stderr.close
+      Log.logfile = old_logfile
+    end
+  end
+
+
   def self._ignore_stderr
     backup_stderr = STDERR.dup
     File.open('/dev/null', 'w') do |f|
@@ -47,6 +111,7 @@ module Log
     end
   end
 
+
   def self.ignore_stderr(&block)
     LOG_MUTEX.synchronize do
       _ignore_stderr &block
@@ -174,9 +239,12 @@ module Log
     str = prefix << " " << message.to_s
 
     LOG_MUTEX.synchronize do
-      STDERR.write str
+      if logfile.nil?
+        STDERR.write str
+      else
+        logfile.write str 
+      end
       Log::LAST.replace "log"
-      logfile.write str unless logfile.nil?
       nil
     end
   end
@@ -389,14 +457,3 @@ def eef(obj=nil, file = $stdout)
   Log.log_obj_fingerprint(obj, :error, file)
 end
 
-if __FILE__ == $0
-  Log.severity = 0
-
-  (0..6).each do |level|
-    Log.log("Level #{level}", level)
-  end
-
-  require 'rbbt/util/misc'
-  eee [1,2,3]
-  eef [1,2,3]
-end

data/lib/rbbt/util/python.rb

@@ -1,23 +1,45 @@
 require 'rbbt-util'
-
 require 'pycall/import'
 
 module RbbtPython
   extend PyCall::Import
   def self.run(mod = nil, imports = nil, &block)
-    begin
-      if mod
-        if imports
-          pyfrom mod, :import => imports
-        else
-          pyimport mod 
-        end
+    if mod
+      if imports
+        pyfrom mod, :import => imports
+      else
+        pyimport mod 
       end
+    end
+
+    module_eval(&block)
+  end
+
+  def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
+    if mod
+      if imports
+        pyfrom mod, :import => imports
+      else
+        pyimport mod 
+      end
+    end
 
+    Log.trap_std("Python STDOUT", "Python STDERR", severity, severity_err) do
       module_eval(&block)
+    end
+  end
+
+  def self.run_log_stderr(mod = nil, imports = nil, severity = 0, &block)
+    if mod
+      if imports
+        pyfrom mod, :import => imports
+      else
+        pyimport mod 
+      end
+    end
 
-    rescue
-      Log.exception $!
+    Log.trap_stderr("Python STDERR", severity) do
+      module_eval(&block)
     end
   end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -283,7 +283,7 @@ class Step
       now = Time.now
       str = Log.color :reset
       str << "#{ Log.color status_color, status}"
-      str << ": #{ message }" if message
+      str << ": #{ message }" if message and message != :result
       str << " -- #{Log.color :blue, path.to_s}" if path
       str << " #{Log.color :yellow, Process.pid}"
       str
@@ -293,6 +293,7 @@ class Step
     Log.info do 
       now = Time.now
       str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.2f" % (eend - start)}"
+      str << ": #{ res }" if message == :result
       str << " -- #{Log.color :blue, path.to_s}" if path
       str << " #{Log.color :yellow, Process.pid}"
       str

data/test/rbbt/util/test_log.rb

@@ -2,7 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
 require 'rbbt/util/log'
 
 class TestLog < Test::Unit::TestCase
-  def test_get_level
+  def _test_get_level
     assert_equal 0, Log.get_level(:debug)
     assert_equal 1, Log.get_level(:low)
     assert_equal 1, Log.get_level("LOW")
@@ -10,14 +10,45 @@ class TestLog < Test::Unit::TestCase
     assert_equal 0, Log.get_level(nil)
   end
 
-  def test_color
+  def _test_color
     assert Log.color(:green, "green")
   end
 
-  def test_no_stderr
+  def _test_no_stderr
     Log.ignore_stderr do
       STDERR.puts "NOPRINT"
     end
   end
+
+  def _test_trap_stderr
+    Log.severity = 0
+    Log.trap_stderr do
+      STDERR.puts "NOPRINT"
+      STDERR.puts "NOPRINT"
+      STDERR.puts "NOPRINT"
+      STDOUT.puts "PRINT STDOUT"
+      STDOUT.puts "PRINT STDOUT"
+    end
+    Log.trap_stderr do
+      STDERR.puts "NOPRINT"
+      STDOUT.puts "PRINT STDOUT"
+    end
+    Log.trap_std do
+      STDERR.puts "NOPRINT"
+      STDOUT.puts "NOPRINT STDOUT"
+    end
+  end
+
+  def test_trap_std
+    Log.severity = 0
+    Log.trap_std do
+      STDERR.puts "NOPRINT STDERR"
+      STDOUT.puts "NOPRINT STDOUT"
+    end
+    Log.trap_std "OUT", "ERR", 4, 2 do
+      STDERR.puts "NOPRINT STDERR"
+      STDOUT.puts "NOPRINT STDOUT"
+    end
+  end
 end
 

data/test/rbbt/util/test_python.rb

@@ -40,5 +40,29 @@ def python_test(a, b):
       assert_equal 3, res
     end
   end
-end
+
+  def test_run_log
+    Log.severity = 0
+    TmpFile.with_file do |tmpdir|
+      code =<<-EOF
+import sys
+def python_print():
+	print("Test STDERR", file=sys.stderr)
+	print("Test STDOUT")
+      EOF
+      Open.write(File.join(tmpdir, 'file_print.py'), code)
+      RbbtPython.add_path tmpdir
+
+      RbbtPython.run_log 'file_print' do 
+        file_print.python_print
+      end
+      RbbtPython.run_log_stderr 'file_print' do 
+        file_print.python_print
+      end
+
+      RbbtPython.run_log 'file_print' do 
+        file_print.python_print
+      end
+    end
+  end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.85
+  version: 5.26.86
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-08 00:00:00.000000000 Z
+date: 2019-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake