rbbt-util 5.31.9 → 5.31.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9627d833faf734e912d68d60714081cd8179b8f65de153842c2a77bc356e58ab
-  data.tar.gz: 4d75542d741d61732ddead0ab6295e97461e5cd4aa7b8baeb4436f78f875465d
+  metadata.gz: 99337ec5d1fb8c3dbc9d1b8f89bc2965dc659c3e6043defe718b0007ba00456b
+  data.tar.gz: c61228a72814e48ff2c7bf90a0e44ace075f6f4c1e29cfb7f36e6a182896f7b6
 SHA512:
-  metadata.gz: 7105097e07ee1da675b2ee4cc5fa7f7be61bd49051553a28ca39a829105fda3e08e684ea804928000863d83d67bdad3d462121ab23a379a5ba76d3ce4c74ccc5
-  data.tar.gz: ba67880edc6ecad00c8a06fa802fc56a035524a321ab31abe8d4cae07ea521ce641abd948731cd7da8f0582b8d5ea9cdea8bca3e9152acb070f379b35fd0a40d
+  metadata.gz: eeb8b9d43a4199cbea11693f7028ba6216507bb3c1cc149e30d30070652ddfd15f7660db2df1435a2ed108354d6ce261bfc23907d5cc6866c2561f2eac9285eb
+  data.tar.gz: 405081e84408572eb3a78cda085dba453c44b5583bc93e35287f9feb8d7693912710a1946b30e7ab931517da592f5a469cc66c4c923cf126070e7c01edbff134

data/lib/rbbt/hpc/batch.rb

@@ -121,6 +121,9 @@ EOF
         :nodes,
         :queue,
         :singularity,
+        :singularity_img,
+        :singularity_opt_dir,
+        :singularity_ruby_inline,
         :sync,
         :task_cpus,
         :time,
@@ -163,6 +166,8 @@ EOF
       batch_options[:scratch_group_dir] = File.join('/gpfs/scratch/', group)
       batch_options[:projects_group_dir] = File.join('/gpfs/projects/', group)
 
+      batch_options[:singularity] = true if batch_options[:singularity_img]
+
       if batch_options[:contain_and_sync]
         if batch_options[:contain].nil?
           contain_base = Rbbt::Config.get(:contain_base_dir, :batch_contain, :batch, :default => "/scratch/tmp/rbbt-[USER]")

data/lib/rbbt/hpc/slurm.rb

@@ -23,6 +23,7 @@ BATCH_SYSTEM=SLURM
       nodes          = Misc.process_options options, :nodes
       workdir        = Misc.process_options options, :workdir
       exclusive      = Misc.process_options options, :exclusive
+      highmem        = Misc.process_options options, :highmem
 
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
@@ -45,6 +46,7 @@ BATCH_SYSTEM=SLURM
       EOF
 
       header << "#SBATCH --exclusive" << "\n" if exclusive
+      header << "#SBATCH --constraint=highmem" << "\n" if highmem
 
       header
     end

data/lib/rbbt/tsv/attach.rb

@@ -189,6 +189,7 @@ module TSV
     other.each do |k,v|
       self.zip_new k, v
     end
+    self
   end
 
   

data/lib/rbbt/util/cmd.rb

@@ -10,6 +10,14 @@ module CMD
     TOOLS[tool] = [claim, test, block, cmd]
   end
 
+  def self.conda(tool, env = nil, channel = 'bioconda')
+    if env
+      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")      
+    else
+      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")      
+    end
+  end
+
   def self.get_tool(tool)
     return tool.to_s unless TOOLS[tool]
 
@@ -30,16 +38,17 @@ module CMD
         end
       end
       version_txt = ""
-      %w(--version -version --help).each do |f|
+      version = nil
+      ["--version", "-version", "--help", ""].each do |f|
         begin
-          version_txt += CMD.cmd("#{tool} #{f}").read
-          break
+          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version = Misc.scan_version_text(version_txt, tool)
+          break if version
         rescue
+          Log.exception $!
         end
       end
 
-      version = Misc.scan_version_text(version_txt, tool)
-
       @@init_cmd_tool[tool] = version || true
 
       return cmd if cmd
@@ -67,7 +76,8 @@ module CMD
     string = ""
     options.each do |option, value|
       raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=]+$/i
-      raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
+      #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
+      value = value.gsub("'","\\'") if value.to_s.include? "'"
 
       option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
 

data/lib/rbbt/util/misc/inspect.rb

@@ -431,8 +431,20 @@ module Misc
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text.split("\n").each do |line|
+      next unless line =~ /\W#{cmd}\W/i
+      m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
+      return m[1] if m
+      next unless line =~ /\Wversion\W/i
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+    end
     m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-    return nil if m.nil?
-    m[1]
+    return m[1] if m
+    m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
+    return m[1] if m
+    nil
   end
 end

data/lib/rbbt/util/misc/options.rb

@@ -46,8 +46,8 @@ module Misc
   def self.hash2string(hash)
     hash.sort_by{|k,v| k.to_s}.collect{|k,v| 
       next unless %w(Symbol String Float Fixnum Integer Numeric TrueClass FalseClass Module Class Object).include? v.class.to_s
-      [ Symbol === k ? ":" << k.to_s : k,
-        Symbol === v ? ":" << v.to_s : v] * "="
+      [ Symbol === k ? ":" << k.to_s : k.to_s.chomp,
+        Symbol === v ? ":" << v.to_s : v.to_s.chomp] * "="
     }.compact * "#"
   end
 

data/lib/rbbt/util/misc/pipes.rb

@@ -35,10 +35,11 @@ module Misc
     res
   end
 
-  def self.with_fifo(path = nil, &block)
+  def self.with_fifo(path = nil, clean = true, &block)
     begin
       erase = path.nil?
       path = TmpFile.tmp_file if path.nil?
+      File.rm path if clean && File.exists?(path)
       File.mkfifo path
       yield path
     ensure

data/lib/rbbt/util/open.rb

@@ -11,7 +11,7 @@ module Open
   class OpenGzipError < StandardError; end
 
   REMOTE_CACHEDIR = File.join(ENV["HOME"], "/tmp/open_cache")
-  FileUtils.mkdir_p REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
+  #FileUtils.mkdir_p REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
 
   GREP_CMD = begin
                if ENV["GREP_CMD"] 
@@ -346,15 +346,18 @@ module Open
     target = File.join(target, File.basename(source)) if File.directory? target
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
     FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
     FileUtils.ln_s source, target
   end
 
   def self.ln(source, target, options = {})
     source = source.find if Path === source
     target = target.find if Path === target
+    source = File.realpath(source) if File.symlink?(source)
 
     FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
     FileUtils.rm target if File.exist?(target)
+    FileUtils.rm target if File.symlink?(target)
     FileUtils.ln source, target
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -76,8 +76,10 @@ module Workflow
     extension workflow.tasks[oname].extension if workflow.tasks.include?(oname) unless @extension
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
-      dep = dependencies.last.join
+      dep = dependencies.last
+      dep.join
       raise dep.get_exception if dep.error?
+      raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
       set_info :result_type, dep.info[:result_type]
       forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
@@ -103,7 +105,11 @@ module Workflow
           Open.rm_rf self.files_dir 
           Open.link dep.files_dir, self.files_dir
         end
-        Open.link dep.path, self.path
+        if defined?(RemoteStep) && RemoteStep === dep
+          Open.write(self.tmp_path, Open.read(dep.path))
+        else
+          Open.link dep.path, self.path
+        end
       end
       nil
     end

data/lib/rbbt/workflow/examples.rb

@@ -54,7 +54,8 @@ module Workflow
             inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             if File.symlink?(file)
-              inputs[input.to_sym]  = File.readlink(file)
+              link_target = File.expand_path(File.readlink(file), File.dirname(file))
+              inputs[input.to_sym]  = link_target
             else
               inputs[input.to_sym]  = Open.realpath(file)
             end

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -20,7 +20,7 @@ class RemoteWorkflow
 
     def self.encode(url)
       begin
-        URI.encode(url)
+        URI::DEFAULT_PARSER.escape(url)
       rescue
         Log.warn $!.message
         url

data/lib/rbbt/workflow/step/accessor.rb

@@ -368,7 +368,7 @@ class Step
   end
 
   def self.log_progress(status, options = {}, path = nil, &block)
-    options = Misc.add_defaults options, :severity => Log::INFO, :file => path
+    options = Misc.add_defaults options, :severity => Log::INFO, :file => (@exec ? nil : path)
     max = Misc.process_options options, :max
     Log::ProgressBar.with_bar(max, options) do |bar|
       begin
@@ -394,7 +394,7 @@ class Step
     options = {} if options.nil?
 
     max = options[:max]
-    Log::ProgressBar.new_bar(max, {:desc => msg, :file => file(:progress)}.merge(options))
+    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
   end
 
   def self.log(status, message, path, &block)

data/lib/rbbt/workflow/usage.rb

@@ -215,7 +215,7 @@ module Workflow
 
       dep_tree = {[self, task_name] => dep_tree(task_name)}
       prov_tree = prov_tree(dep_tree)
-      if prov_tree && ! prov_tree.empty?
+      if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
 
         puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
         puts

data/share/rbbt_commands/hpc/orchestrate

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs

data/share/rbbt_commands/hpc/task

@@ -11,6 +11,7 @@ $slurm_options = SOPT.get <<EOF
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs

data/share/rbbt_commands/log

@@ -31,7 +31,9 @@ if ARGV.empty?
     puts Rbbt.etc.log_severity.find + ' does not exist'                                                                                                                                  
   end                                                                                                                                                                                      
 else
-  Open.write(Rbbt.etc.log_severity, ARGV[0].upcase)
+  level = ARGV[0]
+  level = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE)[level.to_i] if level =~ /^\d+$/
+  Open.write(Rbbt.etc.log_severity, level.upcase)
 end
 
 

data/share/rbbt_commands/workflow/trace

@@ -23,7 +23,7 @@ $ rbbt workflow trace <job-result>
 -h--help Help
 -w--width* Image Width
 -h--height* Image Height
--p--plot Plot file
+-p--plot* Plot file
 -s--size* Image Size (Height and Width)
 -fg--fix_gap Remove execution gaps
 -pd--plot_data Print plot data

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.31.9
+  version: 5.31.14
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-26 00:00:00.000000000 Z
+date: 2021-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake