rbbt-util 5.34.12 → 5.34.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d36e35e1495c3c319e8aa0485851b870e985bc34ea282f21c8e744adc302f50
-  data.tar.gz: e4df8738231ce9e1f8ebf70090595413fef4555ad9bedde4b52b36ebc219e39e
+  metadata.gz: b2e50428f936aab209874c5f26e7504aa14da9a30bc7e2b13575432527a5b6a4
+  data.tar.gz: e522b76bb910b477e303e2cfc323c8708fa4d3bbecc0a7c46de473fe81f1ef6a
 SHA512:
-  metadata.gz: 3ebc6c5ea53567b06a58854ab0b58c83777d939bb67a80cc28dbe7ebaa992269f59c20c2dba28d799402bfded6cbd9771ad1036e31ebbb365e3995500cd37f44
-  data.tar.gz: 2d3870b76a07c096ee1df74593b6bd641c0f1f8aab68c09827bae57cf64ee97e237b22ee6d16edb81ddcc83b7d8b6f4cb6298da8d203c63f8cdebc408b44fb84
+  metadata.gz: e73a4ceda5eca11497ea4f0fe91fda7275375aaf8aeb6f9f70a45a46d4c71022527d7cd32c0cb30700a6bc2724bd1e2d2e0bd808a553ee3c8d6e729baa1dc2c0
+  data.tar.gz: a050c5defff3e3d36fe5682c117802d54cfeb930e3e61ff96da29e326719bf97af6fe85b5c30812dd91c5c6a447201164c15a3e1714aa1226690a0c49782ba9a

data/lib/rbbt/hpc/batch.rb

@@ -51,9 +51,15 @@ module HPC
 
         group, user, user_group, scratch_group_dir, projects_group_dir = options.values_at :group, :user, :user_group, :scratch_group_dir, :projects_group_dir
 
-        singularity_img, singularity_opt_dir, singularity_ruby_inline = options.values_at :singularity_img, :singularity_opt_dir, :singularity_ruby_inline
+        singularity_img, singularity_opt_dir, singularity_ruby_inline, singularity_mounts = options.values_at :singularity_img, :singularity_opt_dir, :singularity_ruby_inline, :singularity_mounts
+
+        singularity_cmd = %(singularity exec -e -B "#{File.expand_path singularity_opt_dir}":/singularity_opt/ -B "#{File.expand_path singularity_ruby_inline}":"/.singularity_ruby_inline":rw ) 
 
-        singularity_cmd = %(singularity exec -e -B #{singularity_opt_dir}:/singularity_opt/ -B "#{singularity_ruby_inline}":"/.singularity_ruby_inline":rw ) 
+        if singularity_mounts
+          singularity_mounts.split(",").each do |mount|
+            singularity_cmd += "-B #{ mount } "
+          end
+        end
 
         if contain && options[:hardened]
           singularity_cmd << %( -C -H "#{contain}" \
@@ -150,6 +156,7 @@ EOF
         :mem_per_cpu,
         :gres,
         :lua_modules,
+        :conda,
         :contraints,
         :licenses,
         :batch_dir,
@@ -167,6 +174,7 @@ EOF
         :purge_deps,
         :singularity,
         :singularity_img,
+        :singularity_mounts,
         :singularity_opt_dir,
         :singularity_ruby_inline
       ]
@@ -188,6 +196,7 @@ EOF
         :env_cmd,
         :user_group,
         :singularity_img,
+        :singularity_mounts,
         :singularity_opt_dir,
         :singularity_ruby_inline,
         :singularity
@@ -284,6 +293,20 @@ EOF
       str
     end
 
+    def load_conda(env = nil)
+      return "" if env.nil? || env.empty?
+
+      <<-EOF
+if ! type conda | grep function &> /dev/null; then
+    if [ ! -z $CONDA_EXE ]; then
+            source "$(dirname $(dirname $CONDA_EXE))/etc/profile.d/conda.sh" &> /dev/null
+    fi
+fi
+conda activate #{ env }
+      EOF
+    end
+
+
     def batch_system_variables
       <<-EOF
 let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
@@ -292,6 +315,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
 
     def prepare_environment(options = {})
       modules = options[:lua_modules]
+      conda = options[:conda]
 
       prepare_environment = ""
 
@@ -382,7 +406,7 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
         end
       end
 
-      batch_system_variables + load_modules(modules) + "\n" + functions + "\n" + prepare_environment
+      batch_system_variables + load_modules(modules) + "\n" + load_conda(conda) + "\n"  + functions + "\n" + prepare_environment
     end
 
     def execute(options)

data/lib/rbbt/resource.rb

@@ -204,6 +204,10 @@ module Resource
             case type
             when :string
               Misc.sensiblewrite(final_path, content)
+            when :csv
+              require 'rbbt/tsv/csv'
+              tsv = TSV.csv Open.open(content)
+              Misc.sensiblewrite(final_path, tsv.to_s)
             when :url
               options = {}
               options[:noz] = true if Open.gzip?(final_path) || Open.bgzip?(final_path) || Open.zip?(final_path)

data/lib/rbbt/tsv/csv.rb

@@ -25,7 +25,9 @@ module TSV
           when Path
             CSV.read obj.find.open, options
           when String
-            if Misc.is_filename?(obj)
+            if Open.remote?(obj)
+              CSV.read Open.open(obj), options
+            elsif Misc.is_filename?(obj)
               CSV.read obj, options
             else
               CSV.new obj, **options

data/lib/rbbt/tsv/excel.rb

@@ -266,7 +266,7 @@ module TSV
 
   def self.xls(filename, options ={})
     if Open.remote? filename
-      TmpFile.with_file do |tmp|
+      TmpFile.with_file nil, :extension => 'xls' do |tmp|
         Open.download(filename, tmp)
         TSV::XLS.read(tmp, options)
       end
@@ -278,7 +278,7 @@ module TSV
   def self.xlsx(filename, options ={})
     if Open.remote? filename
 
-      TmpFile.with_file do |tmp|
+      TmpFile.with_file nil, :extension => 'xlsx' do |tmp|
         Open.download(filename, tmp)
         TSV::XLSX.read(tmp, options)
       end

data/lib/rbbt/util/misc/development.rb

@@ -193,6 +193,16 @@ def self.add_libdir(dir=nil)
     insist(3, &block)
   end
 
+  def self.chunk(array, num)
+    total = array.length
+    current = 0
+    while current < total
+      last = current + num - 1
+      yield array[current..last]
+      current = last + 1
+    end
+  end
+
   # Divides the array into +num+ chunks of the same size by placing one
   # element in each chunk iteratively.
   def self.divide(array, num)

data/lib/rbbt/util/misc.rb

@@ -71,6 +71,9 @@ module Misc
   end
 
   def self.timespan(str, default = "s")
+
+    return - timespan(str[1..-1], default) if str[0] == "-"
+    
     if str.include?(":")
       seconds, minutes, hours = str.split(":").reverse
       return seconds.to_i + minutes.to_i * 60 + hours.to_i * 60 * 60
@@ -86,7 +89,7 @@ module Misc
       "h" => (60 * 60),
       "d" => (60 * 60 * 24),
       "w" => (60 * 60 * 24 * 7),
-      "mo" => (60 * 60 * 24 * 30),
+      "mo" => (60 * 60 * 24 * 31),
       "y" => (60 * 60 * 24 * 365),
     }
 

data/lib/rbbt/util/open.rb

@@ -640,7 +640,7 @@ module Open
     io
   end
 
-  def self.download(url, file)
+  def self.download_old(url, file)
     Open.open(url, :mode => 'rb', :noz => true) do |sin|
       Open.open(file, :mode => 'wb') do |sout|
         Misc.consume_stream(sin, false, sout)
@@ -648,6 +648,10 @@ module Open
     end
   end
 
+  def self.download(url, path)
+    Open.wget(url, "--output-document" => path, :pipe => false)
+  end
+
   def self.can_open?(file)
     String === file and (Open.exist?(file) or remote?(file))
   end
@@ -841,7 +845,4 @@ module Open
     File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
-  def self.download(url, path)
-    Open.wget(url, "--output-document" => path)
-  end
 end

data/lib/rbbt/util/python/util.rb

@@ -19,8 +19,9 @@ module RbbtPython
 
   def self.df2tsv(tuple, options = {})
     options = Misc.add_defaults options, :type => :list
+    IndiferentHash.setup options
     tsv = TSV.setup({}, options)
-    tsv.key_field = tuple.columns.name
+    tsv.key_field = options[:key_field] || tuple.columns.name
     tsv.fields = py2ruby_a(tuple.columns.values)
     keys = tuple.index.values
     PyCall.len(tuple.index).times do |i|

data/lib/rbbt/workflow/step/accessor.rb

@@ -203,7 +203,7 @@ class Step
     files
   end
 
-  def file(name)
+  def file(name=nil)
     Path.setup(File.join(files_dir, name.to_s), workflow, self)
   end
 

data/share/Rlib/plot.R

@@ -39,3 +39,9 @@ geom_entity <- function (real.geom = NULL, mapping = NULL, data = NULL, stat = "
 }
 
 rbbt.ggplot2.rotate_x_labels <- function(){ theme(axis.text.x = element_text(angle = 45, vjust = 1, hjust=1)) }
+
+rbbt.ggplot2.theme <- function(plot){
+
+    plot + theme_classic() + scale_y_continuous(labels=scales::comma) + scale_x_continuous(labels=scales::comma) + rbbt.ggplot2.rotate_x_labels()
+
+}

data/share/Rlib/util.R

@@ -812,7 +812,7 @@ rbbt.plot.text_scatter <- function(formula, data) {
 rbbt.install.CRAN <- function(pkg){
     cat("Try CRAN install:", pkg, "\n")
     res = FALSE
-    tryCatch({ install.packages(pkg); library(pkg); res = TRUE; }, error = function(e){ str(e); warning(paste("Could not install CRAN ", pkg)); res = FALSE })
+    tryCatch({ install.packages(pkg); library(pkg, character.only=T); res = TRUE; }, error = function(e){ str(e); warning(paste("Could not install CRAN ", pkg)); res = FALSE })
     return(res)
 }
 

data/share/rbbt_commands/hpc/clean

@@ -138,7 +138,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
- aborted = error = true if aborted.nil? && error.nil?
+ aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
  #  select = true if done && exit_status && exit_status.to_i == 0 

data/share/rbbt_commands/hpc/orchestrate

@@ -13,6 +13,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -36,8 +37,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 -OR--orchestration_rules* Orchestration rules
 EOF
 

data/share/rbbt_commands/hpc/task

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -35,8 +36,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/lsf/clean

@@ -138,7 +138,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
- aborted = error = true if aborted.nil? && error.nil?
+ aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
  #  select = true if done && exit_status && exit_status.to_i == 0 

data/share/rbbt_commands/lsf/orchestrate

@@ -13,6 +13,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -36,8 +37,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 -OR--orchestration_rules* Orchestration rules
 EOF
 

data/share/rbbt_commands/lsf/task

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -35,8 +36,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/slurm/clean

@@ -138,7 +138,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
- aborted = error = true if aborted.nil? && error.nil?
+ aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
  #  select = true if done && exit_status && exit_status.to_i == 0 

data/share/rbbt_commands/slurm/orchestrate

@@ -13,6 +13,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -36,8 +37,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 -OR--orchestration_rules* Orchestration rules
 EOF
 

data/share/rbbt_commands/slurm/task

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -35,8 +36,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/test/rbbt/util/test_misc.rb

@@ -610,6 +610,8 @@ EOF
     assert_equal 60*60*24, Misc.timespan('1d')
     assert_equal 60*60*24, Misc.timespan('1d')
     assert_equal 60*60*24, Misc.timespan('24:00:00')
+
+    assert_equal Misc.timespan('1min'), - Misc.timespan('-1min')
   end
 
   def test_remove_long_items

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.12
+  version: 5.34.14
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-23 00:00:00.000000000 Z
+date: 2022-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake