rbbt-util 5.34.12 → 5.34.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d36e35e1495c3c319e8aa0485851b870e985bc34ea282f21c8e744adc302f50
-  data.tar.gz: e4df8738231ce9e1f8ebf70090595413fef4555ad9bedde4b52b36ebc219e39e
+  metadata.gz: 9481e1e7a7e7341b95824a48c314d2316a7283e356ea916b424edb2f15886616
+  data.tar.gz: 5012fb19a8cb88d1e9764a2b6ab049e203c38bc2daf758e112c8e256dca68565
 SHA512:
-  metadata.gz: 3ebc6c5ea53567b06a58854ab0b58c83777d939bb67a80cc28dbe7ebaa992269f59c20c2dba28d799402bfded6cbd9771ad1036e31ebbb365e3995500cd37f44
-  data.tar.gz: 2d3870b76a07c096ee1df74593b6bd641c0f1f8aab68c09827bae57cf64ee97e237b22ee6d16edb81ddcc83b7d8b6f4cb6298da8d203c63f8cdebc408b44fb84
+  metadata.gz: e72b2d6f831128ace58d54e62fdad1f180acdd74d9c73304cc315760dd66c7705b733d024b62794c165a2ea00cdbf6b1d67569f1b40a8e19822424a1638ecd1f
+  data.tar.gz: f8ffee23f77ac7024a06b50d7b48ba6db900080fa06e78d44e61dd4274b4843d6fd203644a5c9be0b730e1728a3b0550e32fc503401744737dcc2bed7401ce8d

data/lib/rbbt/hpc/batch.rb

@@ -51,9 +51,15 @@ module HPC
 
         group, user, user_group, scratch_group_dir, projects_group_dir = options.values_at :group, :user, :user_group, :scratch_group_dir, :projects_group_dir
 
-        singularity_img, singularity_opt_dir, singularity_ruby_inline = options.values_at :singularity_img, :singularity_opt_dir, :singularity_ruby_inline
+        singularity_img, singularity_opt_dir, singularity_ruby_inline, singularity_mounts = options.values_at :singularity_img, :singularity_opt_dir, :singularity_ruby_inline, :singularity_mounts
+
+        singularity_cmd = %(singularity exec -e -B "#{File.expand_path singularity_opt_dir}":/singularity_opt/ -B "#{File.expand_path singularity_ruby_inline}":"/.singularity_ruby_inline":rw ) 
 
-        singularity_cmd = %(singularity exec -e -B #{singularity_opt_dir}:/singularity_opt/ -B "#{singularity_ruby_inline}":"/.singularity_ruby_inline":rw ) 
+        if singularity_mounts
+          singularity_mounts.split(",").each do |mount|
+            singularity_cmd += "-B #{ mount } "
+          end
+        end
 
         if contain && options[:hardened]
           singularity_cmd << %( -C -H "#{contain}" \
@@ -150,6 +156,7 @@ EOF
         :mem_per_cpu,
         :gres,
         :lua_modules,
+        :conda,
         :contraints,
         :licenses,
         :batch_dir,
@@ -167,6 +174,7 @@ EOF
         :purge_deps,
         :singularity,
         :singularity_img,
+        :singularity_mounts,
         :singularity_opt_dir,
         :singularity_ruby_inline
       ]
@@ -188,6 +196,7 @@ EOF
         :env_cmd,
         :user_group,
         :singularity_img,
+        :singularity_mounts,
         :singularity_opt_dir,
         :singularity_ruby_inline,
         :singularity
@@ -284,6 +293,20 @@ EOF
       str
     end
 
+    def load_conda(env = nil)
+      return "" if env.nil? || env.empty?
+
+      <<-EOF
+if ! type conda | grep function &> /dev/null; then
+    if [ ! -z $CONDA_EXE ]; then
+            source "$(dirname $(dirname $CONDA_EXE))/etc/profile.d/conda.sh" &> /dev/null
+    fi
+fi
+conda activate #{ env }
+      EOF
+    end
+
+
     def batch_system_variables
       <<-EOF
 let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
@@ -292,6 +315,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
 
     def prepare_environment(options = {})
       modules = options[:lua_modules]
+      conda = options[:conda]
 
       prepare_environment = ""
 
@@ -382,7 +406,7 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
         end
       end
 
-      batch_system_variables + load_modules(modules) + "\n" + functions + "\n" + prepare_environment
+      batch_system_variables + load_modules(modules) + "\n" + load_conda(conda) + "\n"  + functions + "\n" + prepare_environment
     end
 
     def execute(options)

data/lib/rbbt/util/python/util.rb

@@ -19,8 +19,9 @@ module RbbtPython
 
   def self.df2tsv(tuple, options = {})
     options = Misc.add_defaults options, :type => :list
+    IndiferentHash.setup options
     tsv = TSV.setup({}, options)
-    tsv.key_field = tuple.columns.name
+    tsv.key_field = options[:key_field] || tuple.columns.name
     tsv.fields = py2ruby_a(tuple.columns.values)
     keys = tuple.index.values
     PyCall.len(tuple.index).times do |i|

data/share/rbbt_commands/hpc/clean

@@ -138,7 +138,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
- aborted = error = true if aborted.nil? && error.nil?
+ aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
  #  select = true if done && exit_status && exit_status.to_i == 0 

data/share/rbbt_commands/hpc/orchestrate

@@ -13,6 +13,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -36,8 +37,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 -OR--orchestration_rules* Orchestration rules
 EOF
 

data/share/rbbt_commands/hpc/task

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -35,8 +36,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/lsf/clean

@@ -138,7 +138,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
- aborted = error = true if aborted.nil? && error.nil?
+ aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
  #  select = true if done && exit_status && exit_status.to_i == 0 

data/share/rbbt_commands/lsf/orchestrate

@@ -13,6 +13,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -36,8 +37,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 -OR--orchestration_rules* Orchestration rules
 EOF
 

data/share/rbbt_commands/lsf/task

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -35,8 +36,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 EOF
 
 batch_system = $slurm_options.delete :batch_system

data/share/rbbt_commands/slurm/clean

@@ -138,7 +138,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
- aborted = error = true if aborted.nil? && error.nil?
+ aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
  #  select = true if done && exit_status && exit_status.to_i == 0 

data/share/rbbt_commands/slurm/orchestrate

@@ -13,6 +13,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -36,8 +37,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 -OR--orchestration_rules* Orchestration rules
 EOF
 

data/share/rbbt_commands/slurm/task

@@ -12,6 +12,7 @@ $slurm_options = SOPT.get <<EOF
 --drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
 -si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
@@ -35,8 +36,9 @@ $slurm_options = SOPT.get <<EOF
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
--lmod--lua_modules* Lua Modules to load 
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
 EOF
 
 batch_system = $slurm_options.delete :batch_system

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.12
+  version: 5.34.13
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-08-23 00:00:00.000000000 Z
+date: 2022-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake