rbbt-util 5.26.93 → 5.26.97

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45fa23a77186b3e5f47cf3ccf754a4b2906329639ca1d188e2b9f347850abd27
-  data.tar.gz: 49b4d7b48339260536aebbd8c9f35e59155bf253a7a63685683db54967ca5359
+  metadata.gz: 19e2d3760c86a3a14044b46b95d43380190332270ec3047e620c1235df0ebdf8
+  data.tar.gz: 929a4ff0a6fa72825b0e916e07ebd38f55d09dfd9bf4486cc069dd502d683033
 SHA512:
-  metadata.gz: 37d1a5ac5167e6f4612413c86b03bb96babf71164c9c7433d14d841d3fee489ecc4abc6a613ea49f2580876cd78bd32e8a40e7437953efcbcf0e5464d1f11589
-  data.tar.gz: db18ad88a2c5eae7b478eeebde69ea498ffd8f123b8e1113e4e7dc91a21ed7689147436ac690d5b5f4777bf7cd2efa1227250fa3a477ef76ebca88c8add80a8c
+  metadata.gz: 8afac300f9e35d72708e096c6ef878cf1573dac955ce5e85c1d9dd422c01fecbd78f9ea86ce19e65315b564cf1e510ddb5bb173be5f82d8003625d00b08ef949
+  data.tar.gz: a1021842c668f85c4a759263e40720c603875bbce8de2db8b26160e35de9056a3ab768101e047b4e2fc58222b1470593e84ac23a6e00ab713637183b10210264

data/lib/rbbt/hpc.rb

@@ -33,15 +33,13 @@ module Marenostrum
       config_keys      = options.delete :config_keys
 
       if contain_and_sync
-        contain = "/scratch/tmp/rbbt-${USER}" if contain.nil?
+        contain = "/scratch/tmp/rbbt-#{ENV["USER"]}" if contain.nil?
         sync = "~/.rbbt/var/jobs" if sync.nil?
+        wipe_container = "post" if wipe_container.nil?
       end
 
       contain = File.expand_path(contain) if contain
 
-      singularity = true if contain 
-
-
       name = options[:name] ||= Misc.obj2digest({:options => options.collect{|k,v| [k,v]}.sort_by{|k,v| k.to_s }, :args => args})
       options.delete(:name)
       slurm_basedir = options[:slurm_basedir] ||= File.expand_path(File.join('~/rbbt-slurm', name)) if slurm_basedir.nil?
@@ -120,7 +118,17 @@ module load java
 let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_ON_NODE"
       EOF
 
+
+      # RUN
+      run = ""
+      exec_cmd = %(env _JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m")
+
+
       if singularity
+        #{{{ SINGULARITY
+        
+        singularity_exec = %(singularity exec -e -B $SINGULARITY_OPT_DIR:/singularity_opt/ -B /apps/)
+        
         env +=<<-EOF
 module load intel/2018.1
 module load singularity
@@ -130,15 +138,14 @@ SINGULARITY_OPT_DIR="$PROJECTS_ROOT/singularity_opt/"
 SINGULARITY_RUBY_INLINE="$HOME/.singularity_ruby_inline"
 mkdir -p "$SINGULARITY_RUBY_INLINE"
         EOF
-      end
 
-      if contain
-        user = ENV['USER'] || `whoami`.strip
-        group = File.basename(File.dirname(ENV['HOME']))
-        scratch_group_dir = File.join('/gpfs/scratch/', group)
-        projects_group_dir = File.join('/gpfs/projects/', group)
+        if contain
+          user = ENV['USER'] || `whoami`.strip
+          group = File.basename(File.dirname(ENV['HOME']))
+          scratch_group_dir = File.join('/gpfs/scratch/', group)
+          projects_group_dir = File.join('/gpfs/projects/', group)
 
-        env +=<<-EOF
+          env +=<<-EOF
 
 # Prepare container dir
 CONTAINER_DIR="#{contain}"
@@ -168,43 +175,36 @@ echo "$CONTAINER_DIR/projects/rbbt/workflows/" > $CONTAINER_DIR/.rbbt/etc/workfl
 [[ -a "$CONTAINER_DIR/scratch" ]] || ln -s '#{scratch_group_dir}' "$CONTAINER_DIR/scratch"
         EOF
         
-        if inputs_dir
-          env +=<<-EOF
+          if inputs_dir
+            env +=<<-EOF
 
 # Copy inputs
 [[ -d '#{inputs_dir}' ]] && cp -R '#{inputs_dir}' $CONTAINER_DIR/inputs
-          EOF
-          rbbt_cmd = rbbt_cmd.sub(inputs_dir, "#{contain}/inputs")
-        end
+            EOF
+            rbbt_cmd = rbbt_cmd.sub(inputs_dir, "#{contain}/inputs")
+          end
 
-        if copy_image
-          env +=<<EOF
+          if copy_image
+            env +=<<EOF
 
 # Copy image
 rsync -avz "$SINGULARITY_IMG" "$CONTAINER_DIR/rbbt.singularity.img" 1>&2
 SINGULARITY_IMG="$CONTAINER_DIR/rbbt.singularity.img"
 EOF
-        end
+          end
 
-        if  wipe_container == "pre" || wipe_container == "both"
-          env +=<<-EOF
+          if  wipe_container == "pre" || wipe_container == "both"
+            if singularity
+              env +=<<-EOF
 
 # Clean container pre
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv .rbbt/var/jobs &>> #{fsync}
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv tmp/ &>> #{fsync}
 EOF
+            end
+          end
         end
-      end
-
-      # RUN
-      run = ""
-
-
-      exec_cmd = %(env _JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m")
-
-      if singularity
-        singularity_exec = %(singularity exec -e -B $SINGULARITY_OPT_DIR:/singularity_opt/ -B /apps/)
 
         if contain
           singularity_exec << %( -C -H "$CONTAINER_DIR" \
@@ -222,7 +222,7 @@ EOF
         end
 
         if development
-          exec_cmd += ' rbbt --dev=git'
+          exec_cmd += " rbbt --dev='#{development}'"
         else
           exec_cmd += ' rbbt'
         end
@@ -230,10 +230,14 @@ EOF
         exec_cmd = singularity_exec + " " + exec_cmd
       else
         if development
-          exec_cmd << " " << %(~/git/rbbt-util/bin/rbbt --dev=~/git/)
+          exec_cmd << " " << %(~/git/rbbt-util/bin/rbbt --dev=#{development})
         else
           exec_cmd << " " << 'rbbt'
         end
+
+        if contain
+          rbbt_cmd << " " << %(--workdir_all='#{contain}')
+        end
       end
 
 
@@ -260,6 +264,10 @@ EOF
         if singularity
           coda +=<<-EOF
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean all -q &>> #{fsync}
+EOF
+        else
+          coda +=<<-EOF
+rbbt system clean all -q &>> #{fsync}
 EOF
         end
 
@@ -280,7 +288,13 @@ find '#{target}' -type l -ls | awk '$13 ~ /^#{target.gsub('/','\/')}/ { sub("#{s
 EOF
 
         if  contain && (wipe_container == "post" || wipe_container == "both")
-          coda +=<<-EOF
+          run =<<-EOF + run
+if $(ls -A '#{contain}'); then
+    echo "ERROR: Container directory not empty, refusing to wipe. #{contain}"
+fi
+          EOF
+          if singularity
+            coda +=<<-EOF
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -v /dev/shm/sem.*.{in,out,process} /dev/shm/sem.Session-PID.*.sem 2> /dev/null >> #{fsync}
 if [ $sync_es == '0' ]; then 
@@ -291,6 +305,18 @@ else
 fi
 unset sync_es
 EOF
+          else
+            coda +=<<-EOF
+#{exec_cmd} system clean
+if [ $sync_es == '0' ]; then 
+    rm -Rfv #{contain} &>> #{fsync}
+else
+    echo "WARNING: Results could not sync correctly. Contain directory not purged"
+fi
+unset sync_es
+EOF
+
+          end
         end
       end
       coda +=<<-EOF

data/lib/rbbt/util/cmd.rb

@@ -6,8 +6,8 @@ require 'rbbt/util/misc/indiferent_hash'
 module CMD
 
   TOOLS = IndiferentHash.setup({})
-  def self.tool(tool, claim = nil, test = nil, &block)
-    TOOLS[tool] = [claim, test, block]
+  def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
+    TOOLS[tool] = [claim, test, block, cmd]
   end
 
   def self.get_tool(tool)
@@ -15,10 +15,10 @@ module CMD
 
     @@init_cmd_tool ||= IndiferentHash.setup({})
     if !@@init_cmd_tool[tool]
-      claim, test, block = TOOLS[tool]
+      claim, test, block, cmd = TOOLS[tool]
       begin
         if test
-          CMD.cmd(test)
+          CMD.cmd(test + " ")
         else
           CMD.cmd("#{tool} --help")
         end
@@ -30,6 +30,8 @@ module CMD
         end
       end
       @@init_cmd_tool[tool] = true
+
+      return cmd if cmd
     end
 
     tool.to_s

data/lib/rbbt/workflow/definition.rb

@@ -73,7 +73,7 @@ module Workflow
       set_info :result_type, dep.info[:result_type]
       forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
       if forget
-        self.archive
+        self.archive_deps
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist? dep.files_dir

data/lib/rbbt/workflow/step.rb

@@ -108,7 +108,7 @@ class Step
     @inputs || []
   end
 
-  def archive
+  def archive_deps
     self.set_info :archived_info, archived_info
     self.set_info :archived_dependencies, info[:dependencies]
   end

data/share/rbbt_commands/workflow/task

@@ -301,7 +301,7 @@ saved_job_options = job_options
 workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir))) if options[:workdir]
 
 if override_deps
-  override_deps.split($array_separator).each do |part|
+  override_deps.split($array_separator || ",").each do |part|
     t_, value = part.split("=")
     job_options.merge!( t_ => value)
   end

data/share/rbbt_commands/workflow/trace

@@ -218,6 +218,6 @@ if plot
       labs(x=NULL, y=NULL) + 
       theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
 
-    rbbt.png_plot('#{plot}', 'timeline', width=#{width}, height=#{height}, pointsize=6)
+    rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
   EOF
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.93
+  version: 5.26.97
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-16 00:00:00.000000000 Z
+date: 2019-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake