rbbt-util 5.32.10 → 5.32.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 258c55a2b59d81426d1aa3980f35104ef193774068330b25c9aad29061d70e6a
-  data.tar.gz: 8265a128c161349455f7803d5a70a259c03481bb7a441aa1ffec58626fbb0226
+  metadata.gz: fab4cf5fb60bb998eeed28a4e07758efd68f1103de0eff61a5b5c1d4a9fd2583
+  data.tar.gz: 47c9a2bdc555645ebc833c2a3b92a3452df259546c1dbf200d77dcee04b8969f
 SHA512:
-  metadata.gz: 8eb143ab5cf5e4e55851ccd11b24eeae7d3e60948e30a71de3b5fb87f5333d7e40823b268d6ac894eea5d3d73ffa47101f7bb229378837799597df54545bb8c8
-  data.tar.gz: 0ebddaae9d523d9dd767dfde1b351b011ba3dd65c4abd7c0bcc0e78cc3e604a6850250bf0e74dac9f5ca0172a56a067541bdc34b5037e2ffc7ba972558f2ce4c
+  metadata.gz: 92121d5df854b036a37413c27d2244cef0325874c018433abce1a93887c93efc650474e059c13b84c0be9a708d6fdc735daba6f861e1d05a670950fea3d32916
+  data.tar.gz: 0d24d99a64101fd33f2ac5774c30267f238c81d6479a754445fc02424384ba716e4c672b8a7c0d1c6f22b9f16c2ac1e1104ba9d35f7697bf4a9631e3a776bb02

data/bin/rbbt

@@ -102,6 +102,7 @@ end
 
 if options[:config_keys]
   options[:config_keys].split(",").each do |config|
+    config = config.strip
     Rbbt::Config.process_config config
   end
 end

data/lib/rbbt/hpc/batch.rb

@@ -95,14 +95,15 @@ module HPC
 
       task = Symbol === job.overriden ? job.overriden : job.task_name
 
-      if job.overriden
-        override_deps = job.rec_dependencies.
-          select{|dep| Symbol === dep.overriden }.
+      if job.overriden?
+        #override_deps = job.rec_dependencies.
+        #  select{|dep| Symbol === dep.overriden }.
+        
+        override_deps = job.overriden_deps.
           collect do |dep| 
-
           name = [dep.workflow.to_s, dep.task_name] * "#"
           [name, dep.path] * "="  
-        end * ","
+        end.uniq * ","
 
         options[:override_deps] = override_deps unless override_deps.empty?
       end
@@ -156,10 +157,15 @@ EOF
         :singularity_ruby_inline,
         :sync,
         :task_cpus,
+        :mem,
+        :mem_per_cpu,
+        :licenses,
+        :contraints,
         :time,
         :user_group,
         :wipe_container,
         :workdir,
+        :purge_deps
       ]
 
       keys.each do |key|
@@ -378,9 +384,9 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
     end
 
     def execute(options)
-      exec_cmd, job_cmd = options.values_at :exec_cmd, :rbbt_cmd
+      exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :rbbt_cmd, :task_cpus
 
-      <<-EOF
+      script=<<-EOF
 step_path=$( 
       #{exec_cmd} #{job_cmd} --printpath
 )
@@ -388,7 +394,10 @@ exit_status=$?
 
 [[ -z $BATCH_JOB_ID ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_job $BATCH_JOB_ID
 [[ -z $BATCH_SYSTEM ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_system $BATCH_SYSTEM
+#{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_cpus #{task_cpus}
       EOF
+
+      script
     end
 
     def sync_environment(options = {})
@@ -409,6 +418,13 @@ fi
 
     def cleanup_environment(options = {})
       cleanup_environment = ""
+
+      cleanup_environment +=<<-EOF if options[:purge_deps]
+if [ $exit_status == '0' ]; then 
+  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}' 
+fi
+      EOF
+
       if options[:sync]
         if options[:wipe_container] == 'force'
           cleanup_environment +=<<-EOF

data/lib/rbbt/hpc/slurm.rb

@@ -31,6 +31,9 @@ export BATCH_SYSTEM=SLURM
       licenses       = Misc.process_options options, :licenses
       constraint     = Misc.process_options options, :constraint
 
+      mem            = Misc.process_options options, :mem
+      mem_per_cpu    = Misc.process_options options, :mem_per_cpu
+
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
 
@@ -39,22 +42,34 @@ export BATCH_SYSTEM=SLURM
 
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
+      sbatch_params = {"job-name" => batch_name,
+                       "output" => fout,
+                       "error" => ferr,
+                       "cpus-per-task" => task_cpus,
+                       "nodes" => nodes,
+                       "time" => time,
+                       "exclusive" => exclusive,
+                       "licenses" => licenses,
+                       "mem" => mem,
+                       "mem-per-cpu" => mem_per_cpu,
+      }
+
       header =<<-EOF
 #!/bin/bash
-#SBATCH --job-name="#{batch_name}"
-#SBATCH --workdir="#{workdir}"
-#SBATCH --output="#{fout}"
-#SBATCH --error="#{ferr}"
-#SBATCH --qos="#{queue}"
-#SBATCH --cpus-per-task="#{task_cpus}"
-#SBATCH --time="#{time}"
-#SBATCH --nodes="#{nodes}"
       EOF
 
-      header << "#SBATCH --exclusive" << "\n" if exclusive
-      header << "#SBATCH --constraint=highmem" << "\n" if highmem
-      header << "#SBATCH --licenses=#{licenses}" << "\n" if licenses
-      header << "#SBATCH --constraint=#{constraint}" << "\n" if constraint
+      sbatch_params.each do |name,value|
+        next if value.nil? || value == ""
+        if TrueClass === value
+          header << "#SBATCH --#{name}" << "\n"
+        elsif Array === value
+          value.each do |v|
+            header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
+          end
+        else
+          header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
+        end
+      end
 
       header
     end

data/lib/rbbt/resource.rb

@@ -87,65 +87,67 @@ module Resource
       lock_filename = nil # it seems like this was locked already.
 
       Misc.lock lock_filename do
-        uri = URI(url)
-
-        http = Net::HTTP.new(uri.host, uri.port)
-
-        if uri.scheme == "https"
-          http.use_ssl = true
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
-        end
+        begin
+          uri = URI(url)
 
-        timeout = 60 * 10
-        http.read_timeout = timeout
-        http.open_timeout = timeout
-        request = Net::HTTP::Get.new(uri.request_uri)
-        http.request request do |response|
-          filename = if response["Content-Disposition"] 
-                       response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
-                     else
-                       nil
-                     end
+          http = Net::HTTP.new(uri.host, uri.port)
 
-          if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
-            extension = filename.split(".").last
-            final_path += '.' + extension
+          if uri.scheme == "https"
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
           end
-          case response
-          when Net::HTTPSuccess, Net::HTTPOK
-            Misc.sensiblewrite(final_path) do |file|
-              response.read_body do |chunk|
-                file.write chunk
-              end
+
+          timeout = 60 * 10
+          http.read_timeout = timeout
+          http.open_timeout = timeout
+          request = Net::HTTP::Get.new(uri.request_uri)
+          http.request request do |response|
+            filename = if response["Content-Disposition"] 
+                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
+                       else
+                         nil
+                       end
+
+            if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
+              extension = filename.split(".").last
+              final_path += '.' + extension
             end
-          when Net::HTTPRedirection, Net::HTTPFound
-            location = response['location']
-            if location.include? 'get_directory'
-              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-              FileUtils.mkdir_p final_path unless File.exist? final_path
-              TmpFile.with_file do |tmp_dir|
-                Misc.in_dir tmp_dir do
-                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+            case response
+            when Net::HTTPSuccess, Net::HTTPOK
+              Misc.sensiblewrite(final_path) do |file|
+                response.read_body do |chunk|
+                  file.write chunk
                 end
-                FileUtils.mv tmp_dir, final_path
               end
+            when Net::HTTPRedirection, Net::HTTPFound
+              location = response['location']
+              if location.include? 'get_directory'
+                Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+                FileUtils.mkdir_p final_path unless File.exist? final_path
+                TmpFile.with_file do |tmp_dir|
+                  Misc.in_dir tmp_dir do
+                    CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+                  end
+                  FileUtils.mv tmp_dir, final_path
+                end
+              else
+                url = location
+                raise TryAgain
+                #Open.open(location, :nocache => true) do |s|
+                #  Misc.sensiblewrite(final_path, s)
+                #end
+              end
+            when Net::HTTPInternalServerError
+              @server_missing_resource_cache << url
+              raise "Resource Not Found"
             else
-              url = location
-              raise TryAgain
-              #Open.open(location, :nocache => true) do |s|
-              #  Misc.sensiblewrite(final_path, s)
-              #end
+              raise "Response not understood: #{response.inspect}"
             end
-          when Net::HTTPInternalServerError
-            @server_missing_resource_cache << url
-            raise "Resource Not Found"
-          else
-            raise "Response not understood: #{response.inspect}"
           end
+        rescue TryAgain
+          retry
         end
-      rescue TryAgain
-        retry
       end
     rescue
       Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"

data/lib/rbbt/tsv/csv.rb

@@ -28,10 +28,10 @@ module TSV
             if Misc.is_filename?(obj)
               CSV.read obj, options
             else
-              CSV.new obj, options
+              CSV.new obj, **options
             end
           else
-            CSV.new obj, options
+            CSV.new obj, **options
           end
 
     tsv = if noheaders

data/lib/rbbt/util/cmd.rb

@@ -192,25 +192,48 @@ module CMD
 
       ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail 
 
-      err_thread = Thread.new do
-        while line = serr.gets
-          bar.process(line) if bar
-          sout.log = line
-          Log.log "STDERR [#{pid}]: " +  line, stderr 
-        end if Integer === stderr and log
-        serr.close
+      if (Integer === stderr and log) || bar
+        err_thread = Thread.new do
+          while line = serr.gets
+            bar.process(line) if bar
+            sout.log = line
+            Log.log "STDERR [#{pid}]: " +  line, stderr if log
+          end 
+          serr.close
+        end
+      else
+        Misc.consume_stream(serr, true)
+        err_thread = nil
       end
 
       sout.threads = [in_thread, err_thread, wait_thr].compact
 
       sout
     else
-      err = ""
-      err_thread = Thread.new do
-        while not serr.eof?
-          err << serr.gets if Integer === stderr
+
+      if bar
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            line = serr.gets 
+            bar.process(line) 
+            err << line if Integer === stderr and log
+          end
+          serr.close
         end
-        serr.close
+      elsif log and Integer === stderr
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            err << serr.gets 
+          end
+          serr.close
+        end
+      else
+        Misc.consume_stream(serr, true)
+        #serr.close 
+        err_thread = nil
+        err = ""
       end
 
       ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
@@ -220,7 +243,11 @@ module CMD
 
       status = wait_thr.value
       if not status.success? and not no_fail
-        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+        if !err.empty?
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+        else
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
+        end
       else
         Log.log err, stderr if Integer === stderr and log
       end
@@ -263,11 +290,6 @@ module CMD
       if c == "\n"
         bar.process(line) if bar
         starting = true
-        #if pid
-        #  Log.logn "STDOUT [#{pid}]: ", level
-        #else
-        #  Log.logn "STDOUT: ", level
-        #end
         line = "" if bar
       end
     end 

data/lib/rbbt/util/simpleDSL.rb

@@ -38,13 +38,13 @@ module SimpleDSL
     hook_method(method)
 
     # Execute
+    @config ||= {}
     if actions.is_a? Proc
       begin
-        require 'parse_tree_extensions'
-        require 'parse_tree'
-        require 'ruby2ruby'
-        @config[@@method_name] = actions.to_ruby.collect[1..-2].join
+        require 'method_source'
+        @config[@@method_name] = actions.source.split("\n")[1..-2] * "\n"
       rescue Exception
+        Log.exception $!
         @config[@@method_name] = NoRuby2Ruby.new "The gem ruby2ruby is not installed. It will not work on ruby 1.9."
       end
       

data/lib/rbbt/workflow.rb

@@ -347,6 +347,11 @@ module Workflow
 
     inputs = IndiferentHash.setup(inputs)
 
+    not_overriden = inputs.delete :not_overriden 
+    if not_overriden
+      inputs[:not_overriden] = :not_overriden_dep
+    end
+
     Workflow.resolve_locals(inputs)
 
     task_info = task_info(taskname)
@@ -400,7 +405,11 @@ module Workflow
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
 
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
-    overriden = has_overriden_inputs || dependencies.select{|dep| dep.overriden }.any?
+
+    overriden_deps = dependencies.select{|d| d.overriden }
+    true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
+
+    overriden = has_overriden_inputs || overriden_deps.any?
 
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], task.extension
@@ -413,7 +422,16 @@ module Workflow
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
-    job.overriden = overriden
+
+    case not_overriden 
+    when TrueClass
+      job.overriden = has_overriden_inputs || true_overriden_deps.any?
+    when :not_overriden_dep
+      job.overriden = true if has_overriden_inputs || true_overriden_deps.any?
+    else
+      job.overriden = true if has_overriden_inputs || overriden_deps.any?
+    end
+
     job.real_inputs = real_inputs.keys
     job
   end

data/lib/rbbt/workflow/accessor.rb

@@ -302,7 +302,7 @@ module Workflow
         workflow, task = m.values_at 1, 2
         workflow = self.to_s if workflow.empty?
         override_dependencies[workflow] ||= IndiferentHash.setup({})
-        override_dependencies[workflow][task] = value
+        override_dependencies[workflow][task] = value.dup
       end
     end
     override_dependencies

data/lib/rbbt/workflow/step.rb

@@ -34,13 +34,40 @@ class Step
                     end
   end
 
+
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
   def overriden
-    if @overriden.nil? 
-      return [] if dependencies.nil?
-      dependencies.select{|dep| dep.overriden }.any?
-    else
-      @overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
     end
+    ord
   end
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
@@ -134,7 +161,11 @@ class Step
 
     archived_info = {}
     dependencies.each do |dep|
-      archived_info[dep.path] = dep.info
+      if Symbol === dep.overriden
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
       archived_info.merge!(dep.archived_info)
     end if dependencies
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -248,7 +248,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -551,7 +551,7 @@ class Step
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/share/rbbt_commands/hpc/list

@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end

data/share/rbbt_commands/hpc/orchestrate

@@ -19,6 +19,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -26,6 +27,8 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows

data/share/rbbt_commands/hpc/task

@@ -18,6 +18,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -25,6 +26,8 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows

data/share/rbbt_commands/lsf/list

@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end

data/share/rbbt_commands/lsf/orchestrate

@@ -19,6 +19,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -26,6 +27,8 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows

data/share/rbbt_commands/lsf/task

@@ -18,6 +18,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -25,6 +26,8 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows

data/share/rbbt_commands/slurm/list

@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end

data/share/rbbt_commands/slurm/orchestrate

@@ -19,6 +19,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -26,6 +27,8 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows

data/share/rbbt_commands/slurm/task

@@ -18,6 +18,7 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
@@ -25,6 +26,8 @@ $slurm_options = SOPT.get <<EOF
 -q--queue* Queue
 -t--task_cpus* Tasks
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
 -lin--licenses* SLURM licenses
 -cons--constraint* SLURM constraint
 -W--workflows* Additional workflows

data/share/rbbt_commands/workflow/forget_deps

@@ -38,11 +38,11 @@ step.copy_files_dir
 dependencies = step.dependencies
 
 if remove && remove.any?
- remove_paths = dependencies.select do |dep|
-  remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
- end.collect{|dep| dep.path } 
+  remove_paths = dependencies.select do |dep|
+    remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
+  end.collect{|dep| dep.path } 
 else
- remove_paths = dependencies.collect{|dep| dep.path }
+  remove_paths = dependencies.collect{|dep| dep.path }
 end
 
 step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths.include? info.last}
@@ -50,6 +50,7 @@ step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths
 if options[:purge] || options[:recursive_purge]
   dependencies.each do |dependency|
     next unless remove_paths.include? dependency.path
+    next unless Open.exists?(dependency.info_file)
     Step.purge(dependency.path, options[:recursive_purge])
   end
 end

data/test/rbbt/util/test_simpleDSL.rb

@@ -35,12 +35,12 @@ class TestDSL < Test::Unit::TestCase
 
   def test_config
     config = <<-EOC
-  action1("Hello")
-  action2("Good bye")
+  action1 "Hello"  
+  action2 "Good bye" 
     EOC
 
     begin
-      assert(@parser.config(:action) == config)
+      assert_equal config.split("\n").collect{|l| l.strip}, @parser.config(:action).split("\n").collect{|l| l.strip}
     rescue SimpleDSL::NoRuby2Ruby
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.10
+  version: 5.32.11
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-04 00:00:00.000000000 Z
+date: 2021-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -555,7 +555,7 @@ homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -570,104 +570,103 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/test_helper.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_association.rb
-- test/rbbt/hpc/test_slurm.rb
-- test/rbbt/hpc/test_batch.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/test_packed_index.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_fix_width_table.rb
 - test/rbbt/workflow/test_remote_workflow.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/util/test_archive.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
-- test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/util/test_migrate.rb
-- test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/test_procpath.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/util/R/test_plot.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_concurrency.rb
+- test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_config.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/test_python.rb
-- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_procpath.rb
+- test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_bgzf.rb
-- test/rbbt/util/misc/test_development.rb
-- test/rbbt/util/misc/test_format.rb
-- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/R/test_plot.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_config.rb
+- test/rbbt/util/test_log.rb
+- test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_R.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/test_hpc.rb
-- test/rbbt/test_persist.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/test_python.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_migrate.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_csv.rb
-- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_manipulate.rb
 - test/rbbt/tsv/test_excel.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/test_association.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_slurm.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/test_fix_width_table.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_hpc.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/test_helper.rb