rbbt-util 5.32.10 → 5.32.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 258c55a2b59d81426d1aa3980f35104ef193774068330b25c9aad29061d70e6a
-  data.tar.gz: 8265a128c161349455f7803d5a70a259c03481bb7a441aa1ffec58626fbb0226
+  metadata.gz: c2898a052d60bd9d6c05e3d277482e2221d07c18067b592fe7b8fefd796e8a6a
+  data.tar.gz: 55bdd8b9a78d2d35a2c4c1574da11113c66b7621128076f29128941a5a0caf77
 SHA512:
-  metadata.gz: 8eb143ab5cf5e4e55851ccd11b24eeae7d3e60948e30a71de3b5fb87f5333d7e40823b268d6ac894eea5d3d73ffa47101f7bb229378837799597df54545bb8c8
-  data.tar.gz: 0ebddaae9d523d9dd767dfde1b351b011ba3dd65c4abd7c0bcc0e78cc3e604a6850250bf0e74dac9f5ca0172a56a067541bdc34b5037e2ffc7ba972558f2ce4c
+  metadata.gz: 3dd350c68a0c931da771f789aa934e3a94cf52be343101743a14eb2e049b782665db8f941da1f28e7503386af3f17c6ef4e53afa0ef388aecb65c9571ea974a5
+  data.tar.gz: 110a0a0eed2343835e3c4598f9c91c0280e28c64911a4fa18662c423d4f3d1774893b2e0ed848676a60878cae608a9b3d2e50de7c1c724935894971320628f2e

data/bin/rbbt

@@ -102,6 +102,7 @@ end
 
 if options[:config_keys]
   options[:config_keys].split(",").each do |config|
+    config = config.strip
     Rbbt::Config.process_config config
   end
 end

data/lib/rbbt/annotations/util.rb

@@ -152,6 +152,7 @@ module Annotated
       tsv = TSV.setup({}, :key_field => "List", :fields => fields, :type => :list, :unnamed => true)
 
       annot_id = annotations.id
+      annot_id = annot_id * "," if Array === annot_id
       tsv[annot_id] = annotations.tsv_values(*fields).dup
 
     when Array === annotations 

data/lib/rbbt/entity.rb

@@ -197,7 +197,7 @@ module Entity
             if self.instance_variable_get("@multiple_result_" + name.to_s)
               return self.instance_variable_get("@multiple_result_" + name.to_s)
             end
-            raise MultipleEntity, "Entity #{name} runs with multiple entities"
+            raise MultipleEntity, "Entity property #{name} runs with multiple entities"
           end
 
           define_method name do |*args|
@@ -225,10 +225,12 @@ module Entity
             case res
             when Array
               missing.zip(res).each do |o,res|
+                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
               end
             when Hash
               res.each do |o,res|
+                raise "Multiple function #{name} result nil for element #{o}" if res.nil?
                 o.instance_variable_set("@multiple_result_" + name.to_s, res)
               end
             end
@@ -254,7 +256,10 @@ module Entity
 
         orig_method_name = method_name
         multi_name = "_multiple_" + method_name.to_s
+        single_name = "_single_" + method_name.to_s
+
         method_name = multi_name if self.instance_methods.include?(multi_name.to_sym)
+        method_name = single_name if self.instance_methods.include?(single_name.to_sym)
 
         orig_name = UNPERSISTED_PREFIX + method_name.to_s
         alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym

data/lib/rbbt/hpc/batch.rb

@@ -95,14 +95,15 @@ module HPC
 
       task = Symbol === job.overriden ? job.overriden : job.task_name
 
-      if job.overriden
-        override_deps = job.rec_dependencies.
-          select{|dep| Symbol === dep.overriden }.
+      if job.overriden?
+        #override_deps = job.rec_dependencies.
+        #  select{|dep| Symbol === dep.overriden }.
+        
+        override_deps = job.overriden_deps.
           collect do |dep| 
-
           name = [dep.workflow.to_s, dep.task_name] * "#"
           [name, dep.path] * "="  
-        end * ","
+        end.uniq * ","
 
         options[:override_deps] = override_deps unless override_deps.empty?
       end
@@ -138,7 +139,7 @@ EOF
 
       keys = [
         :batch_dir,
-        :batch_modules,
+        :lua_modules,
         :batch_name,
         :contain,
         :contain_and_sync,
@@ -156,10 +157,16 @@ EOF
         :singularity_ruby_inline,
         :sync,
         :task_cpus,
+        :gres,
+        :mem,
+        :mem_per_cpu,
+        :licenses,
+        :contraints,
         :time,
         :user_group,
         :wipe_container,
         :workdir,
+        :purge_deps
       ]
 
       keys.each do |key|
@@ -283,7 +290,7 @@ let MAX_MEMORY="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / 1024"
     end
 
     def prepare_environment(options = {})
-      modules = options[:batch_modules]
+      modules = options[:lua_modules]
 
       prepare_environment = ""
 
@@ -378,9 +385,9 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
     end
 
     def execute(options)
-      exec_cmd, job_cmd = options.values_at :exec_cmd, :rbbt_cmd
+      exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :rbbt_cmd, :task_cpus
 
-      <<-EOF
+      script=<<-EOF
 step_path=$( 
       #{exec_cmd} #{job_cmd} --printpath
 )
@@ -388,7 +395,10 @@ exit_status=$?
 
 [[ -z $BATCH_JOB_ID ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_job $BATCH_JOB_ID
 [[ -z $BATCH_SYSTEM ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_system $BATCH_SYSTEM
+#{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_cpus #{task_cpus}
       EOF
+
+      script
     end
 
     def sync_environment(options = {})
@@ -409,6 +419,13 @@ fi
 
     def cleanup_environment(options = {})
       cleanup_environment = ""
+
+      cleanup_environment +=<<-EOF if options[:purge_deps]
+if [ $exit_status == '0' ]; then 
+  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}' 
+fi
+      EOF
+
       if options[:sync]
         if options[:wipe_container] == 'force'
           cleanup_environment +=<<-EOF

data/lib/rbbt/hpc/orchestrate.rb

@@ -97,6 +97,7 @@ module HPC
         chains[job] ||= [] 
         chains[job] << dep 
         chains[job].concat chains[dep] if chains[dep]
+        chains[job].uniq!
       end
 
       chains

data/lib/rbbt/hpc/slurm.rb

@@ -30,6 +30,10 @@ export BATCH_SYSTEM=SLURM
       highmem        = Misc.process_options options, :highmem
       licenses       = Misc.process_options options, :licenses
       constraint     = Misc.process_options options, :constraint
+      gres           = Misc.process_options options, :gres
+
+      mem            = Misc.process_options options, :mem
+      mem_per_cpu    = Misc.process_options options, :mem_per_cpu
 
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
@@ -39,22 +43,35 @@ export BATCH_SYSTEM=SLURM
 
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
+      sbatch_params = {"job-name" => batch_name,
+                       "output" => fout,
+                       "error" => ferr,
+                       "cpus-per-task" => task_cpus,
+                       "nodes" => nodes,
+                       "time" => time,
+                       "exclusive" => exclusive,
+                       "licenses" => licenses,
+                       "gres" => gres,
+                       "mem" => mem,
+                       "mem-per-cpu" => mem_per_cpu,
+      }
+
       header =<<-EOF
 #!/bin/bash
-#SBATCH --job-name="#{batch_name}"
-#SBATCH --workdir="#{workdir}"
-#SBATCH --output="#{fout}"
-#SBATCH --error="#{ferr}"
-#SBATCH --qos="#{queue}"
-#SBATCH --cpus-per-task="#{task_cpus}"
-#SBATCH --time="#{time}"
-#SBATCH --nodes="#{nodes}"
       EOF
 
-      header << "#SBATCH --exclusive" << "\n" if exclusive
-      header << "#SBATCH --constraint=highmem" << "\n" if highmem
-      header << "#SBATCH --licenses=#{licenses}" << "\n" if licenses
-      header << "#SBATCH --constraint=#{constraint}" << "\n" if constraint
+      sbatch_params.each do |name,value|
+        next if value.nil? || value == ""
+        if TrueClass === value
+          header << "#SBATCH --#{name}" << "\n"
+        elsif Array === value
+          value.each do |v|
+            header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
+          end
+        else
+          header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
+        end
+      end
 
       header
     end

data/lib/rbbt/resource.rb

@@ -87,65 +87,67 @@ module Resource
       lock_filename = nil # it seems like this was locked already.
 
       Misc.lock lock_filename do
-        uri = URI(url)
-
-        http = Net::HTTP.new(uri.host, uri.port)
-
-        if uri.scheme == "https"
-          http.use_ssl = true
-          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
-        end
+        begin
+          uri = URI(url)
 
-        timeout = 60 * 10
-        http.read_timeout = timeout
-        http.open_timeout = timeout
-        request = Net::HTTP::Get.new(uri.request_uri)
-        http.request request do |response|
-          filename = if response["Content-Disposition"] 
-                       response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
-                     else
-                       nil
-                     end
+          http = Net::HTTP.new(uri.host, uri.port)
 
-          if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
-            extension = filename.split(".").last
-            final_path += '.' + extension
+          if uri.scheme == "https"
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
           end
-          case response
-          when Net::HTTPSuccess, Net::HTTPOK
-            Misc.sensiblewrite(final_path) do |file|
-              response.read_body do |chunk|
-                file.write chunk
-              end
+
+          timeout = 60 * 10
+          http.read_timeout = timeout
+          http.open_timeout = timeout
+          request = Net::HTTP::Get.new(uri.request_uri)
+          http.request request do |response|
+            filename = if response["Content-Disposition"] 
+                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
+                       else
+                         nil
+                       end
+
+            if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
+              extension = filename.split(".").last
+              final_path += '.' + extension
             end
-          when Net::HTTPRedirection, Net::HTTPFound
-            location = response['location']
-            if location.include? 'get_directory'
-              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-              FileUtils.mkdir_p final_path unless File.exist? final_path
-              TmpFile.with_file do |tmp_dir|
-                Misc.in_dir tmp_dir do
-                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+            case response
+            when Net::HTTPSuccess, Net::HTTPOK
+              Misc.sensiblewrite(final_path) do |file|
+                response.read_body do |chunk|
+                  file.write chunk
                 end
-                FileUtils.mv tmp_dir, final_path
               end
+            when Net::HTTPRedirection, Net::HTTPFound
+              location = response['location']
+              if location.include? 'get_directory'
+                Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+                FileUtils.mkdir_p final_path unless File.exist? final_path
+                TmpFile.with_file do |tmp_dir|
+                  Misc.in_dir tmp_dir do
+                    CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+                  end
+                  FileUtils.mv tmp_dir, final_path
+                end
+              else
+                url = location
+                raise TryAgain
+                #Open.open(location, :nocache => true) do |s|
+                #  Misc.sensiblewrite(final_path, s)
+                #end
+              end
+            when Net::HTTPInternalServerError
+              @server_missing_resource_cache << url
+              raise "Resource Not Found"
             else
-              url = location
-              raise TryAgain
-              #Open.open(location, :nocache => true) do |s|
-              #  Misc.sensiblewrite(final_path, s)
-              #end
+              raise "Response not understood: #{response.inspect}"
             end
-          when Net::HTTPInternalServerError
-            @server_missing_resource_cache << url
-            raise "Resource Not Found"
-          else
-            raise "Response not understood: #{response.inspect}"
           end
+        rescue TryAgain
+          retry
         end
-      rescue TryAgain
-        retry
       end
     rescue
       Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"

data/lib/rbbt/tsv.rb

@@ -98,6 +98,11 @@ module TSV
         stream = get_stream source, options.merge(open_options)
         parse stream, data, options.merge(:tsv_grep => tsv_grep)
 
+        if ! open_options[:noclose]
+          stream.close unless stream.closed?
+          stream.join if stream.respond_to?(:join)
+        end
+
         data.filename = filename.to_s unless filename.nil?
 
         if data.identifiers.nil? and Path === filename and filename.identifier_file_path

data/lib/rbbt/tsv/csv.rb

@@ -28,10 +28,10 @@ module TSV
             if Misc.is_filename?(obj)
               CSV.read obj, options
             else
-              CSV.new obj, options
+              CSV.new obj, **options
             end
           else
-            CSV.new obj, options
+            CSV.new obj, **options
           end
 
     tsv = if noheaders

data/lib/rbbt/tsv/manipulate.rb

@@ -174,6 +174,8 @@ module TSV
         if Hash === @monitor
           desc = @monitor[:desc] if @monitor.include? :desc 
           step = @monitor[:step] if @monitor.include? :step 
+        elsif String === @monitor
+          desc = @monitor
         end
         progress_monitor = Log::ProgressBar.new_bar(size, :desc => desc)
       end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -97,12 +97,11 @@ module TSV
         end
       end
       rescue
-        Log.exception $!
         error = true
         raise $!
       ensure
         join.call(error) if join
-        Log::ProgressBar.remove_bar(bar) if bar
+        Log::ProgressBar.remove_bar(bar, error) if bar
       end
   end
 
@@ -138,7 +137,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -178,7 +177,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -220,7 +219,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -274,7 +273,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -319,7 +318,7 @@ module TSV
       raise $!
     ensure
       join.call(error) if join
-      Log::ProgressBar.remove_bar(bar) if bar
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end
 
@@ -390,7 +389,7 @@ module TSV
       when Set
         traverse_array(obj.to_a, options, &block)
       when String
-        if Open.remote? obj or Misc.is_filename? obj
+        if Open.remote?(obj) or Misc.is_filename?(obj)
           Open.open(obj) do |s|
             traverse_obj(s, options, &block)
           end
@@ -491,9 +490,7 @@ module TSV
       q.join
       raise $!
     ensure
-      if bar
-        Log::ProgressBar.remove_bar(bar, error)
-      end
+      Log::ProgressBar.remove_bar(bar, error) if bar
     end
   end