rbbt-util 5.32.5 → 5.32.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 421178e8bb0a2b8631af02c890c177132a9bfdad08d62cc7bb728cab9679f8f0
-  data.tar.gz: 893045f7ff08c8bfbd1701dd6da74aee5e8b59a1f2af7d8fba21c06535f06e92
+  metadata.gz: fab4cf5fb60bb998eeed28a4e07758efd68f1103de0eff61a5b5c1d4a9fd2583
+  data.tar.gz: 47c9a2bdc555645ebc833c2a3b92a3452df259546c1dbf200d77dcee04b8969f
 SHA512:
-  metadata.gz: fd00a95c8217671959abcddd6591243afdc9920f46bbda881161eb7f60e0c08a591194057c25d8e69f1ffcd192abcea1feacfc714c032033debdfa7378e60290
-  data.tar.gz: 645deeb97dfd4b30a0e552efa5f7fede9edbb0d6936d0c3849e4354ceafcdbba41739efbd1a421e80c5fcf462a426d3765e05f04933748765ed066fb1edd168f
+  metadata.gz: 92121d5df854b036a37413c27d2244cef0325874c018433abce1a93887c93efc650474e059c13b84c0be9a708d6fdc735daba6f861e1d05a670950fea3d32916
+  data.tar.gz: 0d24d99a64101fd33f2ac5774c30267f238c81d6479a754445fc02424384ba716e4c672b8a7c0d1c6f22b9f16c2ac1e1104ba9d35f7697bf4a9631e3a776bb02

data/bin/rbbt

@@ -102,6 +102,7 @@ end
 
 if options[:config_keys]
   options[:config_keys].split(",").each do |config|
+    config = config.strip
     Rbbt::Config.process_config config
   end
 end

data/lib/rbbt/hpc/batch.rb

@@ -95,14 +95,15 @@ module HPC
 
       task = Symbol === job.overriden ? job.overriden : job.task_name
 
-      if job.overriden
-        override_deps = job.rec_dependencies.
-          select{|dep| Symbol === dep.overriden }.
+      if job.overriden?
+        #override_deps = job.rec_dependencies.
+        #  select{|dep| Symbol === dep.overriden }.
+        
+        override_deps = job.overriden_deps.
           collect do |dep| 
-
           name = [dep.workflow.to_s, dep.task_name] * "#"
           [name, dep.path] * "="  
-        end * ","
+        end.uniq * ","
 
         options[:override_deps] = override_deps unless override_deps.empty?
       end
@@ -156,10 +157,15 @@ EOF
         :singularity_ruby_inline,
         :sync,
         :task_cpus,
+        :mem,
+        :mem_per_cpu,
+        :licenses,
+        :contraints,
         :time,
         :user_group,
         :wipe_container,
         :workdir,
+        :purge_deps
       ]
 
       keys.each do |key|
@@ -378,9 +384,9 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
     end
 
     def execute(options)
-      exec_cmd, job_cmd = options.values_at :exec_cmd, :rbbt_cmd
+      exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :rbbt_cmd, :task_cpus
 
-      <<-EOF
+      script=<<-EOF
 step_path=$( 
       #{exec_cmd} #{job_cmd} --printpath
 )
@@ -388,7 +394,10 @@ exit_status=$?
 
 [[ -z $BATCH_JOB_ID ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_job $BATCH_JOB_ID
 [[ -z $BATCH_SYSTEM ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_system $BATCH_SYSTEM
+#{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_cpus #{task_cpus}
       EOF
+
+      script
     end
 
     def sync_environment(options = {})
@@ -409,6 +418,13 @@ fi
 
     def cleanup_environment(options = {})
       cleanup_environment = ""
+
+      cleanup_environment +=<<-EOF if options[:purge_deps]
+if [ $exit_status == '0' ]; then 
+  #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}' 
+fi
+      EOF
+
       if options[:sync]
         if options[:wipe_container] == 'force'
           cleanup_environment +=<<-EOF

data/lib/rbbt/hpc/slurm.rb

@@ -28,6 +28,11 @@ export BATCH_SYSTEM=SLURM
       workdir        = Misc.process_options options, :workdir
       exclusive      = Misc.process_options options, :exclusive
       highmem        = Misc.process_options options, :highmem
+      licenses       = Misc.process_options options, :licenses
+      constraint     = Misc.process_options options, :constraint
+
+      mem            = Misc.process_options options, :mem
+      mem_per_cpu    = Misc.process_options options, :mem_per_cpu
 
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
@@ -37,20 +42,34 @@ export BATCH_SYSTEM=SLURM
 
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
+      sbatch_params = {"job-name" => batch_name,
+                       "output" => fout,
+                       "error" => ferr,
+                       "cpus-per-task" => task_cpus,
+                       "nodes" => nodes,
+                       "time" => time,
+                       "exclusive" => exclusive,
+                       "licenses" => licenses,
+                       "mem" => mem,
+                       "mem-per-cpu" => mem_per_cpu,
+      }
+
       header =<<-EOF
 #!/bin/bash
-#SBATCH --job-name="#{batch_name}"
-#SBATCH --workdir="#{workdir}"
-#SBATCH --output="#{fout}"
-#SBATCH --error="#{ferr}"
-#SBATCH --qos="#{queue}"
-#SBATCH --cpus-per-task="#{task_cpus}"
-#SBATCH --time="#{time}"
-#SBATCH --nodes="#{nodes}"
       EOF
 
-      header << "#SBATCH --exclusive" << "\n" if exclusive
-      header << "#SBATCH --constraint=highmem" << "\n" if highmem
+      sbatch_params.each do |name,value|
+        next if value.nil? || value == ""
+        if TrueClass === value
+          header << "#SBATCH --#{name}" << "\n"
+        elsif Array === value
+          value.each do |v|
+            header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
+          end
+        else
+          header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
+        end
+      end
 
       header
     end

data/lib/rbbt/persist/tsv/adapter.rb

@@ -148,11 +148,7 @@ module Persist
     def write_lock
       write if closed?
       if write?
-        begin
-          return yield
-        ensure
-          close
-        end
+        return yield
       end
 
       lock do

data/lib/rbbt/resource.rb

@@ -4,10 +4,10 @@ require 'rbbt/resource/path'
 require 'net/http'
 require 'set'
 
- 
 module Resource
   class ResourceNotFound < RbbtException; end
 
+ 
   class << self
     attr_accessor :lock_dir
     
@@ -87,12 +87,21 @@ module Resource
       lock_filename = nil # it seems like this was locked already.
 
       Misc.lock lock_filename do
-        uri = URI(url)
+        begin
+          uri = URI(url)
+
+          http = Net::HTTP.new(uri.host, uri.port)
 
-        http = Net::HTTP.new(uri.host, uri.port)
-        request = Net::HTTP::Get.new(uri.request_uri)
-        timeout = 60 * 10
-        Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
+          if uri.scheme == "https"
+            http.use_ssl = true
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
+          end
+
+          timeout = 60 * 10
+          http.read_timeout = timeout
+          http.open_timeout = timeout
+          request = Net::HTTP::Get.new(uri.request_uri)
           http.request request do |response|
             filename = if response["Content-Disposition"] 
                          response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
@@ -123,9 +132,11 @@ module Resource
                   FileUtils.mv tmp_dir, final_path
                 end
               else
-                Open.open(location, :nocache => true) do |s|
-                  Misc.sensiblewrite(final_path, s)
-                end
+                url = location
+                raise TryAgain
+                #Open.open(location, :nocache => true) do |s|
+                #  Misc.sensiblewrite(final_path, s)
+                #end
               end
             when Net::HTTPInternalServerError
               @server_missing_resource_cache << url
@@ -134,6 +145,8 @@ module Resource
               raise "Response not understood: #{response.inspect}"
             end
           end
+        rescue TryAgain
+          retry
         end
       end
     rescue
@@ -313,7 +326,10 @@ url='#{url}'
   def identify(path)
     path = File.expand_path(path)
     resource ||= Rbbt
-    (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
+    locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
+    locations -= [:current, "current"]
+    locations << :current
+    locations.uniq.each do |name|
       pattern = resource.search_paths[name]
       next if pattern.nil?
       pattern = pattern.sub('{PWD}', Dir.pwd)

data/lib/rbbt/tsv/csv.rb

@@ -28,10 +28,10 @@ module TSV
             if Misc.is_filename?(obj)
               CSV.read obj, options
             else
-              CSV.new obj, options
+              CSV.new obj, **options
             end
           else
-            CSV.new obj, options
+            CSV.new obj, **options
           end
 
     tsv = if noheaders

data/lib/rbbt/util/cmd.rb

@@ -192,25 +192,48 @@ module CMD
 
       ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail 
 
-      err_thread = Thread.new do
-        while line = serr.gets
-          bar.process(line) if bar
-          sout.log = line
-          Log.log "STDERR [#{pid}]: " +  line, stderr 
-        end if Integer === stderr and log
-        serr.close
+      if (Integer === stderr and log) || bar
+        err_thread = Thread.new do
+          while line = serr.gets
+            bar.process(line) if bar
+            sout.log = line
+            Log.log "STDERR [#{pid}]: " +  line, stderr if log
+          end 
+          serr.close
+        end
+      else
+        Misc.consume_stream(serr, true)
+        err_thread = nil
       end
 
       sout.threads = [in_thread, err_thread, wait_thr].compact
 
       sout
     else
-      err = ""
-      err_thread = Thread.new do
-        while not serr.eof?
-          err << serr.gets if Integer === stderr
+
+      if bar
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            line = serr.gets 
+            bar.process(line) 
+            err << line if Integer === stderr and log
+          end
+          serr.close
         end
-        serr.close
+      elsif log and Integer === stderr
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            err << serr.gets 
+          end
+          serr.close
+        end
+      else
+        Misc.consume_stream(serr, true)
+        #serr.close 
+        err_thread = nil
+        err = ""
       end
 
       ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
@@ -220,7 +243,11 @@ module CMD
 
       status = wait_thr.value
       if not status.success? and not no_fail
-        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+        if !err.empty?
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+        else
+          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
+        end
       else
         Log.log err, stderr if Integer === stderr and log
       end
@@ -248,16 +275,21 @@ module CMD
     pid = io.pids.first
 
     line = "" if bar
+    starting = true
     while c = io.getc
-      STDERR << c if Log.severity <= level
-      line << c if bar
-      if c == "\n"
-        bar.process(line) if bar
+      if starting
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        starting = false
+      end
+      STDERR << c if Log.severity <= level
+      line << c if bar
+      if c == "\n"
+        bar.process(line) if bar
+        starting = true
         line = "" if bar
       end
     end 

data/lib/rbbt/util/migrate.rb

@@ -0,0 +1,118 @@
+module Rbbt 
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil)
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
+require 'rbbt-util'
+path = "#{path}"
+if Open.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+puts path
+puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      if File.exists?(path)
+        path = resource.identify(path)
+      else
+        path = Path.setup(path)
+      end
+
+      [path, (path.directory? ? path.glob_all : path.find_all), path]
+    end
+  end
+
+  def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
+    if target
+      Misc.ssh_run(target, <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "#{path}"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+      EOF
+    else
+      resource[path].find(search_path)
+    end
+  end
+
+  def self.migrate_files(real_paths, target, options = {})
+    excludes = %w(.save .crap .source tmp filecache open-remote)
+    excludes += (options[:exclude] || "").split(/,\s*/)
+    excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
+
+    other = options[:other] || []
+
+    test_str = options[:test] ? '-nv' : ''
+
+    real_paths.each do |source_path|
+      if File.directory?(source_path) || source_path =~ /\/$/
+        source_path += "/" unless source_path[-1] == "/"
+        target += "/" unless target[-1] == "/"
+      end
+
+      next if source_path == target
+
+      if options[:target]
+        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+      else
+        Open.mkdir File.dirname(target)
+      end
+
+      if options[:target]
+        target_path = [options[:target], "'" + target + "'"] * ":" 
+      else
+        target_path = "'" + target + "'"
+      end
+
+      TmpFile.with_file do |tmp_files|
+        if options[:files]
+          Open.write(tmp_files, options[:files] * "\n")
+          files_from_str = "--files-from='#{tmp_files}'"
+        else
+          files_from_str = ""
+        end
+
+        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+
+
+        cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
+
+        if options[:print]
+          puts cmd 
+          exit 0
+        else
+          CMD.cmd_log(cmd, :log => Log::INFO)
+
+          if options[:delete] && options[:files]
+            remove_files = options[:files].collect{|f| File.join(source_path, f) }
+            dirs = remove_files.select{|f| File.directory? f }
+            remove_files.each do |file|
+              next if dirs.include? file
+              Open.rm file
+            end
+            dirs.each do |dir|
+              FileUtils.rmdir dir if Dir.glob(dir).empty?
+            end
+          end 
+        end
+      end
+    end
+  end
+
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
+
+    target = migrate_target_path(lpath, search_path, resource, options[:target])
+
+    migrate_files(real_paths, target, options)
+  end
+end