rbbt-util 5.32.4 → 5.32.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5623a5f5b4b78dc051b02bfaa9f7503f999bc58b7cc3569d8a58f9caead7b212
-  data.tar.gz: 195a44841df5ef636806413bf4bcfe1b36304d0b9e4178d3e787b1ee2e5d5a71
+  metadata.gz: 258c55a2b59d81426d1aa3980f35104ef193774068330b25c9aad29061d70e6a
+  data.tar.gz: 8265a128c161349455f7803d5a70a259c03481bb7a441aa1ffec58626fbb0226
 SHA512:
-  metadata.gz: 9d9a436961cd66fd58bdd1b3908879b16f3e9f7a09d1d05a5d4e1980ad3c0a960a102895ca257bbd18298352c59f0448d21520c749bb652dc63c0042ef56d694
-  data.tar.gz: 9c9ec75fd411deb4849ed02834294da897a38ca2230a2af78e1bcb8722fe8ab1c1dd4eebe41e073270da02d60288363eb548f3bfdbf8cabd31256d5616ce9d09
+  metadata.gz: 8eb143ab5cf5e4e55851ccd11b24eeae7d3e60948e30a71de3b5fb87f5333d7e40823b268d6ac894eea5d3d73ffa47101f7bb229378837799597df54545bb8c8
+  data.tar.gz: 0ebddaae9d523d9dd767dfde1b351b011ba3dd65c4abd7c0bcc0e78cc3e604a6850250bf0e74dac9f5ca0172a56a067541bdc34b5037e2ffc7ba972558f2ce4c

data/lib/rbbt/hpc/slurm.rb

@@ -28,6 +28,8 @@ export BATCH_SYSTEM=SLURM
       workdir        = Misc.process_options options, :workdir
       exclusive      = Misc.process_options options, :exclusive
       highmem        = Misc.process_options options, :highmem
+      licenses       = Misc.process_options options, :licenses
+      constraint     = Misc.process_options options, :constraint
 
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
@@ -51,6 +53,8 @@ export BATCH_SYSTEM=SLURM
 
       header << "#SBATCH --exclusive" << "\n" if exclusive
       header << "#SBATCH --constraint=highmem" << "\n" if highmem
+      header << "#SBATCH --licenses=#{licenses}" << "\n" if licenses
+      header << "#SBATCH --constraint=#{constraint}" << "\n" if constraint
 
       header
     end

data/lib/rbbt/persist/tsv/adapter.rb

@@ -148,11 +148,7 @@ module Persist
     def write_lock
       write if closed?
       if write?
-        begin
-          return yield
-        ensure
-          close
-        end
+        return yield
       end
 
       lock do

data/lib/rbbt/resource.rb

@@ -4,10 +4,10 @@ require 'rbbt/resource/path'
 require 'net/http'
 require 'set'
 
- 
 module Resource
   class ResourceNotFound < RbbtException; end
 
+ 
   class << self
     attr_accessor :lock_dir
     
@@ -90,51 +90,62 @@ module Resource
         uri = URI(url)
 
         http = Net::HTTP.new(uri.host, uri.port)
-        request = Net::HTTP::Get.new(uri.request_uri)
+
+        if uri.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
+        end
+
         timeout = 60 * 10
-        Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
-          http.request request do |response|
-            filename = if response["Content-Disposition"] 
-                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
-                       else
-                         nil
-                       end
-
-            if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
-              extension = filename.split(".").last
-              final_path += '.' + extension
-            end
-            case response
-            when Net::HTTPSuccess, Net::HTTPOK
-              Misc.sensiblewrite(final_path) do |file|
-                response.read_body do |chunk|
-                  file.write chunk
-                end
+        http.read_timeout = timeout
+        http.open_timeout = timeout
+        request = Net::HTTP::Get.new(uri.request_uri)
+        http.request request do |response|
+          filename = if response["Content-Disposition"] 
+                       response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
+                     else
+                       nil
+                     end
+
+          if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
+            extension = filename.split(".").last
+            final_path += '.' + extension
+          end
+          case response
+          when Net::HTTPSuccess, Net::HTTPOK
+            Misc.sensiblewrite(final_path) do |file|
+              response.read_body do |chunk|
+                file.write chunk
               end
-            when Net::HTTPRedirection, Net::HTTPFound
-              location = response['location']
-              if location.include? 'get_directory'
-                Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-                FileUtils.mkdir_p final_path unless File.exist? final_path
-                TmpFile.with_file do |tmp_dir|
-                  Misc.in_dir tmp_dir do
-                    CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
-                  end
-                  FileUtils.mv tmp_dir, final_path
-                end
-              else
-                Open.open(location, :nocache => true) do |s|
-                  Misc.sensiblewrite(final_path, s)
+            end
+          when Net::HTTPRedirection, Net::HTTPFound
+            location = response['location']
+            if location.include? 'get_directory'
+              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+              FileUtils.mkdir_p final_path unless File.exist? final_path
+              TmpFile.with_file do |tmp_dir|
+                Misc.in_dir tmp_dir do
+                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
                 end
+                FileUtils.mv tmp_dir, final_path
               end
-            when Net::HTTPInternalServerError
-              @server_missing_resource_cache << url
-              raise "Resource Not Found"
             else
-              raise "Response not understood: #{response.inspect}"
+              url = location
+              raise TryAgain
+              #Open.open(location, :nocache => true) do |s|
+              #  Misc.sensiblewrite(final_path, s)
+              #end
             end
+          when Net::HTTPInternalServerError
+            @server_missing_resource_cache << url
+            raise "Resource Not Found"
+          else
+            raise "Response not understood: #{response.inspect}"
           end
         end
+      rescue TryAgain
+        retry
       end
     rescue
       Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
@@ -159,7 +170,11 @@ module Resource
         begin
           produce(path.annotate(path + '.gz'), force)
         rescue ResourceNotFound
-          produce(path.annotate(path + '.bgz'), force)
+          begin
+            produce(path.annotate(path + '.bgz'), force)
+          rescue ResourceNotFound
+            raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+          end
         end
       else
         raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
@@ -309,7 +324,10 @@ url='#{url}'
   def identify(path)
     path = File.expand_path(path)
     resource ||= Rbbt
-    (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
+    locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
+    locations -= [:current, "current"]
+    locations << :current
+    locations.uniq.each do |name|
       pattern = resource.search_paths[name]
       next if pattern.nil?
       pattern = pattern.sub('{PWD}', Dir.pwd)

data/lib/rbbt/util/cmd.rb

@@ -248,16 +248,26 @@ module CMD
     pid = io.pids.first
 
     line = "" if bar
+    starting = true
     while c = io.getc
-      STDERR << c if Log.severity <= level
-      line << c if bar
-      if c == "\n"
-        bar.process(line) if bar
+      if starting
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        starting = false
+      end
+      STDERR << c if Log.severity <= level
+      line << c if bar
+      if c == "\n"
+        bar.process(line) if bar
+        starting = true
+        #if pid
+        #  Log.logn "STDOUT [#{pid}]: ", level
+        #else
+        #  Log.logn "STDOUT: ", level
+        #end
         line = "" if bar
       end
     end 

data/lib/rbbt/util/migrate.rb

@@ -0,0 +1,118 @@
+module Rbbt 
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil)
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
+require 'rbbt-util'
+path = "#{path}"
+if Open.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+puts path
+puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      if File.exists?(path)
+        path = resource.identify(path)
+      else
+        path = Path.setup(path)
+      end
+
+      [path, (path.directory? ? path.glob_all : path.find_all), path]
+    end
+  end
+
+  def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
+    if target
+      Misc.ssh_run(target, <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "#{path}"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+      EOF
+    else
+      resource[path].find(search_path)
+    end
+  end
+
+  def self.migrate_files(real_paths, target, options = {})
+    excludes = %w(.save .crap .source tmp filecache open-remote)
+    excludes += (options[:exclude] || "").split(/,\s*/)
+    excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
+
+    other = options[:other] || []
+
+    test_str = options[:test] ? '-nv' : ''
+
+    real_paths.each do |source_path|
+      if File.directory?(source_path) || source_path =~ /\/$/
+        source_path += "/" unless source_path[-1] == "/"
+        target += "/" unless target[-1] == "/"
+      end
+
+      next if source_path == target
+
+      if options[:target]
+        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+      else
+        Open.mkdir File.dirname(target)
+      end
+
+      if options[:target]
+        target_path = [options[:target], "'" + target + "'"] * ":" 
+      else
+        target_path = "'" + target + "'"
+      end
+
+      TmpFile.with_file do |tmp_files|
+        if options[:files]
+          Open.write(tmp_files, options[:files] * "\n")
+          files_from_str = "--files-from='#{tmp_files}'"
+        else
+          files_from_str = ""
+        end
+
+        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+
+
+        cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
+
+        if options[:print]
+          puts cmd 
+          exit 0
+        else
+          CMD.cmd_log(cmd, :log => Log::INFO)
+
+          if options[:delete] && options[:files]
+            remove_files = options[:files].collect{|f| File.join(source_path, f) }
+            dirs = remove_files.select{|f| File.directory? f }
+            remove_files.each do |file|
+              next if dirs.include? file
+              Open.rm file
+            end
+            dirs.each do |dir|
+              FileUtils.rmdir dir if Dir.glob(dir).empty?
+            end
+          end 
+        end
+      end
+    end
+  end
+
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
+
+    target = migrate_target_path(lpath, search_path, resource, options[:target])
+
+    migrate_files(real_paths, target, options)
+  end
+end

data/lib/rbbt/workflow.rb

@@ -600,6 +600,17 @@ module Workflow
         true
       end
     end
+
+    if ! Open.exists?(step.info_file)
+      begin
+        workflow = step.path.split("/")[-3]
+        task_name = step.path.split("/")[-2]
+        workflow = Kernel.const_get workflow
+        step.task = workflow.tasks[task_name.to_sym]
+      rescue
+        Log.exception $!
+      end
+    end
     step
   end
 

data/lib/rbbt/workflow/util/archive.rb

@@ -1,3 +1,4 @@
+require 'rbbt/util/migrate'
 class Step
 
   MAIN_RSYNC_ARGS="-avztAXHP --copy-links"
@@ -126,63 +127,48 @@ class Step
     end
   end
 
-  def self.migrate(path, search_path, options = {})
-    resource=Rbbt
-
-    orig_path = path
-    other_rsync_args = options[:rsync]
-
-    recursive = options[:recursive]
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil, recursive = true)
     recursive = false if recursive.nil?
-
-    paths = if options[:source]
-              Misc.ssh_run(options[:source], <<-EOF).split("\n")
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
 require 'rbbt-util'
 require 'rbbt/workflow'
 
-path = "#{path}"
 recursive = #{ recursive.to_s }
+path = "#{path}"
 
-if File.exists?(path)
+if Open.exists?(path)
   path = #{resource.to_s}.identify(path)
 else
   path = Path.setup(path)
 end
 
-files = path.glob_all
-
+files = path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p }
 files = Step.job_files_for_archive(files, recursive)
 
+puts path
 puts files * "\n"
-              EOF
-
-            else
-              if File.exists?(path)
-                path = resource.identify(path)
-                raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
-              else
-                path = Path.setup(path)
-              end
-              files = path.glob_all
-              files = Step.job_files_for_archive(files, recursive)
-              files
-            end
-
-
-    target = if options[:target] 
-               target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
-require 'rbbt-util'
-path = "var/jobs"
-resource = #{resource.to_s}
-search_path = "#{search_path}"
-puts resource[path].find(search_path)
-               EOF
-             else
-               resource['var/jobs'].find(search_path)
-             end
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      path = Path.setup(path.dup)
+      files = path.glob_all
+      files = Step.job_files_for_archive(files, recursive)
+
+      [path, files, path]
+    end
+  end
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
 
     subpath_files = {}
-    paths.sort.each do |path|
+    real_paths.sort.each do |path|
       parts = path.split("/")
       subpath = parts[0..-4] * "/" + "/"
 
@@ -190,73 +176,16 @@ puts resource[path].find(search_path)
         subpath = subpath_files.keys.last
       end
 
-      source = path[subpath.length..-1]
+      source = path.chars[subpath.length..-1] * ""
 
       subpath_files[subpath] ||= []
       subpath_files[subpath] << source
     end
 
-    synced_files = []
-    subpath_files.each do |subpath, files|
-      if options[:target]
-        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
-      else
-        Open.mkdir File.dirname(target)
-      end
+    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
 
-      if options[:source]
-        source = [options[:source], subpath] * ":"
-      else
-        source = subpath
-      end
-      target = [options[:target], target] * ":" if options[:target]
-
-      next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
-
-      files_and_dirs = Set.new( files )
-      files.each do |file|
-        synced_files << File.join(subpath, file)
-
-        parts = file.split("/")[0..-2].reject{|p| p.empty?}
-        while parts.any?
-          files_and_dirs <<  parts * "/"
-          parts.pop
-        end
-      end
-
-      TmpFile.with_file(files_and_dirs.sort_by{|l| l.length}.to_a * "\n") do |tmp_include_file|
-        test_str = options[:test] ? '-nv' : ''
-
-        cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
-
-        #cmd << " && rm -Rf #{source}" if options[:delete]
-        if options[:print]
-          ppp Open.read(tmp_include_file)
-          puts cmd 
-        else
-          CMD.cmd_log(cmd, :log => Log::INFO)
-        end
-      end
-    end
-
-    if options[:delete] && synced_files.any?
-      puts Log.color :magenta, "About to erase these files:"
-      synced_files.each do |p|
-        puts Log.color :red, p
-      end 
-
-      if options[:non_interactive]
-        response = 'yes'
-      else
-        puts Log.color :magenta, "Type 'yes' if you are sure:"
-        response = STDIN.gets.chomp
-      end
-
-      if response == 'yes'
-        synced_files.each do |p|
-          Open.rm p
-        end
-      end
+    subpath_files.each do |subpath, files|
+      Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
   end