rbbt-util 5.32.9 → 5.32.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b9d236294c4bdcc32e517eadcffa4d103e5a99a220096497a50f8baa26f746d
-  data.tar.gz: 6370930ac76bd8b86666fa59556d2d8648789c3e3dda56bb71c71c2ff65b88d3
+  metadata.gz: 258c55a2b59d81426d1aa3980f35104ef193774068330b25c9aad29061d70e6a
+  data.tar.gz: 8265a128c161349455f7803d5a70a259c03481bb7a441aa1ffec58626fbb0226
 SHA512:
-  metadata.gz: 7dcd6fadf6424add27b6773d16ef0b794183c4b2f52616498d8c2ec21a77c5b5ad8f3fed65ca69ce5111c71e0942f84be79c107b40156c331919533f3fc5bea7
-  data.tar.gz: ea294d8aa5f8ff04a9e902fdd2d7d2287a34934f7288a11234996d06fb72dba4f333e655d591d9b78c5bc00104036356625088f9aa4c7e808ae2d42d0b253781
+  metadata.gz: 8eb143ab5cf5e4e55851ccd11b24eeae7d3e60948e30a71de3b5fb87f5333d7e40823b268d6ac894eea5d3d73ffa47101f7bb229378837799597df54545bb8c8
+  data.tar.gz: 0ebddaae9d523d9dd767dfde1b351b011ba3dd65c4abd7c0bcc0e78cc3e604a6850250bf0e74dac9f5ca0172a56a067541bdc34b5037e2ffc7ba972558f2ce4c

data/lib/rbbt/hpc/slurm.rb

@@ -28,6 +28,8 @@ export BATCH_SYSTEM=SLURM
       workdir        = Misc.process_options options, :workdir
       exclusive      = Misc.process_options options, :exclusive
       highmem        = Misc.process_options options, :highmem
+      licenses       = Misc.process_options options, :licenses
+      constraint     = Misc.process_options options, :constraint
 
       batch_dir  = Misc.process_options options, :batch_dir
       batch_name = Misc.process_options options, :batch_name
@@ -51,6 +53,8 @@ export BATCH_SYSTEM=SLURM
 
       header << "#SBATCH --exclusive" << "\n" if exclusive
       header << "#SBATCH --constraint=highmem" << "\n" if highmem
+      header << "#SBATCH --licenses=#{licenses}" << "\n" if licenses
+      header << "#SBATCH --constraint=#{constraint}" << "\n" if constraint
 
       header
     end

data/lib/rbbt/persist/tsv/adapter.rb

@@ -148,11 +148,7 @@ module Persist
     def write_lock
       write if closed?
       if write?
-        begin
-          return yield
-        ensure
-          close
-        end
+        return yield
       end
 
       lock do

data/lib/rbbt/resource.rb

@@ -4,10 +4,10 @@ require 'rbbt/resource/path'
 require 'net/http'
 require 'set'
 
- 
 module Resource
   class ResourceNotFound < RbbtException; end
 
+ 
   class << self
     attr_accessor :lock_dir
     
@@ -90,51 +90,62 @@ module Resource
         uri = URI(url)
 
         http = Net::HTTP.new(uri.host, uri.port)
-        request = Net::HTTP::Get.new(uri.request_uri)
+
+        if uri.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
+        end
+
         timeout = 60 * 10
-        Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
-          http.request request do |response|
-            filename = if response["Content-Disposition"] 
-                         response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
-                       else
-                         nil
-                       end
-
-            if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
-              extension = filename.split(".").last
-              final_path += '.' + extension
-            end
-            case response
-            when Net::HTTPSuccess, Net::HTTPOK
-              Misc.sensiblewrite(final_path) do |file|
-                response.read_body do |chunk|
-                  file.write chunk
-                end
+        http.read_timeout = timeout
+        http.open_timeout = timeout
+        request = Net::HTTP::Get.new(uri.request_uri)
+        http.request request do |response|
+          filename = if response["Content-Disposition"] 
+                       response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
+                     else
+                       nil
+                     end
+
+          if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
+            extension = filename.split(".").last
+            final_path += '.' + extension
+          end
+          case response
+          when Net::HTTPSuccess, Net::HTTPOK
+            Misc.sensiblewrite(final_path) do |file|
+              response.read_body do |chunk|
+                file.write chunk
               end
-            when Net::HTTPRedirection, Net::HTTPFound
-              location = response['location']
-              if location.include? 'get_directory'
-                Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-                FileUtils.mkdir_p final_path unless File.exist? final_path
-                TmpFile.with_file do |tmp_dir|
-                  Misc.in_dir tmp_dir do
-                    CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
-                  end
-                  FileUtils.mv tmp_dir, final_path
-                end
-              else
-                Open.open(location, :nocache => true) do |s|
-                  Misc.sensiblewrite(final_path, s)
+            end
+          when Net::HTTPRedirection, Net::HTTPFound
+            location = response['location']
+            if location.include? 'get_directory'
+              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+              FileUtils.mkdir_p final_path unless File.exist? final_path
+              TmpFile.with_file do |tmp_dir|
+                Misc.in_dir tmp_dir do
+                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
                 end
+                FileUtils.mv tmp_dir, final_path
               end
-            when Net::HTTPInternalServerError
-              @server_missing_resource_cache << url
-              raise "Resource Not Found"
             else
-              raise "Response not understood: #{response.inspect}"
+              url = location
+              raise TryAgain
+              #Open.open(location, :nocache => true) do |s|
+              #  Misc.sensiblewrite(final_path, s)
+              #end
             end
+          when Net::HTTPInternalServerError
+            @server_missing_resource_cache << url
+            raise "Resource Not Found"
+          else
+            raise "Response not understood: #{response.inspect}"
           end
         end
+      rescue TryAgain
+        retry
       end
     rescue
       Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"

data/share/rbbt_commands/hpc/orchestrate

@@ -25,8 +25,10 @@ $slurm_options = SOPT.get <<EOF
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF

data/share/rbbt_commands/hpc/task

@@ -24,8 +24,10 @@ $slurm_options = SOPT.get <<EOF
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF

data/share/rbbt_commands/lsf/orchestrate

@@ -25,8 +25,10 @@ $slurm_options = SOPT.get <<EOF
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF

data/share/rbbt_commands/lsf/task

@@ -24,8 +24,10 @@ $slurm_options = SOPT.get <<EOF
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF

data/share/rbbt_commands/slurm/orchestrate

@@ -25,8 +25,10 @@ $slurm_options = SOPT.get <<EOF
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF

data/share/rbbt_commands/slurm/task

@@ -24,8 +24,10 @@ $slurm_options = SOPT.get <<EOF
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.9
+  version: 5.32.10
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-03 00:00:00.000000000 Z
+date: 2021-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake