rbbt-util 5.32.9 → 5.32.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/slurm.rb +4 -0
- data/lib/rbbt/persist/tsv/adapter.rb +1 -5
- data/lib/rbbt/resource.rb +49 -38
- data/share/rbbt_commands/hpc/orchestrate +3 -1
- data/share/rbbt_commands/hpc/task +3 -1
- data/share/rbbt_commands/lsf/orchestrate +3 -1
- data/share/rbbt_commands/lsf/task +3 -1
- data/share/rbbt_commands/slurm/orchestrate +3 -1
- data/share/rbbt_commands/slurm/task +3 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 258c55a2b59d81426d1aa3980f35104ef193774068330b25c9aad29061d70e6a
|
4
|
+
data.tar.gz: 8265a128c161349455f7803d5a70a259c03481bb7a441aa1ffec58626fbb0226
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8eb143ab5cf5e4e55851ccd11b24eeae7d3e60948e30a71de3b5fb87f5333d7e40823b268d6ac894eea5d3d73ffa47101f7bb229378837799597df54545bb8c8
|
7
|
+
data.tar.gz: 0ebddaae9d523d9dd767dfde1b351b011ba3dd65c4abd7c0bcc0e78cc3e604a6850250bf0e74dac9f5ca0172a56a067541bdc34b5037e2ffc7ba972558f2ce4c
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -28,6 +28,8 @@ export BATCH_SYSTEM=SLURM
|
|
28
28
|
workdir = Misc.process_options options, :workdir
|
29
29
|
exclusive = Misc.process_options options, :exclusive
|
30
30
|
highmem = Misc.process_options options, :highmem
|
31
|
+
licenses = Misc.process_options options, :licenses
|
32
|
+
constraint = Misc.process_options options, :constraint
|
31
33
|
|
32
34
|
batch_dir = Misc.process_options options, :batch_dir
|
33
35
|
batch_name = Misc.process_options options, :batch_name
|
@@ -51,6 +53,8 @@ export BATCH_SYSTEM=SLURM
|
|
51
53
|
|
52
54
|
header << "#SBATCH --exclusive" << "\n" if exclusive
|
53
55
|
header << "#SBATCH --constraint=highmem" << "\n" if highmem
|
56
|
+
header << "#SBATCH --licenses=#{licenses}" << "\n" if licenses
|
57
|
+
header << "#SBATCH --constraint=#{constraint}" << "\n" if constraint
|
54
58
|
|
55
59
|
header
|
56
60
|
end
|
data/lib/rbbt/resource.rb
CHANGED
@@ -4,10 +4,10 @@ require 'rbbt/resource/path'
|
|
4
4
|
require 'net/http'
|
5
5
|
require 'set'
|
6
6
|
|
7
|
-
|
8
7
|
module Resource
|
9
8
|
class ResourceNotFound < RbbtException; end
|
10
9
|
|
10
|
+
|
11
11
|
class << self
|
12
12
|
attr_accessor :lock_dir
|
13
13
|
|
@@ -90,51 +90,62 @@ module Resource
|
|
90
90
|
uri = URI(url)
|
91
91
|
|
92
92
|
http = Net::HTTP.new(uri.host, uri.port)
|
93
|
-
|
93
|
+
|
94
|
+
if uri.scheme == "https"
|
95
|
+
http.use_ssl = true
|
96
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
97
|
+
http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
|
98
|
+
end
|
99
|
+
|
94
100
|
timeout = 60 * 10
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
101
|
+
http.read_timeout = timeout
|
102
|
+
http.open_timeout = timeout
|
103
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
104
|
+
http.request request do |response|
|
105
|
+
filename = if response["Content-Disposition"]
|
106
|
+
response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
|
107
|
+
else
|
108
|
+
nil
|
109
|
+
end
|
110
|
+
|
111
|
+
if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
|
112
|
+
extension = filename.split(".").last
|
113
|
+
final_path += '.' + extension
|
114
|
+
end
|
115
|
+
case response
|
116
|
+
when Net::HTTPSuccess, Net::HTTPOK
|
117
|
+
Misc.sensiblewrite(final_path) do |file|
|
118
|
+
response.read_body do |chunk|
|
119
|
+
file.write chunk
|
113
120
|
end
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
FileUtils.mv tmp_dir, final_path
|
124
|
-
end
|
125
|
-
else
|
126
|
-
Open.open(location, :nocache => true) do |s|
|
127
|
-
Misc.sensiblewrite(final_path, s)
|
121
|
+
end
|
122
|
+
when Net::HTTPRedirection, Net::HTTPFound
|
123
|
+
location = response['location']
|
124
|
+
if location.include? 'get_directory'
|
125
|
+
Log.debug("Feching directory from: #{location}. Into: #{final_path}")
|
126
|
+
FileUtils.mkdir_p final_path unless File.exist? final_path
|
127
|
+
TmpFile.with_file do |tmp_dir|
|
128
|
+
Misc.in_dir tmp_dir do
|
129
|
+
CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
|
128
130
|
end
|
131
|
+
FileUtils.mv tmp_dir, final_path
|
129
132
|
end
|
130
|
-
when Net::HTTPInternalServerError
|
131
|
-
@server_missing_resource_cache << url
|
132
|
-
raise "Resource Not Found"
|
133
133
|
else
|
134
|
-
|
134
|
+
url = location
|
135
|
+
raise TryAgain
|
136
|
+
#Open.open(location, :nocache => true) do |s|
|
137
|
+
# Misc.sensiblewrite(final_path, s)
|
138
|
+
#end
|
135
139
|
end
|
140
|
+
when Net::HTTPInternalServerError
|
141
|
+
@server_missing_resource_cache << url
|
142
|
+
raise "Resource Not Found"
|
143
|
+
else
|
144
|
+
raise "Response not understood: #{response.inspect}"
|
136
145
|
end
|
137
146
|
end
|
147
|
+
rescue TryAgain
|
148
|
+
retry
|
138
149
|
end
|
139
150
|
rescue
|
140
151
|
Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
|
@@ -25,8 +25,10 @@ $slurm_options = SOPT.get <<EOF
|
|
25
25
|
-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
|
26
26
|
-q--queue* Queue
|
27
27
|
-t--task_cpus* Tasks
|
28
|
-
-W--workflows* Additional workflows
|
29
28
|
-tm--time* Time
|
29
|
+
-lin--licenses* SLURM licenses
|
30
|
+
-cons--constraint* SLURM constraint
|
31
|
+
-W--workflows* Additional workflows
|
30
32
|
-OR--orchestration_rules* Orchestration rules
|
31
33
|
-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
|
32
34
|
EOF
|
@@ -24,8 +24,10 @@ $slurm_options = SOPT.get <<EOF
|
|
24
24
|
-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
|
25
25
|
-q--queue* Queue
|
26
26
|
-t--task_cpus* Tasks
|
27
|
-
-W--workflows* Additional workflows
|
28
27
|
-tm--time* Time
|
28
|
+
-lin--licenses* SLURM licenses
|
29
|
+
-cons--constraint* SLURM constraint
|
30
|
+
-W--workflows* Additional workflows
|
29
31
|
-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
|
30
32
|
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
31
33
|
EOF
|
@@ -25,8 +25,10 @@ $slurm_options = SOPT.get <<EOF
|
|
25
25
|
-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
|
26
26
|
-q--queue* Queue
|
27
27
|
-t--task_cpus* Tasks
|
28
|
-
-W--workflows* Additional workflows
|
29
28
|
-tm--time* Time
|
29
|
+
-lin--licenses* SLURM licenses
|
30
|
+
-cons--constraint* SLURM constraint
|
31
|
+
-W--workflows* Additional workflows
|
30
32
|
-OR--orchestration_rules* Orchestration rules
|
31
33
|
-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
|
32
34
|
EOF
|
@@ -24,8 +24,10 @@ $slurm_options = SOPT.get <<EOF
|
|
24
24
|
-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
|
25
25
|
-q--queue* Queue
|
26
26
|
-t--task_cpus* Tasks
|
27
|
-
-W--workflows* Additional workflows
|
28
27
|
-tm--time* Time
|
28
|
+
-lin--licenses* SLURM licenses
|
29
|
+
-cons--constraint* SLURM constraint
|
30
|
+
-W--workflows* Additional workflows
|
29
31
|
-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
|
30
32
|
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
31
33
|
EOF
|
@@ -25,8 +25,10 @@ $slurm_options = SOPT.get <<EOF
|
|
25
25
|
-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
|
26
26
|
-q--queue* Queue
|
27
27
|
-t--task_cpus* Tasks
|
28
|
-
-W--workflows* Additional workflows
|
29
28
|
-tm--time* Time
|
29
|
+
-lin--licenses* SLURM licenses
|
30
|
+
-cons--constraint* SLURM constraint
|
31
|
+
-W--workflows* Additional workflows
|
30
32
|
-OR--orchestration_rules* Orchestration rules
|
31
33
|
-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
|
32
34
|
EOF
|
@@ -24,8 +24,10 @@ $slurm_options = SOPT.get <<EOF
|
|
24
24
|
-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
|
25
25
|
-q--queue* Queue
|
26
26
|
-t--task_cpus* Tasks
|
27
|
-
-W--workflows* Additional workflows
|
28
27
|
-tm--time* Time
|
28
|
+
-lin--licenses* SLURM licenses
|
29
|
+
-cons--constraint* SLURM constraint
|
30
|
+
-W--workflows* Additional workflows
|
29
31
|
-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
|
30
32
|
-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
|
31
33
|
EOF
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.32.
|
4
|
+
version: 5.32.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-05-
|
11
|
+
date: 2021-05-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|