rbbt-util 5.32.6 → 5.32.12

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2bc1c3443b09a4274a030b64479091e506ed1e165371d999eb154b750b2806b3
4
- data.tar.gz: 83d0ae91ee88ed3283985b5d7101759bbb45db37fbb54271bc7f16fae8ab4f52
3
+ metadata.gz: 355cf2694fc6b449600580f6c09f2a0a6055013285b5c53ada6ad2f920d7f63c
4
+ data.tar.gz: 26e430d330e39371b8fb59f3508b3e1eb215bf042bf2faa5262040fd19369e24
5
5
  SHA512:
6
- metadata.gz: 0b83533d504d6d66b06083a7f19a689e9b3f82fefbc26278d9bd9af0efbdfb010f87cc609bf90396bb17257a643715e335ac8127f9ff1f11ccdd52f639b77ca8
7
- data.tar.gz: a4b93ca7d481120edde1d65f529984e652c261f365c67bea6d1c463d360733006d29486d3851d0b071ba2005cb90cf71382eb5665c7d4e1828a25f322b570ce7
6
+ metadata.gz: ecec79da1a0b8ecdbcec6e600965d6cc8a336f6d33a4dd3ada4a75e2c6ae8d98300a210bc17eec7bb16639c4d73b4a6efe7bceb043037fa0aeaf76d3da20b172
7
+ data.tar.gz: 3af9d791b8423f99c67c473541fcc6b685c2cb4ca454423ff0b69d686442b40574d337df749dcf811369b81d68c648ff733dcf3c4e2e655953aba0f4569b85b1
data/bin/rbbt CHANGED
@@ -102,6 +102,7 @@ end
102
102
 
103
103
  if options[:config_keys]
104
104
  options[:config_keys].split(",").each do |config|
105
+ config = config.strip
105
106
  Rbbt::Config.process_config config
106
107
  end
107
108
  end
@@ -95,14 +95,15 @@ module HPC
95
95
 
96
96
  task = Symbol === job.overriden ? job.overriden : job.task_name
97
97
 
98
- if job.overriden
99
- override_deps = job.rec_dependencies.
100
- select{|dep| Symbol === dep.overriden }.
98
+ if job.overriden?
99
+ #override_deps = job.rec_dependencies.
100
+ # select{|dep| Symbol === dep.overriden }.
101
+
102
+ override_deps = job.overriden_deps.
101
103
  collect do |dep|
102
-
103
104
  name = [dep.workflow.to_s, dep.task_name] * "#"
104
105
  [name, dep.path] * "="
105
- end * ","
106
+ end.uniq * ","
106
107
 
107
108
  options[:override_deps] = override_deps unless override_deps.empty?
108
109
  end
@@ -156,10 +157,15 @@ EOF
156
157
  :singularity_ruby_inline,
157
158
  :sync,
158
159
  :task_cpus,
160
+ :mem,
161
+ :mem_per_cpu,
162
+ :licenses,
163
+ :contraints,
159
164
  :time,
160
165
  :user_group,
161
166
  :wipe_container,
162
167
  :workdir,
168
+ :purge_deps
163
169
  ]
164
170
 
165
171
  keys.each do |key|
@@ -378,9 +384,9 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
378
384
  end
379
385
 
380
386
  def execute(options)
381
- exec_cmd, job_cmd = options.values_at :exec_cmd, :rbbt_cmd
387
+ exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :rbbt_cmd, :task_cpus
382
388
 
383
- <<-EOF
389
+ script=<<-EOF
384
390
  step_path=$(
385
391
  #{exec_cmd} #{job_cmd} --printpath
386
392
  )
@@ -388,7 +394,10 @@ exit_status=$?
388
394
 
389
395
  [[ -z $BATCH_JOB_ID ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_job $BATCH_JOB_ID
390
396
  [[ -z $BATCH_SYSTEM ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_system $BATCH_SYSTEM
397
+ #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_cpus #{task_cpus}
391
398
  EOF
399
+
400
+ script
392
401
  end
393
402
 
394
403
  def sync_environment(options = {})
@@ -409,6 +418,13 @@ fi
409
418
 
410
419
  def cleanup_environment(options = {})
411
420
  cleanup_environment = ""
421
+
422
+ cleanup_environment +=<<-EOF if options[:purge_deps]
423
+ if [ $exit_status == '0' ]; then
424
+ #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}'
425
+ fi
426
+ EOF
427
+
412
428
  if options[:sync]
413
429
  if options[:wipe_container] == 'force'
414
430
  cleanup_environment +=<<-EOF
@@ -28,6 +28,11 @@ export BATCH_SYSTEM=SLURM
28
28
  workdir = Misc.process_options options, :workdir
29
29
  exclusive = Misc.process_options options, :exclusive
30
30
  highmem = Misc.process_options options, :highmem
31
+ licenses = Misc.process_options options, :licenses
32
+ constraint = Misc.process_options options, :constraint
33
+
34
+ mem = Misc.process_options options, :mem
35
+ mem_per_cpu = Misc.process_options options, :mem_per_cpu
31
36
 
32
37
  batch_dir = Misc.process_options options, :batch_dir
33
38
  batch_name = Misc.process_options options, :batch_name
@@ -37,20 +42,34 @@ export BATCH_SYSTEM=SLURM
37
42
 
38
43
  time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
39
44
 
45
+ sbatch_params = {"job-name" => batch_name,
46
+ "output" => fout,
47
+ "error" => ferr,
48
+ "cpus-per-task" => task_cpus,
49
+ "nodes" => nodes,
50
+ "time" => time,
51
+ "exclusive" => exclusive,
52
+ "licenses" => licenses,
53
+ "mem" => mem,
54
+ "mem-per-cpu" => mem_per_cpu,
55
+ }
56
+
40
57
  header =<<-EOF
41
58
  #!/bin/bash
42
- #SBATCH --job-name="#{batch_name}"
43
- #SBATCH --workdir="#{workdir}"
44
- #SBATCH --output="#{fout}"
45
- #SBATCH --error="#{ferr}"
46
- #SBATCH --qos="#{queue}"
47
- #SBATCH --cpus-per-task="#{task_cpus}"
48
- #SBATCH --time="#{time}"
49
- #SBATCH --nodes="#{nodes}"
50
59
  EOF
51
60
 
52
- header << "#SBATCH --exclusive" << "\n" if exclusive
53
- header << "#SBATCH --constraint=highmem" << "\n" if highmem
61
+ sbatch_params.each do |name,value|
62
+ next if value.nil? || value == ""
63
+ if TrueClass === value
64
+ header << "#SBATCH --#{name}" << "\n"
65
+ elsif Array === value
66
+ value.each do |v|
67
+ header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
68
+ end
69
+ else
70
+ header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
71
+ end
72
+ end
54
73
 
55
74
  header
56
75
  end
@@ -148,11 +148,7 @@ module Persist
148
148
  def write_lock
149
149
  write if closed?
150
150
  if write?
151
- begin
152
- return yield
153
- ensure
154
- close
155
- end
151
+ return yield
156
152
  end
157
153
 
158
154
  lock do
data/lib/rbbt/resource.rb CHANGED
@@ -4,10 +4,10 @@ require 'rbbt/resource/path'
4
4
  require 'net/http'
5
5
  require 'set'
6
6
 
7
-
8
7
  module Resource
9
8
  class ResourceNotFound < RbbtException; end
10
9
 
10
+
11
11
  class << self
12
12
  attr_accessor :lock_dir
13
13
 
@@ -87,12 +87,21 @@ module Resource
87
87
  lock_filename = nil # it seems like this was locked already.
88
88
 
89
89
  Misc.lock lock_filename do
90
- uri = URI(url)
90
+ begin
91
+ uri = URI(url)
92
+
93
+ http = Net::HTTP.new(uri.host, uri.port)
91
94
 
92
- http = Net::HTTP.new(uri.host, uri.port)
93
- request = Net::HTTP::Get.new(uri.request_uri)
94
- timeout = 60 * 10
95
- Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
95
+ if uri.scheme == "https"
96
+ http.use_ssl = true
97
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
98
+ http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
99
+ end
100
+
101
+ timeout = 60 * 10
102
+ http.read_timeout = timeout
103
+ http.open_timeout = timeout
104
+ request = Net::HTTP::Get.new(uri.request_uri)
96
105
  http.request request do |response|
97
106
  filename = if response["Content-Disposition"]
98
107
  response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
@@ -123,9 +132,11 @@ module Resource
123
132
  FileUtils.mv tmp_dir, final_path
124
133
  end
125
134
  else
126
- Open.open(location, :nocache => true) do |s|
127
- Misc.sensiblewrite(final_path, s)
128
- end
135
+ url = location
136
+ raise TryAgain
137
+ #Open.open(location, :nocache => true) do |s|
138
+ # Misc.sensiblewrite(final_path, s)
139
+ #end
129
140
  end
130
141
  when Net::HTTPInternalServerError
131
142
  @server_missing_resource_cache << url
@@ -134,6 +145,8 @@ module Resource
134
145
  raise "Response not understood: #{response.inspect}"
135
146
  end
136
147
  end
148
+ rescue TryAgain
149
+ retry
137
150
  end
138
151
  end
139
152
  rescue
@@ -313,7 +326,10 @@ url='#{url}'
313
326
  def identify(path)
314
327
  path = File.expand_path(path)
315
328
  resource ||= Rbbt
316
- (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
329
+ locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
330
+ locations -= [:current, "current"]
331
+ locations << :current
332
+ locations.uniq.each do |name|
317
333
  pattern = resource.search_paths[name]
318
334
  next if pattern.nil?
319
335
  pattern = pattern.sub('{PWD}', Dir.pwd)
data/lib/rbbt/tsv/csv.rb CHANGED
@@ -28,10 +28,10 @@ module TSV
28
28
  if Misc.is_filename?(obj)
29
29
  CSV.read obj, options
30
30
  else
31
- CSV.new obj, options
31
+ CSV.new obj, **options
32
32
  end
33
33
  else
34
- CSV.new obj, options
34
+ CSV.new obj, **options
35
35
  end
36
36
 
37
37
  tsv = if noheaders
data/lib/rbbt/util/cmd.rb CHANGED
@@ -192,25 +192,47 @@ module CMD
192
192
 
193
193
  ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail
194
194
 
195
- err_thread = Thread.new do
196
- while line = serr.gets
197
- bar.process(line) if bar
198
- sout.log = line
199
- Log.log "STDERR [#{pid}]: " + line, stderr
200
- end if Integer === stderr and log
201
- serr.close
195
+ if (Integer === stderr and log) || bar
196
+ err_thread = Thread.new do
197
+ while line = serr.gets
198
+ bar.process(line) if bar
199
+ sout.log = line
200
+ Log.log "STDERR [#{pid}]: " + line, stderr if log
201
+ end
202
+ serr.close
203
+ end
204
+ else
205
+ err_thread = Misc.consume_stream(serr, true)
202
206
  end
203
207
 
204
208
  sout.threads = [in_thread, err_thread, wait_thr].compact
205
209
 
206
210
  sout
207
211
  else
208
- err = ""
209
- err_thread = Thread.new do
210
- while not serr.eof?
211
- err << serr.gets if Integer === stderr
212
+
213
+ if bar
214
+ err = ""
215
+ err_thread = Thread.new do
216
+ while not serr.eof?
217
+ line = serr.gets
218
+ bar.process(line)
219
+ err << line if Integer === stderr and log
220
+ end
221
+ serr.close
212
222
  end
213
- serr.close
223
+ elsif log and Integer === stderr
224
+ err = ""
225
+ err_thread = Thread.new do
226
+ while not serr.eof?
227
+ err << serr.gets
228
+ end
229
+ serr.close
230
+ end
231
+ else
232
+ Misc.consume_stream(serr, true)
233
+ #serr.close
234
+ err_thread = nil
235
+ err = ""
214
236
  end
215
237
 
216
238
  ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail
@@ -220,7 +242,11 @@ module CMD
220
242
 
221
243
  status = wait_thr.value
222
244
  if not status.success? and not no_fail
223
- raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
245
+ if !err.empty?
246
+ raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
247
+ else
248
+ raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
249
+ end
224
250
  else
225
251
  Log.log err, stderr if Integer === stderr and log
226
252
  end
@@ -248,16 +274,21 @@ module CMD
248
274
  pid = io.pids.first
249
275
 
250
276
  line = "" if bar
277
+ starting = true
251
278
  while c = io.getc
252
- STDERR << c if Log.severity <= level
253
- line << c if bar
254
- if c == "\n"
255
- bar.process(line) if bar
279
+ if starting
256
280
  if pid
257
281
  Log.logn "STDOUT [#{pid}]: ", level
258
282
  else
259
283
  Log.logn "STDOUT: ", level
260
284
  end
285
+ starting = false
286
+ end
287
+ STDERR << c if Log.severity <= level
288
+ line << c if bar
289
+ if c == "\n"
290
+ bar.process(line) if bar
291
+ starting = true
261
292
  line = "" if bar
262
293
  end
263
294
  end
@@ -12,32 +12,35 @@ module Log
12
12
  attr_accessor :history, :mean_max, :max_history
13
13
  def thr_msg
14
14
  if @history.nil?
15
- @history ||= [[@ticks, Time.now] ]
15
+ @history ||= [[0, @start], [@ticks, Time.now] ]
16
16
  elsif @last_ticks != @ticks
17
17
  @history << [@ticks, Time.now]
18
- max_history ||= case
19
- when @ticks > 20
20
- count = @ticks - @last_count
21
- count = 1 if count == 0
22
- if @max
23
- times = @max / count
24
- num = times / 20
25
- num = 2 if num < 2
26
- else
27
- num = 10
18
+ max_history ||= begin
19
+ max_history = case
20
+ when @ticks > 20
21
+ count = @ticks - @last_count
22
+ count = 1 if count == 0
23
+ if @max
24
+ times = @max / count
25
+ num = times / 20
26
+ num = 2 if num < 2
27
+ else
28
+ num = 10
29
+ end
30
+ count * num
31
+ else
32
+ 20
33
+ end
34
+ max_history = 30 if max_history > 30
35
+ max_history
28
36
  end
29
- count * num
30
- else
31
- 20
32
- end
33
- max_history = 30 if max_history > 30
34
37
  @history.shift if @history.length > max_history
35
38
  end
36
39
 
37
40
  @last_ticks = @ticks
38
41
 
39
42
  @mean_max ||= 0
40
- if @history.length > 2
43
+ if @history.length > 3
41
44
 
42
45
  sticks, stime = @history.first
43
46
  ssticks, sstime = @history[-3]
@@ -0,0 +1,118 @@
1
+ module Rbbt
2
+ def self.migrate_source_paths(path, resource = Rbbt, source = nil)
3
+ if source
4
+ lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
5
+ require 'rbbt-util'
6
+ path = "#{path}"
7
+ if Open.exists?(path)
8
+ path = #{resource.to_s}.identify(path)
9
+ else
10
+ path = Path.setup(path)
11
+ end
12
+ puts path
13
+ puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
14
+ EOF
15
+
16
+ [path, paths.collect{|p| [source, p] * ":"}, lpath]
17
+ else
18
+ if File.exists?(path)
19
+ path = resource.identify(path)
20
+ else
21
+ path = Path.setup(path)
22
+ end
23
+
24
+ [path, (path.directory? ? path.glob_all : path.find_all), path]
25
+ end
26
+ end
27
+
28
+ def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
29
+ if target
30
+ Misc.ssh_run(target, <<-EOF).split("\n").first
31
+ require 'rbbt-util'
32
+ path = "#{path}"
33
+ resource = #{resource.to_s}
34
+ search_path = "#{search_path}"
35
+ puts resource[path].find(search_path)
36
+ EOF
37
+ else
38
+ resource[path].find(search_path)
39
+ end
40
+ end
41
+
42
+ def self.migrate_files(real_paths, target, options = {})
43
+ excludes = %w(.save .crap .source tmp filecache open-remote)
44
+ excludes += (options[:exclude] || "").split(/,\s*/)
45
+ excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
46
+
47
+ other = options[:other] || []
48
+
49
+ test_str = options[:test] ? '-nv' : ''
50
+
51
+ real_paths.each do |source_path|
52
+ if File.directory?(source_path) || source_path =~ /\/$/
53
+ source_path += "/" unless source_path[-1] == "/"
54
+ target += "/" unless target[-1] == "/"
55
+ end
56
+
57
+ next if source_path == target
58
+
59
+ if options[:target]
60
+ CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
61
+ else
62
+ Open.mkdir File.dirname(target)
63
+ end
64
+
65
+ if options[:target]
66
+ target_path = [options[:target], "'" + target + "'"] * ":"
67
+ else
68
+ target_path = "'" + target + "'"
69
+ end
70
+
71
+ TmpFile.with_file do |tmp_files|
72
+ if options[:files]
73
+ Open.write(tmp_files, options[:files] * "\n")
74
+ files_from_str = "--files-from='#{tmp_files}'"
75
+ else
76
+ files_from_str = ""
77
+ end
78
+
79
+ cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
80
+
81
+
82
+ cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
83
+
84
+ if options[:print]
85
+ puts cmd
86
+ exit 0
87
+ else
88
+ CMD.cmd_log(cmd, :log => Log::INFO)
89
+
90
+ if options[:delete] && options[:files]
91
+ remove_files = options[:files].collect{|f| File.join(source_path, f) }
92
+ dirs = remove_files.select{|f| File.directory? f }
93
+ remove_files.each do |file|
94
+ next if dirs.include? file
95
+ Open.rm file
96
+ end
97
+ dirs.each do |dir|
98
+ FileUtils.rmdir dir if Dir.glob(dir).empty?
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+
106
+
107
+ def self.migrate(path, search_path, options = {})
108
+ search_path = 'user' if search_path.nil?
109
+
110
+ resource = Rbbt
111
+
112
+ path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
113
+
114
+ target = migrate_target_path(lpath, search_path, resource, options[:target])
115
+
116
+ migrate_files(real_paths, target, options)
117
+ end
118
+ end