rbbt-util 5.32.5 → 5.32.11

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 421178e8bb0a2b8631af02c890c177132a9bfdad08d62cc7bb728cab9679f8f0
4
- data.tar.gz: 893045f7ff08c8bfbd1701dd6da74aee5e8b59a1f2af7d8fba21c06535f06e92
3
+ metadata.gz: fab4cf5fb60bb998eeed28a4e07758efd68f1103de0eff61a5b5c1d4a9fd2583
4
+ data.tar.gz: 47c9a2bdc555645ebc833c2a3b92a3452df259546c1dbf200d77dcee04b8969f
5
5
  SHA512:
6
- metadata.gz: fd00a95c8217671959abcddd6591243afdc9920f46bbda881161eb7f60e0c08a591194057c25d8e69f1ffcd192abcea1feacfc714c032033debdfa7378e60290
7
- data.tar.gz: 645deeb97dfd4b30a0e552efa5f7fede9edbb0d6936d0c3849e4354ceafcdbba41739efbd1a421e80c5fcf462a426d3765e05f04933748765ed066fb1edd168f
6
+ metadata.gz: 92121d5df854b036a37413c27d2244cef0325874c018433abce1a93887c93efc650474e059c13b84c0be9a708d6fdc735daba6f861e1d05a670950fea3d32916
7
+ data.tar.gz: 0d24d99a64101fd33f2ac5774c30267f238c81d6479a754445fc02424384ba716e4c672b8a7c0d1c6f22b9f16c2ac1e1104ba9d35f7697bf4a9631e3a776bb02
data/bin/rbbt CHANGED
@@ -102,6 +102,7 @@ end
102
102
 
103
103
  if options[:config_keys]
104
104
  options[:config_keys].split(",").each do |config|
105
+ config = config.strip
105
106
  Rbbt::Config.process_config config
106
107
  end
107
108
  end
@@ -95,14 +95,15 @@ module HPC
95
95
 
96
96
  task = Symbol === job.overriden ? job.overriden : job.task_name
97
97
 
98
- if job.overriden
99
- override_deps = job.rec_dependencies.
100
- select{|dep| Symbol === dep.overriden }.
98
+ if job.overriden?
99
+ #override_deps = job.rec_dependencies.
100
+ # select{|dep| Symbol === dep.overriden }.
101
+
102
+ override_deps = job.overriden_deps.
101
103
  collect do |dep|
102
-
103
104
  name = [dep.workflow.to_s, dep.task_name] * "#"
104
105
  [name, dep.path] * "="
105
- end * ","
106
+ end.uniq * ","
106
107
 
107
108
  options[:override_deps] = override_deps unless override_deps.empty?
108
109
  end
@@ -156,10 +157,15 @@ EOF
156
157
  :singularity_ruby_inline,
157
158
  :sync,
158
159
  :task_cpus,
160
+ :mem,
161
+ :mem_per_cpu,
162
+ :licenses,
163
+ :contraints,
159
164
  :time,
160
165
  :user_group,
161
166
  :wipe_container,
162
167
  :workdir,
168
+ :purge_deps
163
169
  ]
164
170
 
165
171
  keys.each do |key|
@@ -378,9 +384,9 @@ echo "user_scratch: #{scratch_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}"
378
384
  end
379
385
 
380
386
  def execute(options)
381
- exec_cmd, job_cmd = options.values_at :exec_cmd, :rbbt_cmd
387
+ exec_cmd, job_cmd, task_cpus = options.values_at :exec_cmd, :rbbt_cmd, :task_cpus
382
388
 
383
- <<-EOF
389
+ script=<<-EOF
384
390
  step_path=$(
385
391
  #{exec_cmd} #{job_cmd} --printpath
386
392
  )
@@ -388,7 +394,10 @@ exit_status=$?
388
394
 
389
395
  [[ -z $BATCH_JOB_ID ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_job $BATCH_JOB_ID
390
396
  [[ -z $BATCH_SYSTEM ]] || #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_system $BATCH_SYSTEM
397
+ #{exec_cmd} workflow write_info --recursive --force=false --check_pid "$step_path" batch_cpus #{task_cpus}
391
398
  EOF
399
+
400
+ script
392
401
  end
393
402
 
394
403
  def sync_environment(options = {})
@@ -409,6 +418,13 @@ fi
409
418
 
410
419
  def cleanup_environment(options = {})
411
420
  cleanup_environment = ""
421
+
422
+ cleanup_environment +=<<-EOF if options[:purge_deps]
423
+ if [ $exit_status == '0' ]; then
424
+ #{options[:exec_cmd]} workflow forget_deps --purge --recursive_purge "$step_path" 2>1 >> '#{options[:fsync]}'
425
+ fi
426
+ EOF
427
+
412
428
  if options[:sync]
413
429
  if options[:wipe_container] == 'force'
414
430
  cleanup_environment +=<<-EOF
@@ -28,6 +28,11 @@ export BATCH_SYSTEM=SLURM
28
28
  workdir = Misc.process_options options, :workdir
29
29
  exclusive = Misc.process_options options, :exclusive
30
30
  highmem = Misc.process_options options, :highmem
31
+ licenses = Misc.process_options options, :licenses
32
+ constraint = Misc.process_options options, :constraint
33
+
34
+ mem = Misc.process_options options, :mem
35
+ mem_per_cpu = Misc.process_options options, :mem_per_cpu
31
36
 
32
37
  batch_dir = Misc.process_options options, :batch_dir
33
38
  batch_name = Misc.process_options options, :batch_name
@@ -37,20 +42,34 @@ export BATCH_SYSTEM=SLURM
37
42
 
38
43
  time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
39
44
 
45
+ sbatch_params = {"job-name" => batch_name,
46
+ "output" => fout,
47
+ "error" => ferr,
48
+ "cpus-per-task" => task_cpus,
49
+ "nodes" => nodes,
50
+ "time" => time,
51
+ "exclusive" => exclusive,
52
+ "licenses" => licenses,
53
+ "mem" => mem,
54
+ "mem-per-cpu" => mem_per_cpu,
55
+ }
56
+
40
57
  header =<<-EOF
41
58
  #!/bin/bash
42
- #SBATCH --job-name="#{batch_name}"
43
- #SBATCH --workdir="#{workdir}"
44
- #SBATCH --output="#{fout}"
45
- #SBATCH --error="#{ferr}"
46
- #SBATCH --qos="#{queue}"
47
- #SBATCH --cpus-per-task="#{task_cpus}"
48
- #SBATCH --time="#{time}"
49
- #SBATCH --nodes="#{nodes}"
50
59
  EOF
51
60
 
52
- header << "#SBATCH --exclusive" << "\n" if exclusive
53
- header << "#SBATCH --constraint=highmem" << "\n" if highmem
61
+ sbatch_params.each do |name,value|
62
+ next if value.nil? || value == ""
63
+ if TrueClass === value
64
+ header << "#SBATCH --#{name}" << "\n"
65
+ elsif Array === value
66
+ value.each do |v|
67
+ header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
68
+ end
69
+ else
70
+ header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
71
+ end
72
+ end
54
73
 
55
74
  header
56
75
  end
@@ -148,11 +148,7 @@ module Persist
148
148
  def write_lock
149
149
  write if closed?
150
150
  if write?
151
- begin
152
- return yield
153
- ensure
154
- close
155
- end
151
+ return yield
156
152
  end
157
153
 
158
154
  lock do
data/lib/rbbt/resource.rb CHANGED
@@ -4,10 +4,10 @@ require 'rbbt/resource/path'
4
4
  require 'net/http'
5
5
  require 'set'
6
6
 
7
-
8
7
  module Resource
9
8
  class ResourceNotFound < RbbtException; end
10
9
 
10
+
11
11
  class << self
12
12
  attr_accessor :lock_dir
13
13
 
@@ -87,12 +87,21 @@ module Resource
87
87
  lock_filename = nil # it seems like this was locked already.
88
88
 
89
89
  Misc.lock lock_filename do
90
- uri = URI(url)
90
+ begin
91
+ uri = URI(url)
92
+
93
+ http = Net::HTTP.new(uri.host, uri.port)
91
94
 
92
- http = Net::HTTP.new(uri.host, uri.port)
93
- request = Net::HTTP::Get.new(uri.request_uri)
94
- timeout = 60 * 10
95
- Net::HTTP.start(uri.host, uri.port, :timeout => timeout, :read_timeout => timeout, :open_timeout => timeout) do |http|
95
+ if uri.scheme == "https"
96
+ http.use_ssl = true
97
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
98
+ http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
99
+ end
100
+
101
+ timeout = 60 * 10
102
+ http.read_timeout = timeout
103
+ http.open_timeout = timeout
104
+ request = Net::HTTP::Get.new(uri.request_uri)
96
105
  http.request request do |response|
97
106
  filename = if response["Content-Disposition"]
98
107
  response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
@@ -123,9 +132,11 @@ module Resource
123
132
  FileUtils.mv tmp_dir, final_path
124
133
  end
125
134
  else
126
- Open.open(location, :nocache => true) do |s|
127
- Misc.sensiblewrite(final_path, s)
128
- end
135
+ url = location
136
+ raise TryAgain
137
+ #Open.open(location, :nocache => true) do |s|
138
+ # Misc.sensiblewrite(final_path, s)
139
+ #end
129
140
  end
130
141
  when Net::HTTPInternalServerError
131
142
  @server_missing_resource_cache << url
@@ -134,6 +145,8 @@ module Resource
134
145
  raise "Response not understood: #{response.inspect}"
135
146
  end
136
147
  end
148
+ rescue TryAgain
149
+ retry
137
150
  end
138
151
  end
139
152
  rescue
@@ -313,7 +326,10 @@ url='#{url}'
313
326
  def identify(path)
314
327
  path = File.expand_path(path)
315
328
  resource ||= Rbbt
316
- (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
329
+ locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
330
+ locations -= [:current, "current"]
331
+ locations << :current
332
+ locations.uniq.each do |name|
317
333
  pattern = resource.search_paths[name]
318
334
  next if pattern.nil?
319
335
  pattern = pattern.sub('{PWD}', Dir.pwd)
data/lib/rbbt/tsv/csv.rb CHANGED
@@ -28,10 +28,10 @@ module TSV
28
28
  if Misc.is_filename?(obj)
29
29
  CSV.read obj, options
30
30
  else
31
- CSV.new obj, options
31
+ CSV.new obj, **options
32
32
  end
33
33
  else
34
- CSV.new obj, options
34
+ CSV.new obj, **options
35
35
  end
36
36
 
37
37
  tsv = if noheaders
data/lib/rbbt/util/cmd.rb CHANGED
@@ -192,25 +192,48 @@ module CMD
192
192
 
193
193
  ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail
194
194
 
195
- err_thread = Thread.new do
196
- while line = serr.gets
197
- bar.process(line) if bar
198
- sout.log = line
199
- Log.log "STDERR [#{pid}]: " + line, stderr
200
- end if Integer === stderr and log
201
- serr.close
195
+ if (Integer === stderr and log) || bar
196
+ err_thread = Thread.new do
197
+ while line = serr.gets
198
+ bar.process(line) if bar
199
+ sout.log = line
200
+ Log.log "STDERR [#{pid}]: " + line, stderr if log
201
+ end
202
+ serr.close
203
+ end
204
+ else
205
+ Misc.consume_stream(serr, true)
206
+ err_thread = nil
202
207
  end
203
208
 
204
209
  sout.threads = [in_thread, err_thread, wait_thr].compact
205
210
 
206
211
  sout
207
212
  else
208
- err = ""
209
- err_thread = Thread.new do
210
- while not serr.eof?
211
- err << serr.gets if Integer === stderr
213
+
214
+ if bar
215
+ err = ""
216
+ err_thread = Thread.new do
217
+ while not serr.eof?
218
+ line = serr.gets
219
+ bar.process(line)
220
+ err << line if Integer === stderr and log
221
+ end
222
+ serr.close
212
223
  end
213
- serr.close
224
+ elsif log and Integer === stderr
225
+ err = ""
226
+ err_thread = Thread.new do
227
+ while not serr.eof?
228
+ err << serr.gets
229
+ end
230
+ serr.close
231
+ end
232
+ else
233
+ Misc.consume_stream(serr, true)
234
+ #serr.close
235
+ err_thread = nil
236
+ err = ""
214
237
  end
215
238
 
216
239
  ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail
@@ -220,7 +243,11 @@ module CMD
220
243
 
221
244
  status = wait_thr.value
222
245
  if not status.success? and not no_fail
223
- raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
246
+ if !err.empty?
247
+ raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}.\n#{err}"
248
+ else
249
+ raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{status.exitstatus}"
250
+ end
224
251
  else
225
252
  Log.log err, stderr if Integer === stderr and log
226
253
  end
@@ -248,16 +275,21 @@ module CMD
248
275
  pid = io.pids.first
249
276
 
250
277
  line = "" if bar
278
+ starting = true
251
279
  while c = io.getc
252
- STDERR << c if Log.severity <= level
253
- line << c if bar
254
- if c == "\n"
255
- bar.process(line) if bar
280
+ if starting
256
281
  if pid
257
282
  Log.logn "STDOUT [#{pid}]: ", level
258
283
  else
259
284
  Log.logn "STDOUT: ", level
260
285
  end
286
+ starting = false
287
+ end
288
+ STDERR << c if Log.severity <= level
289
+ line << c if bar
290
+ if c == "\n"
291
+ bar.process(line) if bar
292
+ starting = true
261
293
  line = "" if bar
262
294
  end
263
295
  end
@@ -0,0 +1,118 @@
1
+ module Rbbt
2
+ def self.migrate_source_paths(path, resource = Rbbt, source = nil)
3
+ if source
4
+ lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
5
+ require 'rbbt-util'
6
+ path = "#{path}"
7
+ if Open.exists?(path)
8
+ path = #{resource.to_s}.identify(path)
9
+ else
10
+ path = Path.setup(path)
11
+ end
12
+ puts path
13
+ puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
14
+ EOF
15
+
16
+ [path, paths.collect{|p| [source, p] * ":"}, lpath]
17
+ else
18
+ if File.exists?(path)
19
+ path = resource.identify(path)
20
+ else
21
+ path = Path.setup(path)
22
+ end
23
+
24
+ [path, (path.directory? ? path.glob_all : path.find_all), path]
25
+ end
26
+ end
27
+
28
+ def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
29
+ if target
30
+ Misc.ssh_run(target, <<-EOF).split("\n").first
31
+ require 'rbbt-util'
32
+ path = "#{path}"
33
+ resource = #{resource.to_s}
34
+ search_path = "#{search_path}"
35
+ puts resource[path].find(search_path)
36
+ EOF
37
+ else
38
+ resource[path].find(search_path)
39
+ end
40
+ end
41
+
42
+ def self.migrate_files(real_paths, target, options = {})
43
+ excludes = %w(.save .crap .source tmp filecache open-remote)
44
+ excludes += (options[:exclude] || "").split(/,\s*/)
45
+ excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
46
+
47
+ other = options[:other] || []
48
+
49
+ test_str = options[:test] ? '-nv' : ''
50
+
51
+ real_paths.each do |source_path|
52
+ if File.directory?(source_path) || source_path =~ /\/$/
53
+ source_path += "/" unless source_path[-1] == "/"
54
+ target += "/" unless target[-1] == "/"
55
+ end
56
+
57
+ next if source_path == target
58
+
59
+ if options[:target]
60
+ CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
61
+ else
62
+ Open.mkdir File.dirname(target)
63
+ end
64
+
65
+ if options[:target]
66
+ target_path = [options[:target], "'" + target + "'"] * ":"
67
+ else
68
+ target_path = "'" + target + "'"
69
+ end
70
+
71
+ TmpFile.with_file do |tmp_files|
72
+ if options[:files]
73
+ Open.write(tmp_files, options[:files] * "\n")
74
+ files_from_str = "--files-from='#{tmp_files}'"
75
+ else
76
+ files_from_str = ""
77
+ end
78
+
79
+ cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
80
+
81
+
82
+ cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
83
+
84
+ if options[:print]
85
+ puts cmd
86
+ exit 0
87
+ else
88
+ CMD.cmd_log(cmd, :log => Log::INFO)
89
+
90
+ if options[:delete] && options[:files]
91
+ remove_files = options[:files].collect{|f| File.join(source_path, f) }
92
+ dirs = remove_files.select{|f| File.directory? f }
93
+ remove_files.each do |file|
94
+ next if dirs.include? file
95
+ Open.rm file
96
+ end
97
+ dirs.each do |dir|
98
+ FileUtils.rmdir dir if Dir.glob(dir).empty?
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+
106
+
107
+ def self.migrate(path, search_path, options = {})
108
+ search_path = 'user' if search_path.nil?
109
+
110
+ resource = Rbbt
111
+
112
+ path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
113
+
114
+ target = migrate_target_path(lpath, search_path, resource, options[:target])
115
+
116
+ migrate_files(real_paths, target, options)
117
+ end
118
+ end