rbbt-util 5.34.26 → 5.34.27

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: da3b496fd0fb7d9ff645cba0fb4ba3914288fb0b88bf80f43a79d02bd1ee2cfd
4
- data.tar.gz: ab120f9bb96f1b0628f1edc6dc114a43c32ed39a5c29ce5252bbf85383d24359
3
+ metadata.gz: 40a3d86f01180255239e05fc23309b2c798eb94ee4c213ab8699f7954c1f5346
4
+ data.tar.gz: 3357e23002f032213a6b4c01ef046527092c8e4bf9ebce96dfc714176c3cb239
5
5
  SHA512:
6
- metadata.gz: 3f5b5326a834f1f972756919ea2f04c9626998be3c9ca058fd386c06787011ad021798e6091eb8eae1f7873dcceb9adcfa02de14a5269dadef4a2902d5475a9d
7
- data.tar.gz: d74344affb899a6b62a494d02e9058dc338a7be4138b9ce32331d0657bc121035b312da21f48827751dc13e1bdee4511fb6f1eeaf481d39f73e5967ff3243800
6
+ metadata.gz: 8454ec0e14dac9fbdb5fa00a2219248b5037c890289fb5b013d69d6dac63e869b5160620d1ac744392546bf58dc8acfb1afba5c9172f366ddae01fc6cc008810
7
+ data.tar.gz: c85e5b6defde582d6af2c08281dd659bd8f71ae9867d4258d2c96c9824b3d0fe7020472fc6ccec3e66f4f70228bdc3dea425756abd510ded61a7f59420b78b98
@@ -560,6 +560,16 @@ env > #{batch_options[:fenv]}
560
560
  def run_job(job, options = {})
561
561
  system = self.to_s.split("::").last
562
562
 
563
+ if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
564
+ Log.info "Job #{job.short_path} already queued in #{batch_job}"
565
+ return batch_job
566
+ end
567
+
568
+ if job.running?
569
+ Log.info "Job #{job.short_path} already running in #{job.info[:pid]}"
570
+ return job.info[:batch_job]
571
+ end
572
+
563
573
  batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options,
564
574
  :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
565
575
  :batch_base_dir => File.expand_path(File.join('~/rbbt-batch'))
@@ -583,6 +593,8 @@ env > #{batch_options[:fenv]}
583
593
 
584
594
  batch_job = run_template(batch_dir, dry_run)
585
595
 
596
+ hold_dependencies(job, batch_job) unless dry_run
597
+
586
598
  return [batch_job, batch_dir] unless tail
587
599
 
588
600
  t_monitor = Thread.new do
@@ -603,6 +615,17 @@ env > #{batch_options[:fenv]}
603
615
  end
604
616
  end
605
617
 
618
+ def hold_dependencies(job, batch_job)
619
+ job.set_info :batch_job, batch_job
620
+ job.set_info :batch_system, self.batch_system
621
+ job.dependencies.each do |dep|
622
+ next unless dep.waiting?
623
+ next if (dep_batch_job = dep.info[:batch_job]) && job_queued(dep_batch_job)
624
+
625
+ hold_dependencies(dep, batch_job)
626
+ end
627
+ end
628
+
606
629
  def follow_job(batch_dir, tail = true)
607
630
  fjob = File.join(batch_dir, 'job.id')
608
631
  fout = File.join(batch_dir, 'std.out')
@@ -672,7 +695,7 @@ env > #{batch_options[:fenv]}
672
695
  terr = Misc.consume_stream(err, true, STDERR) if err
673
696
  tout = Misc.consume_stream(out, true, STDOUT) if out
674
697
 
675
- sleep 3 while job_status(job).include? job.to_s
698
+ sleep 3 while job_queued(job)
676
699
  rescue Aborted
677
700
  ensure
678
701
  begin
@@ -692,6 +715,10 @@ env > #{batch_options[:fenv]}
692
715
  end
693
716
  end
694
717
 
718
+ def job_queued(job)
719
+ job_status(job).split(/\s+/).include?(job.to_s)
720
+ end
721
+
695
722
  def wait_for_job(batch_dir, time = 1)
696
723
  fexit = File.join(batch_dir, 'exit.status')
697
724
  fjob = File.join(batch_dir, 'job.id')
data/lib/rbbt/hpc/lsf.rb CHANGED
@@ -5,6 +5,10 @@ module HPC
5
5
  extend HPC::TemplateGeneration
6
6
  extend HPC::Orchestration
7
7
 
8
+ def self.batch_system
9
+ "LSF"
10
+ end
11
+
8
12
  def self.batch_system_variables
9
13
  <<-EOF
10
14
  let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
@@ -13,7 +17,7 @@ let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") /
13
17
  export MAX_MEMORY_DEFAULT
14
18
  export MAX_MEMORY
15
19
  export BATCH_JOB_ID=$LSF_JOBID
16
- export BATCH_SYSTEM=LSF
20
+ export BATCH_SYSTEM=#{batch_system}
17
21
  EOF
18
22
  end
19
23
 
@@ -206,7 +206,7 @@ module HPC
206
206
  options.delete "detach"
207
207
  options.delete "jobname"
208
208
 
209
- rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
209
+ rules = Misc.load_yaml(options[:orchestration_rules]) if options[:orchestration_rules]
210
210
  rules ||= {}
211
211
  IndiferentHash.setup(rules)
212
212
 
@@ -37,9 +37,9 @@ module HPC
37
37
  prepare_for_execution(job)
38
38
 
39
39
  if options[:orchestration_rules]
40
- rules = YAML.load(Open.read(options[:orchestration_rules]))
40
+ rules = Misc.load_yaml(options[:orchestration_rules])
41
41
  elsif Rbbt.etc.slurm["default.yaml"].exists?
42
- rules = YAML.load(Open.read(Rbbt.etc.slurm["default.yaml"]))
42
+ rules = Misc.load_yaml(Rbbt.etc.slurm["default.yaml"])
43
43
  else
44
44
  rules = {}
45
45
  end
@@ -6,6 +6,10 @@ module HPC
6
6
  extend HPC::TemplateGeneration
7
7
  extend HPC::Orchestration
8
8
 
9
+ def self.batch_system
10
+ "SLURM"
11
+ end
12
+
9
13
  def self.batch_system_variables
10
14
  <<-EOF
11
15
  let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
@@ -16,7 +20,7 @@ MAX_MEMORY="$MAX_MEMORY_DEFAULT"
16
20
  export MAX_MEMORY_DEFAULT
17
21
  export MAX_MEMORY
18
22
  export BATCH_JOB_ID=$SLURM_JOB_ID
19
- export BATCH_SYSTEM=SLURM
23
+ export BATCH_SYSTEM=#{batch_system}
20
24
  EOF
21
25
  end
22
26
 
@@ -145,7 +149,11 @@ export BATCH_SYSTEM=SLURM
145
149
  if job.nil?
146
150
  CMD.cmd("squeue").read
147
151
  else
148
- CMD.cmd("squeue --job #{job}").read
152
+ begin
153
+ CMD.cmd("squeue --job #{job}").read
154
+ rescue
155
+ ""
156
+ end
149
157
  end
150
158
  end
151
159
 
data/lib/rbbt/monitor.rb CHANGED
@@ -64,14 +64,14 @@ module Rbbt
64
64
  lock_info[f].merge!(file_time(f))
65
65
  if File.size(f) > 0
66
66
  info = Open.open(f) do |s|
67
- YAML.load(s)
67
+ Misc.load_yaml(s)
68
68
  end
69
69
  IndiferentHash.setup(info)
70
70
  lock_info[f][:pid] = info[:pid]
71
71
  lock_info[f][:ppid] = info[:ppid]
72
72
  end
73
73
  rescue Exception
74
- #Log.exception $!
74
+ Log.exception $!
75
75
  end
76
76
  end
77
77
  lock_info
@@ -272,7 +272,7 @@ module Rbbt
272
272
  def self.load_lock(lock)
273
273
  begin
274
274
  info = Misc.insist 3 do
275
- YAML.load(Open.read(lock))
275
+ Misc.load_yaml(lock)
276
276
  end
277
277
  info.values_at "pid", "ppid", "time"
278
278
  rescue Exception
data/lib/rbbt/persist.rb CHANGED
@@ -148,9 +148,7 @@ module Persist
148
148
  JSON.parse(stream.read)
149
149
  end
150
150
  when :yaml
151
- Open.open(path) do |stream|
152
- YAML.load(stream)
153
- end
151
+ Misc.load_yaml(path)
154
152
  when :float
155
153
  Open.read(path).to_f
156
154
  when :integer
@@ -150,7 +150,7 @@ module Path
150
150
  search_path_file = File.join(ENV['HOME'], '.rbbt/etc/search_paths')
151
151
  if File.exist?(search_path_file)
152
152
  begin
153
- YAML.load(File.open(search_path_file)).each do |where, location|
153
+ Misc.load_yaml(search_path_file).each do |where, location|
154
154
  SEARCH_PATHS[where.to_sym] = location
155
155
  end
156
156
  rescue
@@ -0,0 +1,26 @@
1
+ module Misc
2
+ def self.load_yaml(yaml)
3
+ case yaml
4
+ when IO, StringIO
5
+ if YAML.respond_to?(:unsafe_load)
6
+ YAML.unsafe_load(yaml)
7
+ else
8
+ YAML.load(yaml)
9
+ end
10
+ when (defined?(Path) && Path)
11
+ yaml.open do |io|
12
+ load_yaml(io)
13
+ end
14
+ when String
15
+ if Misc.is_filename?(yaml)
16
+ File.open(yaml) do |io|
17
+ load_yaml(io)
18
+ end
19
+ else
20
+ load_yaml(StringIO.new(yaml))
21
+ end
22
+ else
23
+ raise "Unkown YAML object: #{Misc.fingerprint yaml}"
24
+ end
25
+ end
26
+ end
@@ -22,6 +22,8 @@ require 'rbbt/util/misc/objects'
22
22
  require 'rbbt/util/misc/manipulation'
23
23
  require 'rbbt/util/misc/communication'
24
24
 
25
+ require 'rbbt/util/misc/serialize'
26
+
25
27
  require 'to_regexp'
26
28
 
27
29
  module MultipleResult; end
@@ -143,8 +143,12 @@ class Step
143
143
 
144
144
  def init_info(force = false)
145
145
  return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
146
+ batch_job = info[:batch_job] if Open.exists?(info_file)
147
+ batch_system = info[:batch_system] if Open.exists?(info_file)
146
148
  Open.lock(info_file, :lock => info_lock) do
147
149
  i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
150
+ i[:batch_job] = batch_job if batch_job
151
+ i[:batch_system] = batch_system if batch_system
148
152
  i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
149
153
  Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
150
154
  @info_cache = IndiferentHash.setup(i)
@@ -253,7 +257,7 @@ class Step
253
257
  #Open.read(file(name)).split /\n|,\s*/
254
258
  Open.read(file(name)).split "\n"
255
259
  when :yaml
256
- YAML.load(Open.open(file(name)))
260
+ Misc.load_yaml(file(name))
257
261
  when :marshal
258
262
  Marshal.load(Open.open(file(name)))
259
263
  else
@@ -698,7 +698,7 @@ class Step
698
698
  end
699
699
 
700
700
  def grace
701
- until done? || result || error? || aborted? || streaming? || waiting?
701
+ until done? || result || error? || aborted? || streaming? || waiting? || running?
702
702
  sleep 1
703
703
  end
704
704
  self
@@ -82,7 +82,7 @@ module Workflow
82
82
  when :file, :binary
83
83
  Log.debug "Pointing #{ input } to #{file}"
84
84
  if file =~ /\.yaml/
85
- inputs[input.to_sym] = YAML.load(Open.read(file))
85
+ inputs[input.to_sym] = Misc.load_yaml(file)
86
86
  else
87
87
  if File.symlink?(file)
88
88
  link_target = File.expand_path(File.readlink(file), File.dirname(file))
@@ -15,7 +15,7 @@ class Step
15
15
  raise "DO NOT CLEAN"
16
16
  end
17
17
 
18
- if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
18
+ if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir) or Open.exists?(pid_file)
19
19
 
20
20
  @result = nil
21
21
  @pid = nil
data/lib/rbbt/workflow.rb CHANGED
@@ -834,8 +834,7 @@ module Workflow
834
834
  end
835
835
 
836
836
  def self.load_remote_tasks(filename)
837
- yaml_text = Open.read(filename)
838
- remote_workflow_tasks = YAML.load(yaml_text)
837
+ remote_workflow_tasks = Misc.load_yaml(filename)
839
838
  Workflow.process_remote_tasks(remote_workflow_tasks)
840
839
  end
841
840
 
@@ -859,8 +858,7 @@ module Workflow
859
858
  end
860
859
 
861
860
  def self.load_relay_tasks(filename)
862
- yaml_text = Open.read(filename)
863
- relay_workflow_tasks = YAML.load(yaml_text)
861
+ remote_workflow_tasks = Misc.load_yaml(filename)
864
862
  Workflow.process_relay_tasks(relay_workflow_tasks)
865
863
  end
866
864
  end
@@ -46,7 +46,7 @@ if locks.any?
46
46
  puts
47
47
  puts Log.color(:magenta, "Locks:")
48
48
  locks.each do |file,info|
49
- if force or (info[:pid] and not Misc.pid_exists? info[:pid])
49
+ if force or (info[:pid] && ! Misc.pid_exists?(info[:pid]))
50
50
  puts " Removing #{ file }"
51
51
  File.unlink file
52
52
  end
@@ -0,0 +1,24 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
+ require 'rbbt/util/misc/serialize'
3
+
4
+ class TestClass < Test::Unit::TestCase
5
+ def test_load_yaml
6
+ yaml_txt=<<-EOF
7
+ ---
8
+ a: b
9
+ EOF
10
+ yaml_sio = StringIO.new yaml_txt
11
+
12
+ assert_equal "b", Misc.load_yaml(yaml_txt)["a"]
13
+ assert_equal "b", Misc.load_yaml(yaml_sio)["a"]
14
+
15
+ TmpFile.with_file yaml_txt do |yaml_file|
16
+ assert_equal "b", Misc.load_yaml(yaml_file)["a"]
17
+ Open.open(yaml_file) do |yaml_io|
18
+ assert_equal "b", Misc.load_yaml(yaml_io)["a"]
19
+ end
20
+ end
21
+
22
+ end
23
+ end
24
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.34.26
4
+ version: 5.34.27
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-12-14 00:00:00.000000000 Z
11
+ date: 2022-12-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -323,6 +323,7 @@ files:
323
323
  - lib/rbbt/util/misc/omics.rb
324
324
  - lib/rbbt/util/misc/options.rb
325
325
  - lib/rbbt/util/misc/pipes.rb
326
+ - lib/rbbt/util/misc/serialize.rb
326
327
  - lib/rbbt/util/misc/system.rb
327
328
  - lib/rbbt/util/named_array.rb
328
329
  - lib/rbbt/util/open.rb
@@ -559,6 +560,7 @@ files:
559
560
  - test/rbbt/util/misc/test_multipart_payload.rb
560
561
  - test/rbbt/util/misc/test_omics.rb
561
562
  - test/rbbt/util/misc/test_pipes.rb
563
+ - test/rbbt/util/misc/test_serialize.rb
562
564
  - test/rbbt/util/python/test_util.rb
563
565
  - test/rbbt/util/simpleopt/test_get.rb
564
566
  - test/rbbt/util/simpleopt/test_parse.rb
@@ -634,6 +636,7 @@ test_files:
634
636
  - test/rbbt/util/misc/test_development.rb
635
637
  - test/rbbt/util/misc/test_omics.rb
636
638
  - test/rbbt/util/misc/test_pipes.rb
639
+ - test/rbbt/util/misc/test_serialize.rb
637
640
  - test/rbbt/util/misc/test_format.rb
638
641
  - test/rbbt/util/misc/test_communication.rb
639
642
  - test/rbbt/util/misc/test_lock.rb