rbbt-util 5.28.8 → 5.28.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d77757da0912cd41138ff33b99e19b7ffa1449be515fb7a8742f9d69c134ffd
-  data.tar.gz: 1edb139f92881591bbac18692dc6a0a739be9dfe0a820a4db2cffd30079fc328
+  metadata.gz: 2fa2f71df9aef8810599ab3660c847c084dea75ccb2485d294cfa1962e341162
+  data.tar.gz: 7eb004ddf8bcf30b00325dc5251d9ed73a307de433c18c0847eb4d70a4bea1e2
 SHA512:
-  metadata.gz: 2219bd7152eda55f2c3c9b1e5d10d9593dd29232e7e77fe0707496031030455663983256930f35c2bcabd8b80e15c05d99f3bcdc9644135ee4a75d0267334c33
-  data.tar.gz: e4021aab6574fc10e960eec29f630075c8e95b27d6648ba47e1f76ecbf1a12b84a5859ad7b61ffde8591466ddcfec8ee5cfd27f6f82997648d712225ac099e1b
+  metadata.gz: 6ba23322f9ac768f1782c4e26605c50171b6d35d1d3c2acc0ef195e26ca39af851a232e1581ca2ec220899b7780f35f8d0d36e1f61f0b0e6500c883999c3760b
+  data.tar.gz: afa0952f228a951ea78506ad0776fd96dbfbd2b91bac73310ee6807e4a7d0a74c7ca8f582880c5abd28bbf709c8be3d9dff1797056be756962091b2e97031f30

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -12,6 +12,7 @@ module Workflow
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
+        workload[job].uniq!
       end
 
       job.input_dependencies.each do |dep|
@@ -19,6 +20,7 @@ module Workflow
         workload.merge!(job_workload(dep))
         workload[job] += workload[dep]
         workload[job] << dep
+        workload[job].uniq!
       end
 
       workload
@@ -32,7 +34,7 @@ module Workflow
       return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
 
       job_rules = IndiferentHash.setup(rules[workflow][task_name])
-      rules["defaults"].each{|k,v| job_rules[k] ||= v } if rules["defaults"]
+      rules["defaults"].each{|k,v| job_rules[k] = v if job_rules[k].nil? } if rules["defaults"]
       job_rules
     end
 
@@ -97,6 +99,7 @@ module Workflow
 
     def release_resources(job)
       if resources_used[job]
+        Log.debug "Orchestrator releasing resouces from #{job.path}"
         resources_used[job].each do |resource,value| 
           next if resource == 'size'
           resources_requested[resource] -= value.to_i
@@ -140,30 +143,51 @@ module Workflow
       end
     end
 
+    def erase_job_dependencies(job, rules, workload, top_level_jobs)
+      job.dependencies.each do |dep|
+        next if top_level_jobs.include? dep.path
+        next unless Orchestrator.job_rules(rules, dep)["erase"].to_s == 'true'
+
+        list = (workload.keys - [job]).collect{|pending| pending.dependencies}.flatten
+        next if list.include?(dep)
+
+        Log.high "Erasing #{dep.path} from #{job.path}"
+        job.archive_deps
+        job.copy_files_dir
+        job.dependencies = job.dependencies - [dep]
+        dep.clean
+      end
+    end
+
     def process(rules, jobs)
       begin
 
         workload = jobs.inject({}){|acc,job| acc.merge!(Orchestrator.job_workload(job)) }
 
-        while workload.values.flatten.any?
+        top_level_jobs = jobs.collect{|job| job.path }
+        while workload.any? 
 
           candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
-          raise "No candidates" if candidates.empty?
+          raise "No candidates and no running jobs" if candidates.empty?
 
           candidates.each do |job|
             case 
             when (job.error? || job.aborted?)
-              if job.recoverable_error?
-                job.clean
-                raise TryAgain
-              else
-                next
+              begin
+                if job.recoverable_error?
+                  job.clean
+                  raise TryAgain
+                else
+                  next
+                end
+              ensure
+                Log.warn "Releases resources from failed job: #{job.path}"
+                release_resources(job)
               end
-              release_resources(job)
             when job.done?
               Log.debug "Orchestrator done #{job.path}"
               release_resources(job)
-              raise TryAgain
+              erase_job_dependencies(job, rules, workload, top_level_jobs)
 
             when job.running?
               next
@@ -180,6 +204,7 @@ module Workflow
             next if k.done?
             new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
           end
+          workload = new_workload
           sleep timer
         end
       rescue TryAgain

data/test/rbbt/workflow/util/test_orchestrator.rb

@@ -17,6 +17,7 @@ module TestWF
     sleep(TestWF::MULT * (rand(10) + 2))
   end
   
+  dep :a
   dep :b
   task :c => :text do
     sleep(TestWF::MULT * (rand(10) + 2))
@@ -29,13 +30,13 @@ module TestWF
 end
 
 class TestClass < Test::Unit::TestCase
-  def _test_orchestrate
+  def test_orchestrate_resources
 
     jobs =[]
 
     num = 10
     num.times do |i|
-      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
     jobs.each do |j| j.recursive_clean end
 
@@ -81,13 +82,13 @@ TestWF:
     assert Misc.mean(second_cpus.values) < 30
   end
 
-  def test_orchestrate_size
+  def test_orchestrate_erase
 
     jobs =[]
 
     num = 10
     num.times do |i|
-      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
     jobs.each do |j| j.recursive_clean end
 
@@ -98,9 +99,11 @@ default_resources:
   IO: 1
 TestWF:
   a:
+    erase: true
     resources:
       cpus: 7
   b:
+    erase: true
     resources:
       cpus: 2
   c:
@@ -112,25 +115,109 @@ TestWF:
     EOF
 
     orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 0 do
+    Log.with_severity 3 do
       orchestrator.process(rules, jobs)
     end
 
-    data = Workflow.trace jobs, :plot_data => true
-    eend = data.column("End.second").values.collect{|v| v.to_f}.max
-    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
-    (0..eend.to_i).each do |second|
-      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
-      cpus = 0
-      tasks.through :key, ["Workflow", "Task"] do |k, values|
-        workflow, task = values
-        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
-      end
-      second_cpus[second] = cpus
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_default
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    erase: false
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_top_level
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
     end
 
-    assert Misc.mean(second_cpus.values) > 15
-    assert Misc.mean(second_cpus.values) < 30
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.8
+  version: 5.28.9
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-02 00:00:00.000000000 Z
+date: 2020-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake