rbbt-util 5.30.9 → 5.30.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8985df2e0e2353224b5b075acb1bb04243d12351b8d904441009b93ccf490288
-  data.tar.gz: 3a1e7d8561fcc72a83c2b0006c49cf50f35a87cfa037c0186bdc3563435a0e8f
+  metadata.gz: 48bec783cd5a61ffb37cb7cd80379ed9c90bbcfd2cecdb756cff0f2efd6240ed
+  data.tar.gz: 9da9e93fc5c89be3ca8f57f87ca2808f298e6a300cd13dfff4dd602a2bf1b4b0
 SHA512:
-  metadata.gz: baae9121f3d757bcda2add9f666c740e5647f38d4bbc511d3456e36d466fc443c8a04b0ae12431bbf226e4e0c75eec5700b28720c117954005fe836ad74b76e8
-  data.tar.gz: ea61b36cec82f6634546aecf5de356d7d3b3c9d28eccfad52282f309e5ad6e1e28c2bfb627a100970d6a553fdc90b3900a4247101210cfbaae461c35d8e0399a
+  metadata.gz: 23976e1fcc0a216a822fa19ec9f4510d87f5d19d334245f6cea87746d3b7b85cf3c41110ce2d36f3a793f6e5af1bd08ad2299ad0acb08f11efe711f3417da590
+  data.tar.gz: 2fe9f8dbea015d105076926f0407c1029a400f6eafce7f17edca075e4975ef5a4bd455f416cad4e2d4deadb2d7486988c00500687fc41352020ba71fa07a0f0c

data/lib/rbbt/hpc/orchestrate.rb

@@ -67,6 +67,7 @@ module HPC
     def self.piggyback(job, job_rules, job_deps)
       return false unless job_rules["skip"]
       final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
+      final_deps = final_deps.reject{|dep| dep.done? && dep.status == :noinfo }
       return final_deps.first if final_deps.length == 1
       return false
     end
@@ -104,19 +105,19 @@ module HPC
     def self.workload(job, rules, chains, options, seen = nil)
       return [] if job.done?
       if seen.nil?
-        seen = {} 
+        seen = {}
         target_job = true
       end
 
       job_rules = self.job_rules(rules, job)
       job_deps = get_job_dependencies(job)
 
-
       chain = chains[job]
-      chain -= seen.keys if chain
+      chain = chain.reject{|j| seen.include? j.path} if chain
+      chain = chain.reject{|dep| dep.done? } if chain
       piggyback = piggyback(job, job_rules, job_deps)
       dep_ids = job_deps.collect do |dep|
-        seen[dep] = nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
+        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
         next_options = IndiferentHash.setup(options.dup)
         if piggyback and piggyback == dep
           next_options[:piggyback] ||= []
@@ -129,12 +130,15 @@ module HPC
 
         ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
 
-        seen[dep] = ids
+        seen[dep.path] = ids
         ids
       end.compact.flatten.uniq
 
-      return seen[job] || dep_ids if seen.include?(job) 
-      return seen[piggyback] if piggyback
+      return seen[job.path] || dep_ids if seen.include?(job.path)
+
+      if piggyback and seen[piggyback.path]
+        return seen[job.path] = seen[piggyback.path] 
+      end
 
       job_rules.delete :chain_tasks
       job_rules.delete :tasks
@@ -172,7 +176,7 @@ module HPC
 
       manifest.uniq!
 
-      job_options[:manifest] = manifest.collect{|j| j.workflow_short_path }
+      job_options[:manifest] = manifest.collect{|j| j.task_signature }
 
       job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
 

data/lib/rbbt/util/config.rb

@@ -90,6 +90,8 @@ module Rbbt::Config
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]
 
+    tokens = ["key:" + key] if tokens.empty?
+
     tokens = tokens.flatten
     file, _sep, line = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
@@ -106,7 +108,6 @@ module Rbbt::Config
 
     entries = CACHE[key.to_s]
     priorities = {}
-    tokens = tokens + ["key:" << key.to_s]
     tokens.each do |token|
       token_prio = match entries, token.to_s
       token_prio.each do |prio, values|
@@ -145,7 +146,6 @@ module Rbbt::Config
       Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
     else
       key, value, *tokens = config.split(/\s/)
-      tokens = ['key:' << key << '::0'] if tokens.empty?
       tokens = tokens.collect do |tok|
         tok, _sep, prio = tok.partition("::")
         prio = "0" if prio.nil? or prio.empty?

data/lib/rbbt/workflow/definition.rb

@@ -79,9 +79,10 @@ module Workflow
       dep = dependencies.last.join
       raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, :remove_dep_tasks, :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+
         self.archive_deps
         self.copy_files_dir
         self.dependencies = self.dependencies - [dep]
@@ -92,7 +93,10 @@ module Workflow
         when 'true'
           dep.clean
         when 'recursive'
-          dep.recursive_clean
+          dep.rec_dependencies.each do |d|
+            d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+          end
+          dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
         end
       else
         if Open.exists?(dep.files_dir)

data/lib/rbbt/workflow/util/archive.rb

@@ -51,7 +51,7 @@ class Step
     end
   end
 
-  def self.job_files_for_archive(files, recursive = false)
+  def self.job_files_for_archive(files, recursive = false, skip_overriden = false)
     job_files = Set.new
 
     jobs = files.collect do |file|  
@@ -65,6 +65,8 @@ class Step
 
     jobs.each do |step|
       next unless File.exists?(step.path)
+      next if skip_overriden && step.overriden
+
       job_files << step.path
       job_files << step.info_file if File.exists?(step.info_file)
       job_files << Step.md5_file(step.path) if File.exists?(Step.md5_file step.path)
@@ -258,9 +260,9 @@ puts resource[path].find(search_path)
     end
   end
 
-  def self.purge(path, recursive = false)
+  def self.purge(path, recursive = false, skip_overriden = true)
     path = [path] if String === path
-    job_files = job_files_for_archive path, recursive
+    job_files = job_files_for_archive path, recursive, skip_overriden
 
     job_files.each do |file|
       begin

data/test/rbbt/util/test_config.rb

@@ -5,8 +5,9 @@ class TestConfig < Test::Unit::TestCase
   def setup
     Rbbt::Config.set({:cpus => 30}, :test_config, :test)
     Rbbt::Config.set(:cpus, 5, "slow::2", :test)
-    Rbbt::Config.set({:token => "token"}, "token", "key:token")
+    Rbbt::Config.set({:token => "token"}, "token")
     Rbbt::Config.set(:notoken, "no_token")
+    Rbbt::Config.set({:emptytoken => "empty"})
   end
 
   def test_simple
@@ -19,8 +20,12 @@ class TestConfig < Test::Unit::TestCase
 
   def test_simple_no_token
     assert_equal "token", Rbbt::Config.get("token", "token")
-    assert_equal "token", Rbbt::Config.get("token")
-    assert_equal "no_token", Rbbt::Config.get("notoken")
+    assert_equal "no_token", Rbbt::Config.get("notoken", "key:notoken")
+    assert_equal 'token', Rbbt::Config.get("token", "key:token")
+    assert_equal 'token', Rbbt::Config.get("token")
+    assert_equal nil, Rbbt::Config.get("token", "someotherthing")
+    assert_equal "default_token", Rbbt::Config.get("token", 'unknown', :default => 'default_token')
+    assert_equal 'empty', Rbbt::Config.get("emptytoken", 'key:emptytoken')
   end
 
   def test_prio
@@ -54,6 +59,11 @@ class TestConfig < Test::Unit::TestCase
     assert_equal "V1", Rbbt::Config.get('key', 'token1', 'token2')
   end
 
+  def test_default
+    Rbbt::Config.add_entry 'key', 'V1', 'token1'
+    assert_equal "V3", Rbbt::Config.get('key', 'token2', :default => 'V3')
+  end
+
 
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.9
+  version: 5.30.10
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake