rbbt-util 5.34.24 → 5.34.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7359f3a8ce8bde23107f23515557f47eb7bb65ee094c64419ba5542378af1c8c
-  data.tar.gz: 0a030eb0b6f9648a60c11abaac383fe33b24845c288ddd7a28eb8e9aa8d8ac1c
+  metadata.gz: da3b496fd0fb7d9ff645cba0fb4ba3914288fb0b88bf80f43a79d02bd1ee2cfd
+  data.tar.gz: ab120f9bb96f1b0628f1edc6dc114a43c32ed39a5c29ce5252bbf85383d24359
 SHA512:
-  metadata.gz: dce1b81193d5d0d7ad86140e7287228b4a9d83fe99f08c4a431e34491d3c2a6057054cc9d80a76be9d4ac34b422c8203470d53579c4ba84bee8a804b6f12a2dd
-  data.tar.gz: 5b1d4e2aa040b183a92ea7b3ccb45fef2e3467a7d94203fd5214a366cbc87bfd6944095888f68c0ca6c8da6d2442cd0f6685b381e4be2e5b68c07bc68365ae72
+  metadata.gz: 3f5b5326a834f1f972756919ea2f04c9626998be3c9ca058fd386c06787011ad021798e6091eb8eae1f7873dcceb9adcfa02de14a5269dadef4a2902d5475a9d
+  data.tar.gz: d74344affb899a6b62a494d02e9058dc338a7be4138b9ce32331d0657bc121035b312da21f48827751dc13e1bdee4511fb6f1eeaf481d39f73e5967ff3243800

data/lib/rbbt/hpc/batch.rb

@@ -101,6 +101,11 @@ module HPC
 
       task = Symbol === job.overriden ? job.overriden : job.task_name
 
+      #override_deps = job.overriden_deps.collect do |dep| 
+      #  name = [dep.workflow.to_s, dep.task_name] * "#"
+      #  [name, dep.path] * "="  
+      #end.uniq * ","
+
       if job.overriden?
         #override_deps = job.rec_dependencies.
         #  select{|dep| Symbol === dep.overriden }.
@@ -568,7 +573,7 @@ env > #{batch_options[:fenv]}
         Misc.add_defaults options, 
           :batch_dir => batch_dir, 
           :inputs_dir => File.join(batch_dir, "inputs_dir"),
-          :workflows => workflows_to_load.uniq * ","
+          :workflows => workflows_to_load.any? ? workflows_to_load.uniq * "," : nil
 
         options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
 

data/lib/rbbt/workflow/dependencies.rb

@@ -72,10 +72,7 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
 
-    if not Step === dep
-      located = Open.exists?(dep)
-      dep = Workflow.load_step(dep)
-    end
+    dep = Workflow.load_step(dep) if not Step === dep
 
     dep.original_workflow ||= dep.workflow if dep.workflow
     dep.original_task_name ||= dep.task_name if dep.task_name
@@ -86,7 +83,6 @@ module Workflow
     dep.workflow = workflow
     dep.info[:name] = dep.name
 
-
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
@@ -95,13 +91,17 @@ module Workflow
     end
 
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name && located
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name 
 
     dep.extend step_module
 
     dep
   end
 
+  def unlocated_override?(dep)
+    String === dep && ! (Open.exists?(dep) || Open.exists?(dep + '.info'))
+  end
+
   def assign_dep_inputs(_inputs, options, all_d, task_info)
     IndiferentHash.setup(_inputs)
 
@@ -153,6 +153,7 @@ module Workflow
 
     override_dependencies = override_dependencies(inputs)
 
+    overriden = false
     dependencies.each do |dependency|
       _inputs = IndiferentHash.setup(inputs.dup)
       jobname = orig_jobname
@@ -163,6 +164,7 @@ module Workflow
                    workflow, dep_task, options = dependency
 
                    if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
+                     overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
                      setup_override_dependency(value, workflow, dep_task)
                    else
 
@@ -175,15 +177,23 @@ module Workflow
 
                      job = workflow._job(dep_task, jobname, _inputs)
                      ComputeDependency.setup(job, compute) if compute
+
+                     overriden = true if TrueClass === job.overriden && (options.nil? || ! options[:not_overriden])
+
                      job
                    end
                  when Step
-                   dependency
+                   job = dependency
+                   overriden = true if TrueClass === job.overriden && (options.nil? || ! options[:not_overriden])
+                   job
                  when Symbol
                    if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
+                     overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
                      setup_override_dependency(value, self, dependency)
                    else
-                     _job(dependency, jobname, _inputs)
+                     job = _job(dependency, jobname, _inputs)
+                     overriden = true if TrueClass === job.overriden && (options.nil? || ! options[:not_overriden])
+                     job
                    end
                  when Proc
                    if DependencyBlock === dependency
@@ -191,6 +201,7 @@ module Workflow
                      wf, task_name, options = orig_dep
 
                      if override_dependencies[wf.to_s] && value = override_dependencies[wf.to_s][task_name]
+                       overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
                        dep = setup_override_dependency(value, wf, task_name)
                      else
 
@@ -206,16 +217,20 @@ module Workflow
                        dep.each{|d| 
                          next if d.nil?
                          if Hash === d
+                           d = d.merge(options)
                            d[:workflow] ||= wf 
                            d[:task] ||= task_name
                            _override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
                            d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
+                                 overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
                                  setup_override_dependency(value, d[:workflow], d[:task])
                                else
                                  task_info = d[:workflow].task_info(d[:task])
 
                                  _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                                 d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                                 job = d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                                 overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
+                                 job
                                end
                          end
                          ComputeDependency.setup(d, compute) if compute
@@ -230,11 +245,15 @@ module Workflow
                        dep[:workflow] ||= wf || self
                        _override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
                        if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
+                         overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
                          setup_override_dependency(value, dep[:workflow], dep[:task])
                        else
                          task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
                          _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
+                         job = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
+                         job = d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                         overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
+                         job
                        end
                      end
                    end
@@ -246,6 +265,6 @@ module Workflow
 
       real_dependencies << real_dep
     end
-    real_dependencies.flatten.compact
+    [real_dependencies.flatten.compact, overriden]
   end
 end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -526,8 +526,9 @@ class Step
   end
 
   def overriden?
+    return @overriden
     return true if @overriden
-    return true if dependencies && dependencies.select{|dep| dep.overriden? }.any?
+    return true if dependencies && dependencies.select{|dep| TrueClass === dep.overriden }.any?
     info[:archived_info].each do |f,i|
       next if Symbol === i
       return true if i[:overriden] || i["overriden"]

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -31,6 +31,8 @@ module Workflow
 
         type = :step_array if file.split(".").last == 'as_step_array'
 
+        type = :number_array if file.split(".").last == 'as_number_array'
+
         type = :step_file if file.split(".").last == 'as_step_file'
 
         type = :step_file_array if file.split(".").last == 'as_step_file_array'
@@ -57,6 +59,9 @@ module Workflow
         when :step_array
           steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
           inputs[input.to_sym] = steps
+        when :number_array
+          numbers = Open.read(file).strip.split("\n").collect{|num| num.to_f }
+          inputs[input.to_sym] = numbers
         when :step
           steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
           inputs[input.to_sym] = steps.first
@@ -176,6 +181,8 @@ class Step
       when Step
         path = path + '.as_step_array'
         value = value.collect{|s| s.path }
+      when Numeric
+        path = path + '.as_number_array'
       end
 
       value = value * "\n"

data/lib/rbbt/workflow.rb

@@ -285,6 +285,7 @@ module Workflow
 
     workdir[taskname][tagged_jobname].find
   end
+
   def import_task(workflow, orig, new)
     orig_task = workflow.tasks[orig]
     new_task = orig_task.dup
@@ -437,15 +438,13 @@ module Workflow
       jobname_input = nil 
     end
 
-    if jobname_input && jobname && inputs[jobname_input].nil?
-      inputs[jobname_input] = jobname
+    if jobname_input && inputs[jobname_input].nil?
+      inputs[jobname_input] = jobname || DEFAULT_NAME
     end
 
     real_inputs = {}
-    has_overriden_inputs = false
 
     inputs.each do |k,v|
-      #has_overriden_inputs = true if String === k and k.include? "#"
       next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
       default = all_defaults[k]
       next if default == v 
@@ -461,12 +460,10 @@ module Workflow
 
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
 
-    dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
-
-    overriden_deps = dependencies.select{|d| d.overriden }
-    true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
+    dependencies, overriden = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
 
-    overriden = has_overriden_inputs || overriden_deps.any?
+    #overriden_deps        = dependencies.select{|d| Symbol === d.overriden }
+    #overriden_target_deps = overriden_deps.select{|d| TrueClass === d.overriden }
 
     extension = task.extension
 
@@ -492,19 +489,20 @@ module Workflow
       step_path = step_path taskname, jobname, input_values, dependencies, extension
     end
 
-
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
+    job.overriden ||= overriden 
     job.clean_name = jobname
 
-    case not_overriden 
-    when TrueClass
-      job.overriden = has_overriden_inputs || true_overriden_deps.any?
-    when :not_overriden_dep
-      job.overriden = true if has_overriden_inputs || true_overriden_deps.any?
-    else
-      job.overriden = true if has_overriden_inputs || overriden_deps.any?
-    end
+    #iif [job, not_overriden]
+    #case not_overriden 
+    #when TrueClass
+    #  job.overriden = false
+    #when :not_overriden_dep
+    #  job.overriden = true if overriden_target_deps.any?
+    #else
+    #  job.overriden = true if overriden_deps.any?
+    #end
 
     job.real_inputs = real_inputs.keys
     job

data/share/rbbt_commands/hpc/orchestrate

@@ -61,7 +61,8 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
         else

data/share/rbbt_commands/lsf/orchestrate

@@ -61,7 +61,8 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
         else

data/share/rbbt_commands/slurm/orchestrate

@@ -61,7 +61,8 @@ class Step
     else
       begin
         Log.debug "Issuing SLURM job for #{self.path}"
-        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        #@batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+        @batch_job_id, @batch_job_dir = HPC::BATCH_MODULE.orchestrate_job(self, $slurm_options) 
         if no_load
           self
         else

data/test/rbbt/test_workflow.rb

@@ -166,6 +166,33 @@ for this dependency
     dep = dependencies.flatten.first
   end
 
+  task :overr_orig => :string do
+    "over"
+  end
+
+  task :overr_alt => :string do
+    "alt"
+  end
+
+  dep :overr_orig
+  task :overr_target => :string do
+    step(:overr_orig).load.reverse
+  end
+
+  dep :overr_alt, :not_overriden => true
+  dep :overr_target, "TestWF#overr_orig" => :overr_alt, :not_overriden => true
+  task :overr_action => :string do
+    step(:overr_target).load.upcase
+  end
+
+
+  dep :overr_alt, :not_overriden => true
+  dep :overr_target, "TestWF#overr_orig" => :overr_alt
+  task :overr_action2 => :string do
+    step(:overr_target).load.upcase
+  end
+
+
 end
 
 TestWF.workdir = Rbbt.tmp.test.jobs.TestWF
@@ -419,7 +446,7 @@ class TestWorkflow < Test::Unit::TestCase
       TmpFile.with_file do |dir|
         Path.setup(dir)
         Step.save_job_inputs(job, dir)
-        assert_equal Dir.glob(dir + "/*"), [dir.file.find]
+        assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.as_path']
       end
     end
 
@@ -427,7 +454,7 @@ class TestWorkflow < Test::Unit::TestCase
     TmpFile.with_file do |dir|
       Path.setup(dir)
       Step.save_job_inputs(job, dir)
-      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.yaml']
+      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.as_path']
       inputs  = Workflow.load_inputs(dir, [:file], :file => :file)
       assert_equal inputs, {:file => 'code'}
     end
@@ -472,4 +499,18 @@ class TestWorkflow < Test::Unit::TestCase
       end
     end
   end
+
+  def test_overriden
+    job = TestWF.job(:overr_action)
+    job.recursive_clean
+    assert_equal "TLA", job.run
+
+    assert Symbol === job.step(:overr_orig).overriden
+    assert TrueClass === job.step(:overr_target).overriden
+    assert ! job.overriden
+
+    job = TestWF.job(:overr_action2)
+    assert_equal "TLA", job.run
+    assert job.overriden
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.24
+  version: 5.34.26
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-09 00:00:00.000000000 Z
+date: 2022-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake