rbbt-util 5.32.18 → 5.32.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adee22216a6bbce0a4292f20e89f3c620fc47fcb28aaaf15082cd3ee54d2c093
-  data.tar.gz: 920f61f95dce1acbecc279ae09731dab297d0eb57cffe7a08734297e917436f8
+  metadata.gz: 900a7061d40ec2cbffd38886ef9629aa0c0ed6d6ee1aef6e0c95c51656230ef0
+  data.tar.gz: 13b965a0ec134a932c6ae0ae6a109fbc2bc83b7ffd855a1ef05ed4e1e51b9d88
 SHA512:
-  metadata.gz: f7fb0024e458db313ab31a46c4142092121d76efc5ca79c85876a8dc412e6483c12cf3058a1f9f5e6300a31ce672e87b60864eff46c2af8662df411e5680ac39
-  data.tar.gz: 4216a058f4329d25ed573129c0b40408921566218267922f1b3831108ab2745fc36279fdf7ecedd580d71c0163fae44a8d36adfc2223fecad76ac5864ce90c5f
+  metadata.gz: 3b890e2b6fb3675df4319d0541d9057a54b0cb37286a814822915f5b5b061d4f7e02deab0afdd7cd0833cb741aeded7454a9585f93ff3486e595b99db4da00f6
+  data.tar.gz: 105072ca6c7b5c43c6a60455eaa9c7167feeb93f505c61c6aeffe6305de3166a6cb440c262abf025407306ba82c2fe3167f5f9e743e086d99fa936e44ea8e448

data/lib/rbbt/hpc/slurm.rb

@@ -44,6 +44,7 @@ export BATCH_SYSTEM=SLURM
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
       sbatch_params = {"job-name" => batch_name,
+                       "qos" => queue,
                        "output" => fout,
                        "error" => ferr,
                        "cpus-per-task" => task_cpus,

data/lib/rbbt/tsv/parser.rb

@@ -133,7 +133,7 @@ module TSV
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| value.nil? ? [] : value.split(@sep2, -1) }
+               end.collect{|value| (value.empty? || value.nil?) ? [nil] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/util/config.rb

@@ -122,7 +122,12 @@ module Rbbt::Config
     Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
     GOT_KEYS << [key, value, tokens]
 
-    value
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    else
+      value
+    end
   end
 
   def self.with_config

data/lib/rbbt/workflow/accessor.rb

@@ -172,50 +172,6 @@ module Workflow
     }.tap{|h| IndiferentHash.setup(h)}
   end
 
-  def assign_dep_inputs(_inputs, options, all_d, task_info)
-    IndiferentHash.setup(_inputs)
-
-    options.each{|i,v|
-      next if i == :compute or i == "compute"
-      case v
-      when :compute
-        compute = v
-      when Symbol
-        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
-
-        if rec_dependency.nil?
-          if _inputs.include?(v)
-            #_inputs[i] = _inputs.delete(v)
-            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
-          else
-            _inputs[i] = v unless _inputs.include? i
-          end
-        else
-          input_options = task_info[:input_options][i] || {}
-
-          #ToDo why was this always true?
-          if input_options[:stream] or true
-            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
-            _inputs[i] = rec_dependency
-          else
-            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
-            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
-            if rec_dependency.streaming? and rec_dependency.running?
-              _inputs[i] = rec_dependency.join.load
-            else
-              rec_dependency.run(true)
-              rec_dependency.join
-              _inputs[i] = rec_dependency.load
-            end
-          end
-        end
-      else
-        _inputs[i] = v
-      end
-    } if options
-
-    _inputs
-  end
 
   def task_from_dep(dep)
     task = case dep

data/lib/rbbt/workflow/definition.rb

@@ -77,15 +77,15 @@ module Workflow
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
-      Step.wait_for_jobs dependencies
+      Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join
       raise dep.get_exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
         self.copy_files_dir
@@ -93,15 +93,17 @@ module Workflow
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
         Open.ln_h dep.path, self.tmp_path
+
         case remove.to_s
         when 'true'
           dep.clean
         when 'recursive'
-          dep.rec_dependencies.each do |d|
+          (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+            next if d.overriden
             d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
           end
           dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
-        end
+        end unless dep.overriden
       else
         if Open.exists?(dep.files_dir)
           Open.rm_rf self.files_dir 

data/lib/rbbt/workflow/dependencies.rb

@@ -71,7 +71,7 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
-    dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep = Step === dep ? dep.dup : Workflow.load_step(dep)
     dep.workflow = workflow
     dep.info[:name] = dep.name
     dep.original_task_name ||= dep.task_name if dep.workflow
@@ -83,14 +83,60 @@ module Workflow
     rescue
       Log.exception $!
     end
+
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
 
     dep.extend step_module
 
     dep
   end
 
+  def assign_dep_inputs(_inputs, options, all_d, task_info)
+    IndiferentHash.setup(_inputs)
+
+    options.each{|i,v|
+      next if i == :compute or i == "compute"
+      case v
+      when :compute
+        compute = v
+      when Symbol
+        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
+
+        if rec_dependency.nil?
+          if _inputs.include?(v)
+            #_inputs[i] = _inputs.delete(v)
+            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
+          else
+            _inputs[i] = v unless _inputs.include? i
+          end
+        else
+          input_options = task_info[:input_options][i] || {}
+
+          #ToDo why was this always true?
+          if input_options[:stream] or true
+            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
+            _inputs[i] = rec_dependency
+          else
+            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
+            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
+            if rec_dependency.streaming? and rec_dependency.running?
+              _inputs[i] = rec_dependency.join.load
+            else
+              rec_dependency.run(true)
+              rec_dependency.join
+              _inputs[i] = rec_dependency.load
+            end
+          end
+        end
+      else
+        _inputs[i] = v
+      end
+    } if options
+
+    _inputs
+  end
+
   def real_dependencies(task, orig_jobname, inputs, dependencies)
     real_dependencies = []
     path_deps = {}

data/lib/rbbt/workflow/step/dependencies.rb

@@ -206,7 +206,7 @@ class Step
     if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
-        other_steps = dep_step[step.path].uniq
+        other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
         return unless other_steps.length > 1
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
@@ -523,4 +523,36 @@ class Step
     set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
   end
 
+  #connected = true means that dependency searching ends when a result is done
+  #but dependencies are absent, meanining that the file could have been dropped
+  #in
+  def rec_dependencies(connected = false, seen = [])
+
+    # A step result with no info_file means that it was manually
+    # placed. In that case, do not consider its dependencies
+    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
+
+    return [] if dependencies.nil? or dependencies.empty?
+
+    new_dependencies = []
+    if self.overriden?
+      archived_deps = []
+    else
+      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+    end
+
+    dependencies.each{|step| 
+      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
+      next if archived_deps.include? step.path
+      next if seen.include? step.path
+      next if self.done? && connected && ! updatable?
+
+      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
+      new_dependencies.concat r
+      new_dependencies << step
+    }
+
+    new_dependencies.uniq
+  end
+
 end

data/lib/rbbt/workflow/step.rb

@@ -179,6 +179,10 @@ class Step
                      end
   end
 
+  def result_type=(type)
+    @result_type = type
+  end
+
   def result_description
     @result_description ||= if @task.nil?
                        info[:result_description]
@@ -351,38 +355,6 @@ class Step
     end
   end
 
-  #connected = true means that dependency searching ends when a result is done
-  #but dependencies are absent, meanining that the file could have been dropped
-  #in
-  def rec_dependencies(connected = false, seen = [])
-
-    # A step result with no info_file means that it was manually
-    # placed. In that case, do not consider its dependencies
-    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
-
-    return [] if dependencies.nil? or dependencies.empty?
-
-    new_dependencies = []
-    if self.overriden?
-      archived_deps = []
-    else
-      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
-    end
-
-    dependencies.each{|step| 
-      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
-      next if archived_deps.include? step.path
-      next if seen.include? step.path
-      next if self.done? && connected && ! updatable?
-
-      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
-      new_dependencies.concat r
-      new_dependencies << step
-    }
-
-    new_dependencies.uniq
-  end
-
   def writable?
     Open.writable?(self.path) && Open.writable?(self.info_file)
   end

data/lib/rbbt/workflow/util/data.rb

@@ -3,13 +3,15 @@ require 'rbbt/workflow/examples'
 
 module Workflow
   module Data
-    def data(directory)
+    class DataNotFound < RbbtException; end
+
+    def data_dir(directory)
       @@data_directory = directory
     end
 
     def get_datadir(clean_name)
       data_dir = File.join(@@data_directory, clean_name)
-      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      raise DataNotFound, "Data dir not found for #{clean_name} in #{@@data_directory}" unless File.directory?(data_dir)
       if Path === @@data_directory
         @@data_directory.annotate data_dir
       else
@@ -17,19 +19,147 @@ module Workflow
       end
     end
 
+    def get_file(clean_name, type = nil, extension = nil)
+      begin
+        extension = nil if String === extension && extension.empty?
+        file1 = File.join(@@data_directory, type.to_s, (extension.nil? ? clean_name.to_s : clean_name.to_s + ".#{extension}"))
+        file2 = File.join(@@data_directory, clean_name.to_s, (extension.nil? ? type.to_s : type.to_s + ".#{extension}"))
+        if Open.exists?(file1)
+          data_file = file1
+        elsif Open.exists?(file2)
+          data_file = file2
+        else
+          raise DataNotFound, "Data type #{type} not found for #{clean_name} in #{@@data_directory}"
+        end
+      end
+
+      if Path === @@data_directory
+        @@data_directory.annotate data_file
+      else
+        Path.setup(data_file)
+      end
+    end
+
     def data_task(name, workflow, oname, *rest, &block)
-      dep_task(name, workflow, oname, *rest) do |jobname, options|
-        data_dir = self.get_datadir(jobname)
-        task_info = workflow.task_info(oname)
-        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
-        data_options = block.call data_dir, dir_options, task_info
-        if data_options.include?(:inputs)
-          data_options
+      dep_task(name, workflow, oname, *rest) do |jobname, options,dependencies|
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          options[:extension] ||= task_info[:extension]
+          path = get_file jobname, name, options[:extension]
+          job = Step.new path
+          job.task_name = name
+          job.result_type = options[:result_type] || options[:type] || options[:extension]
+          job
+        rescue DataNotFound
+        end
+
+        next job if job
+
+        data_options = {}
+        data_options = data_options.merge(Workflow.load_inputs(File.join(@@data_directory,"options"), task_info[:inputs], task_info[:input_types]))
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_file(jobname, :options), task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_datadir(jobname).options, task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          data_options = block.call get_datadir(jobname), data_options, task_info
+        rescue
+          Log.exception $!
+        end if block_given?
+
+        case data_options
+        when Step
+          next data_options
+        when Hash
+          if data_options.include?(:inputs)
+            data_options = data_options.merge(options)
+            workflow = data_options[:workflow] if data_options[:workflow]
+            oname = data_options[:task] if data_options[:task]
+            inputs = options.merge(data_options[:inputs])
+          else
+            inputs = options.merge(data_options)
+          end
+
+          {:workflow => workflow, :task => oname, :jobname => jobname, :inputs => inputs}
         else
-          {:inputs => data_options.merge(options)}
+          raise "Cannot understand data_options: #{data_options}"
         end
+
       end
     end
-  end
 
+    def data(name, options = {}, &block)
+      dep do |jobname, job_options, dependencies|
+        if block_given?
+          block.call jobname, job_options.merge(options), dependencies
+        else
+          begin
+            path = get_file jobname, name, options[:extension]
+            job = Step.new path
+            job.task_name = name
+            job.result_type = options[:result_type] || options[:type] || options[:extension]
+            job
+          rescue DataNotFound
+            {:task => name, :options => options, :jobname => jobname}
+          end
+        end
+      end
+    end
+  end
 end
+#module Workflow
+#  module Data
+#    def data(directory)
+#      @@data_directory = directory
+#    end
+#
+#    def get_datadir(clean_name)
+#      data_dir = File.join(@@data_directory, clean_name)
+#      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+#      if Path === @@data_directory
+#        @@data_directory.annotate data_dir
+#      else
+#        Path.setup(data_dir)
+#      end
+#    end
+#
+#    def data_task(name, workflow, oname, *rest, &block)
+#      dep_task(name, workflow, oname, *rest) do |jobname, options|
+#        data_dir = self.get_datadir(jobname)
+#        task_info = workflow.task_info(oname)
+#
+#        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+#        data_options = block.call data_dir, dir_options, task_info
+#
+#        case data_options
+#        when Step
+#          job = data_options
+#        when Hash
+#          if data_options.include?(:inputs)
+#            data_options = data_options.merge(options)
+#            workflow = data_options[:workflow] if data_options[:workflow]
+#            oname = data_options[:task] if data_options[:task]
+#            inputs = options.merge(data_options[:inputs])
+#          else
+#            inputs = options.merge(data_options)
+#          end
+#
+#          job = workflow.job(oname, jobname, inputs)
+#        else
+#          raise "Cannot understand data_options: #{data_options}"
+#        end
+#
+#        job
+#      end
+#    end
+#  end
+#
+#end

data/test/rbbt/workflow/util/test_data.rb

@@ -1,6 +1,16 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
 require 'rbbt/workflow/util/data'
 
+module TestDataWF2
+  extend Workflow
+  extend Workflow::Data
+
+  input :salutation, :string
+  task :salute_antonio => :string do |name,salutation|
+    "Hi Antonio: #{salutation}"
+  end
+end
+
 module TestDataWF
   extend Workflow
   extend Workflow::Data
@@ -23,6 +33,11 @@ module TestDataWF
   data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
     {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
   end
+
+  data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
+
 end
 
 class TestWorkflowData < Test::Unit::TestCase
@@ -33,15 +48,19 @@ class TestWorkflowData < Test::Unit::TestCase
       Open.write(tmpdir.TestDir.options.name, "Miguel")
       Open.write(tmpdir.TestDir.salutation, "My salutations")
 
-      TestDataWF.data tmpdir
+      TestDataWF.data_dir tmpdir
 
       job = TestDataWF.job(:salute_data, "TestDir")
       job.recursive_clean.run
       assert job.run.include? "Miguel"
 
       job = TestDataWF.job(:salute_data2, "TestDir")
-      job.recursive_clean.run
+      puts job.recursive_clean.run
       assert job.run.include? "Luis"
+
+      job = TestDataWF.job(:salute_data3, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Antonio"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.18
+  version: 5.32.23
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-14 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake