rbbt-util 5.32.20 → 5.32.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 576b7bc079d39a3054c5bdaa3c0b3af5573a26aacd6181fe65e8d450fd3ed148
-  data.tar.gz: 2dc53972fc0e8504ca327de2f680d29ac5c3df63362359528b25ec21f2c10e9b
+  metadata.gz: 7a7eebab331a8908a290e4a3ad8db2ee3dc3c3dec07b93ec1c74f504f9f7ea29
+  data.tar.gz: 0f9a2ce6019676118ab439dea57f16605555d9ef850c35998704b9215c477ffb
 SHA512:
-  metadata.gz: 3cd48cedd0a1957996b59ac51bc56b2b3c8ce8aef54454b1ff10f8f86a08c9df63a60a1d1bad7c29a0741bf687c215f36b628e614b70aa9b8181d687a075364e
-  data.tar.gz: 8955e34256095ea4c2e66cc6937845f6b13cb4c87e2bc13f5f7bb4844135886407e50e31114c94e0bdba157f63aab4027c7d94add60f85f31e0646aab3bf124f
+  metadata.gz: 2d81532c2babaae631ca98784e7b7e36039fed68544b6bce9fc8fc2d74a155ffd98fe0752477c7ad245aaa0c65e82c47f330f65c5d70a7f5f20a77de4bc4dbd6
+  data.tar.gz: 2914786ec23349cabccbb6e8a1c561336b60dd5d862e89c3a95802f040c66f8107d5a2135455f00d8bc7b9e06e390fd6462a861419b78ae25bf3a58c14a33350

data/bin/rbbt

@@ -262,9 +262,8 @@ rescue ParameterException
   puts
   exit_status = -1
   exit exit_status
-rescue SystemExit
-rescue CmdStop
-  exit_status = $!.exit_status
+rescue SystemExit,CmdStop
+  exit_status = $!.status
   exit exit_status
 rescue Exception
   Log.exception $!

data/lib/rbbt/tsv/manipulate.rb

@@ -356,6 +356,16 @@ module TSV
     elems.sort_by{|k,v| v}.collect{|k,v| k}
   end
 
+  def subset(keys)
+    new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
+    self.with_unnamed do
+      keys.each do |k|
+        new[k] = self[k]
+      end
+    end
+    new
+  end
+
   def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 

data/lib/rbbt/tsv/parser.rb

@@ -133,7 +133,7 @@ module TSV
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| value.nil? ? [] : value.split(@sep2, -1) }
+               end.collect{|value| (value.empty? || value.nil?) ? [nil] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/util/config.rb

@@ -122,7 +122,12 @@ module Rbbt::Config
     Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
     GOT_KEYS << [key, value, tokens]
 
-    value
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    else
+      value
+    end
   end
 
   def self.with_config

data/lib/rbbt/workflow/accessor.rb

@@ -172,50 +172,6 @@ module Workflow
     }.tap{|h| IndiferentHash.setup(h)}
   end
 
-  def assign_dep_inputs(_inputs, options, all_d, task_info)
-    IndiferentHash.setup(_inputs)
-
-    options.each{|i,v|
-      next if i == :compute or i == "compute"
-      case v
-      when :compute
-        compute = v
-      when Symbol
-        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
-
-        if rec_dependency.nil?
-          if _inputs.include?(v)
-            #_inputs[i] = _inputs.delete(v)
-            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
-          else
-            _inputs[i] = v unless _inputs.include? i
-          end
-        else
-          input_options = task_info[:input_options][i] || {}
-
-          #ToDo why was this always true?
-          if input_options[:stream] or true
-            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
-            _inputs[i] = rec_dependency
-          else
-            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
-            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
-            if rec_dependency.streaming? and rec_dependency.running?
-              _inputs[i] = rec_dependency.join.load
-            else
-              rec_dependency.run(true)
-              rec_dependency.join
-              _inputs[i] = rec_dependency.load
-            end
-          end
-        end
-      else
-        _inputs[i] = v
-      end
-    } if options
-
-    _inputs
-  end
 
   def task_from_dep(dep)
     task = case dep

data/lib/rbbt/workflow/definition.rb

@@ -98,11 +98,12 @@ module Workflow
         when 'true'
           dep.clean
         when 'recursive'
-          dep.rec_dependencies.each do |d|
+          (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+            next if d.overriden
             d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
           end
           dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
-        end
+        end unless dep.overriden
       else
         if Open.exists?(dep.files_dir)
           Open.rm_rf self.files_dir 

data/lib/rbbt/workflow/dependencies.rb

@@ -71,7 +71,7 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
-    dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep = Step === dep ? dep.dup : Workflow.load_step(dep)
     dep.workflow = workflow
     dep.info[:name] = dep.name
     dep.original_task_name ||= dep.task_name if dep.workflow
@@ -83,14 +83,60 @@ module Workflow
     rescue
       Log.exception $!
     end
+
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
 
     dep.extend step_module
 
     dep
   end
 
+  def assign_dep_inputs(_inputs, options, all_d, task_info)
+    IndiferentHash.setup(_inputs)
+
+    options.each{|i,v|
+      next if i == :compute or i == "compute"
+      case v
+      when :compute
+        compute = v
+      when Symbol
+        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
+
+        if rec_dependency.nil?
+          if _inputs.include?(v)
+            #_inputs[i] = _inputs.delete(v)
+            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
+          else
+            _inputs[i] = v unless _inputs.include? i
+          end
+        else
+          input_options = task_info[:input_options][i] || {}
+
+          #ToDo why was this always true?
+          if input_options[:stream] or true
+            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
+            _inputs[i] = rec_dependency
+          else
+            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
+            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
+            if rec_dependency.streaming? and rec_dependency.running?
+              _inputs[i] = rec_dependency.join.load
+            else
+              rec_dependency.run(true)
+              rec_dependency.join
+              _inputs[i] = rec_dependency.load
+            end
+          end
+        end
+      else
+        _inputs[i] = v
+      end
+    } if options
+
+    _inputs
+  end
+
   def real_dependencies(task, orig_jobname, inputs, dependencies)
     real_dependencies = []
     path_deps = {}

data/lib/rbbt/workflow/examples.rb

@@ -48,7 +48,7 @@ module Workflow
 
 
         case input_types[input]
-        when :file
+        when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
             inputs[input.to_sym]  = YAML.load(Open.read(file))

data/lib/rbbt/workflow/step/dependencies.rb

@@ -206,7 +206,7 @@ class Step
     if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
-        other_steps = dep_step[step.path].uniq
+        other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
         return unless other_steps.length > 1
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
@@ -523,4 +523,36 @@ class Step
     set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
   end
 
+  #connected = true means that dependency searching ends when a result is done
+  #but dependencies are absent, meanining that the file could have been dropped
+  #in
+  def rec_dependencies(connected = false, seen = [])
+
+    # A step result with no info_file means that it was manually
+    # placed. In that case, do not consider its dependencies
+    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
+
+    return [] if dependencies.nil? or dependencies.empty?
+
+    new_dependencies = []
+    if self.overriden?
+      archived_deps = []
+    else
+      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+    end
+
+    dependencies.each{|step| 
+      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
+      next if archived_deps.include? step.path
+      next if seen.include? step.path
+      next if self.done? && connected && ! updatable?
+
+      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
+      new_dependencies.concat r
+      new_dependencies << step
+    }
+
+    new_dependencies.uniq
+  end
+
 end

data/lib/rbbt/workflow/step.rb

@@ -179,6 +179,10 @@ class Step
                      end
   end
 
+  def result_type=(type)
+    @result_type = type
+  end
+
   def result_description
     @result_description ||= if @task.nil?
                        info[:result_description]
@@ -351,38 +355,6 @@ class Step
     end
   end
 
-  #connected = true means that dependency searching ends when a result is done
-  #but dependencies are absent, meanining that the file could have been dropped
-  #in
-  def rec_dependencies(connected = false, seen = [])
-
-    # A step result with no info_file means that it was manually
-    # placed. In that case, do not consider its dependencies
-    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
-
-    return [] if dependencies.nil? or dependencies.empty?
-
-    new_dependencies = []
-    if self.overriden?
-      archived_deps = []
-    else
-      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
-    end
-
-    dependencies.each{|step| 
-      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
-      next if archived_deps.include? step.path
-      next if seen.include? step.path
-      next if self.done? && connected && ! updatable?
-
-      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
-      new_dependencies.concat r
-      new_dependencies << step
-    }
-
-    new_dependencies.uniq
-  end
-
   def writable?
     Open.writable?(self.path) && Open.writable?(self.info_file)
   end

data/lib/rbbt/workflow/util/data.rb

@@ -3,13 +3,15 @@ require 'rbbt/workflow/examples'
 
 module Workflow
   module Data
-    def data(directory)
+    class DataNotFound < RbbtException; end
+
+    def data_dir(directory)
       @@data_directory = directory
     end
 
     def get_datadir(clean_name)
       data_dir = File.join(@@data_directory, clean_name)
-      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      raise DataNotFound, "Data dir not found for #{clean_name} in #{@@data_directory}" unless File.directory?(data_dir)
       if Path === @@data_directory
         @@data_directory.annotate data_dir
       else
@@ -17,19 +19,147 @@ module Workflow
       end
     end
 
+    def get_file(clean_name, type = nil, extension = nil)
+      begin
+        extension = nil if String === extension && extension.empty?
+        file1 = File.join(@@data_directory, type.to_s, (extension.nil? ? clean_name.to_s : clean_name.to_s + ".#{extension}"))
+        file2 = File.join(@@data_directory, clean_name.to_s, (extension.nil? ? type.to_s : type.to_s + ".#{extension}"))
+        if Open.exists?(file1)
+          data_file = file1
+        elsif Open.exists?(file2)
+          data_file = file2
+        else
+          raise DataNotFound, "Data type #{type} not found for #{clean_name} in #{@@data_directory}"
+        end
+      end
+
+      if Path === @@data_directory
+        @@data_directory.annotate data_file
+      else
+        Path.setup(data_file)
+      end
+    end
+
     def data_task(name, workflow, oname, *rest, &block)
-      dep_task(name, workflow, oname, *rest) do |jobname, options|
-        data_dir = self.get_datadir(jobname)
-        task_info = workflow.task_info(oname)
-        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
-        data_options = block.call data_dir, dir_options, task_info
-        if data_options.include?(:inputs)
-          data_options
+      dep_task(name, workflow, oname, *rest) do |jobname, options,dependencies|
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          options[:extension] ||= task_info[:extension]
+          path = get_file jobname, name, options[:extension]
+          job = Step.new path
+          job.task_name = name
+          job.result_type = options[:result_type] || options[:type] || options[:extension]
+          job
+        rescue DataNotFound
+        end
+
+        next job if job
+
+        data_options = {}
+        data_options = data_options.merge(Workflow.load_inputs(File.join(@@data_directory,"options"), task_info[:inputs], task_info[:input_types]))
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_file(jobname, :options), task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_datadir(jobname).options, task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          data_options = block.call get_datadir(jobname), data_options, task_info
+        rescue
+          Log.exception $!
+        end if block_given?
+
+        case data_options
+        when Step
+          next data_options
+        when Hash
+          if data_options.include?(:inputs)
+            data_options = data_options.merge(options)
+            workflow = data_options[:workflow] if data_options[:workflow]
+            oname = data_options[:task] if data_options[:task]
+            inputs = options.merge(data_options[:inputs])
+          else
+            inputs = options.merge(data_options)
+          end
+
+          {:workflow => workflow, :task => oname, :jobname => jobname, :inputs => inputs}
         else
-          {:inputs => data_options.merge(options)}
+          raise "Cannot understand data_options: #{data_options}"
         end
+
       end
     end
-  end
 
+    def data(name, options = {}, &block)
+      dep do |jobname, job_options, dependencies|
+        if block_given?
+          block.call jobname, job_options.merge(options), dependencies
+        else
+          begin
+            path = get_file jobname, name, options[:extension]
+            job = Step.new path
+            job.task_name = name
+            job.result_type = options[:result_type] || options[:type] || options[:extension]
+            job
+          rescue DataNotFound
+            {:task => name, :options => options, :jobname => jobname}
+          end
+        end
+      end
+    end
+  end
 end
+#module Workflow
+#  module Data
+#    def data(directory)
+#      @@data_directory = directory
+#    end
+#
+#    def get_datadir(clean_name)
+#      data_dir = File.join(@@data_directory, clean_name)
+#      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+#      if Path === @@data_directory
+#        @@data_directory.annotate data_dir
+#      else
+#        Path.setup(data_dir)
+#      end
+#    end
+#
+#    def data_task(name, workflow, oname, *rest, &block)
+#      dep_task(name, workflow, oname, *rest) do |jobname, options|
+#        data_dir = self.get_datadir(jobname)
+#        task_info = workflow.task_info(oname)
+#
+#        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+#        data_options = block.call data_dir, dir_options, task_info
+#
+#        case data_options
+#        when Step
+#          job = data_options
+#        when Hash
+#          if data_options.include?(:inputs)
+#            data_options = data_options.merge(options)
+#            workflow = data_options[:workflow] if data_options[:workflow]
+#            oname = data_options[:task] if data_options[:task]
+#            inputs = options.merge(data_options[:inputs])
+#          else
+#            inputs = options.merge(data_options)
+#          end
+#
+#          job = workflow.job(oname, jobname, inputs)
+#        else
+#          raise "Cannot understand data_options: #{data_options}"
+#        end
+#
+#        job
+#      end
+#    end
+#  end
+#
+#end

data/lib/rbbt/workflow.rb

@@ -45,14 +45,10 @@ module Workflow
     load_remote_tasks(Rbbt.root.etc.remote_tasks.find) if Rbbt.root.etc.remote_tasks.exists?
   end
 
-  def self.require_remote_workflow(wf_name, url)
-    require 'rbbt/workflow/remote_workflow'
-    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
-  end
 
   def self.require_remote_workflow(wf_name, url)
     require 'rbbt/workflow/remote_workflow'
-    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
+    eval "Object::#{wf_name.split("+").first} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
   end
 
   def self.load_workflow_libdir(filename)
@@ -134,9 +130,10 @@ module Workflow
   end
 
   def self.require_local_workflow(wf_name)
+
     filename = local_workflow_filename(wf_name)
 
-    if filename and File.exist? filename
+    if filename and File.exist?(filename)
       load_workflow_file filename
     else
       return false
@@ -194,14 +191,23 @@ module Workflow
     end
 
     Log.high{"Loading workflow #{wf_name}"}
-    require_local_workflow(wf_name) or 
-    (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
-    workflow = begin
-                 Misc.string2const Misc.camel_case(wf_name)
-               rescue
-                 Workflow.workflows.last || true
-               end
-    workflow.load_documentation
+
+    first = nil
+    wf_name.split("+").each do |wf_name|
+      require_local_workflow(wf_name) or 
+        (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
+
+      workflow = begin
+                   Misc.string2const Misc.camel_case(wf_name.split("+").first)
+                 rescue
+                   Workflow.workflows.last || true
+                 end
+      workflow.load_documentation
+
+      first ||= workflow
+    end
+    return first
+
     workflow
   end
 

data/share/rbbt_commands/workflow/task

@@ -287,8 +287,8 @@ else
     puts
     puts $!.message
     puts
-    
-    exit 0
+
+    exit -1
   end
 end
 
@@ -572,7 +572,9 @@ when Step
     exit! 0
   else
     res.join
-    out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    Open.open(res.path, :mode => 'rb') do |io|
+      Misc.consume_stream(io, false, out)
+    end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
   end
 else
   if Array === res

data/test/rbbt/workflow/util/test_data.rb

@@ -1,6 +1,16 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
 require 'rbbt/workflow/util/data'
 
+module TestDataWF2
+  extend Workflow
+  extend Workflow::Data
+
+  input :salutation, :string
+  task :salute_antonio => :string do |name,salutation|
+    "Hi Antonio: #{salutation}"
+  end
+end
+
 module TestDataWF
   extend Workflow
   extend Workflow::Data
@@ -23,6 +33,11 @@ module TestDataWF
   data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
     {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
   end
+
+  data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
+
 end
 
 class TestWorkflowData < Test::Unit::TestCase
@@ -33,15 +48,19 @@ class TestWorkflowData < Test::Unit::TestCase
       Open.write(tmpdir.TestDir.options.name, "Miguel")
       Open.write(tmpdir.TestDir.salutation, "My salutations")
 
-      TestDataWF.data tmpdir
+      TestDataWF.data_dir tmpdir
 
       job = TestDataWF.job(:salute_data, "TestDir")
       job.recursive_clean.run
       assert job.run.include? "Miguel"
 
       job = TestDataWF.job(:salute_data2, "TestDir")
-      job.recursive_clean.run
+      puts job.recursive_clean.run
       assert job.run.include? "Luis"
+
+      job = TestDataWF.job(:salute_data3, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Antonio"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.20
+  version: 5.32.24
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-15 00:00:00.000000000 Z
+date: 2021-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake