rbbt-util 5.32.21 → 5.32.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14fc2236d4c0852a44d9db76c43aa19c32619cede05e03801a830ec4a996420a
-  data.tar.gz: ceed6fe15817c5fd4635b5b96a557ae010d9aa50ccfdb774a03f959d3e93314d
+  metadata.gz: 4709cccef1b9d9d46e4b8888f336b65e6059b41f2289a891bcc58795e949d155
+  data.tar.gz: 373e0ec6458f1d6cfdbfbe6aadb9e585ee1c347f2395480ba950beca3b3ad38f
 SHA512:
-  metadata.gz: 0f327628991f8906fc353776ee2d887cd79edfa5cb07def1f8dec207ab2d922c3e4b40a141a8fa128904a28bd5b6d84887f3518bd095d98427e14710567f5495
-  data.tar.gz: a53a644a91737b2cb0e76c7eb641daeace94537da7c3ded5b993fb936f524522f81b7f06ddd458ba72a3672279e7f66bf670553c8683f21de7d7cff66bac9400
+  metadata.gz: 276e9b6ab9988cb3e53ed44ee166e81f72da3f9eb119ed8269ea47bbe9cddc3bbebff5faa44b16d2d67c8f3ae575217e081c9aa8e740579c27ab4c6cfae2644d
+  data.tar.gz: e3c977b86918ed7c8efdf75d6aa6468cd078bbaf4d3acbb18202a6b2325bca8de4b901973c27a111558d83de7bfab7a0bfaa8f919b26301bb3532ba76636b6aa

data/bin/rbbt

@@ -262,9 +262,8 @@ rescue ParameterException
   puts
   exit_status = -1
   exit exit_status
-rescue SystemExit
-rescue CmdStop
-  exit_status = $!.exit_status
+rescue SystemExit,CmdStop
+  exit_status = $!.status
   exit exit_status
 rescue Exception
   Log.exception $!

data/bin/rbbt_find.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Find a path
+
+$ #{$0} [options] [<subpath>] <path>
+
+Use - to read from STDIN
+
+-h--help Print this help
+-w--workflows Workflow to load
+-s--search_path* Workflow to load
+-l--list List contents of resolved directories
+-n--nocolor Don't color output
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+subpath, path = ARGV
+path, subpath = subpath, nil if path.nil?
+
+begin
+  require 'rbbt/workflow'
+  workflow = Workflow.require_workflow subpath
+  subpath = workflow.libdir
+rescue
+  Log.exception $!
+end if subpath && subpath =~ /^[A-Z][a-zA-Z]+$/
+
+path = subpath ? Path.setup(subpath)[path] : Path.setup(path)
+
+search_path = options[:search_path].to_sym if options.include? :search_path
+nocolor = options[:nocolor]
+
+found = if search_path
+          [path.find(search_path)]
+        else
+          path.find_all
+        end
+
+found.each do |path|
+  if options[:list] && File.directory?(path)
+    puts Log.color :blue, path
+    path.glob("*").each do |subpath|
+      if nocolor
+        puts subpath
+      else
+        color = File.directory?(subpath) ? :blue : nil
+        puts " " << Log.color(color, subpath)
+      end
+    end
+  else
+    if nocolor
+      puts path
+    else
+      color = File.exists?(path) ? (File.directory?(path) ? :blue : nil) : :red
+      puts Log.color color, path
+    end
+
+  end
+end
+

data/lib/rbbt/tsv/manipulate.rb

@@ -356,6 +356,16 @@ module TSV
     elems.sort_by{|k,v| v}.collect{|k,v| k}
   end
 
+  def subset(keys)
+    new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
+    self.with_unnamed do
+      keys.each do |k|
+        new[k] = self[k]
+      end
+    end
+    new
+  end
+
   def select(method = nil, invert = false, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 

data/lib/rbbt/tsv/parser.rb

@@ -133,7 +133,7 @@ module TSV
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| value.nil? ? [] : value.split(@sep2, -1) }
+               end.collect{|value| (value.empty? || value.nil?) ? [nil] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/util/config.rb

@@ -122,7 +122,12 @@ module Rbbt::Config
     Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
     GOT_KEYS << [key, value, tokens]
 
-    value
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    else
+      value
+    end
   end
 
   def self.with_config

data/lib/rbbt/workflow/accessor.rb

@@ -172,50 +172,6 @@ module Workflow
     }.tap{|h| IndiferentHash.setup(h)}
   end
 
-  def assign_dep_inputs(_inputs, options, all_d, task_info)
-    IndiferentHash.setup(_inputs)
-
-    options.each{|i,v|
-      next if i == :compute or i == "compute"
-      case v
-      when :compute
-        compute = v
-      when Symbol
-        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
-
-        if rec_dependency.nil?
-          if _inputs.include?(v)
-            #_inputs[i] = _inputs.delete(v)
-            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
-          else
-            _inputs[i] = v unless _inputs.include? i
-          end
-        else
-          input_options = task_info[:input_options][i] || {}
-
-          #ToDo why was this always true?
-          if input_options[:stream] or true
-            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
-            _inputs[i] = rec_dependency
-          else
-            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
-            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
-            if rec_dependency.streaming? and rec_dependency.running?
-              _inputs[i] = rec_dependency.join.load
-            else
-              rec_dependency.run(true)
-              rec_dependency.join
-              _inputs[i] = rec_dependency.load
-            end
-          end
-        end
-      else
-        _inputs[i] = v
-      end
-    } if options
-
-    _inputs
-  end
 
   def task_from_dep(dep)
     task = case dep

data/lib/rbbt/workflow/dependencies.rb

@@ -92,6 +92,51 @@ module Workflow
     dep
   end
 
+  def assign_dep_inputs(_inputs, options, all_d, task_info)
+    IndiferentHash.setup(_inputs)
+
+    options.each{|i,v|
+      next if i == :compute or i == "compute"
+      case v
+      when :compute
+        compute = v
+      when Symbol
+        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
+
+        if rec_dependency.nil?
+          if _inputs.include?(v)
+            #_inputs[i] = _inputs.delete(v)
+            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
+          else
+            _inputs[i] = v unless _inputs.include? i
+          end
+        else
+          input_options = task_info[:input_options][i] || {}
+
+          #ToDo why was this always true?
+          if input_options[:stream] or true
+            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
+            _inputs[i] = rec_dependency
+          else
+            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
+            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
+            if rec_dependency.streaming? and rec_dependency.running?
+              _inputs[i] = rec_dependency.join.load
+            else
+              rec_dependency.run(true)
+              rec_dependency.join
+              _inputs[i] = rec_dependency.load
+            end
+          end
+        end
+      else
+        _inputs[i] = v
+      end
+    } if options
+
+    _inputs
+  end
+
   def real_dependencies(task, orig_jobname, inputs, dependencies)
     real_dependencies = []
     path_deps = {}

data/lib/rbbt/workflow/examples.rb

@@ -48,7 +48,7 @@ module Workflow
 
 
         case input_types[input]
-        when :file
+        when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
             inputs[input.to_sym]  = YAML.load(Open.read(file))

data/lib/rbbt/workflow/step.rb

@@ -179,6 +179,10 @@ class Step
                      end
   end
 
+  def result_type=(type)
+    @result_type = type
+  end
+
   def result_description
     @result_description ||= if @task.nil?
                        info[:result_description]

data/lib/rbbt/workflow/util/data.rb

@@ -3,13 +3,15 @@ require 'rbbt/workflow/examples'
 
 module Workflow
   module Data
-    def data(directory)
+    class DataNotFound < RbbtException; end
+
+    def data_dir(directory)
       @@data_directory = directory
     end
 
     def get_datadir(clean_name)
       data_dir = File.join(@@data_directory, clean_name)
-      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      raise DataNotFound, "Data dir not found for #{clean_name} in #{@@data_directory}" unless File.directory?(data_dir)
       if Path === @@data_directory
         @@data_directory.annotate data_dir
       else
@@ -17,19 +19,147 @@ module Workflow
       end
     end
 
+    def get_file(clean_name, type = nil, extension = nil)
+      begin
+        extension = nil if String === extension && extension.empty?
+        file1 = File.join(@@data_directory, type.to_s, (extension.nil? ? clean_name.to_s : clean_name.to_s + ".#{extension}"))
+        file2 = File.join(@@data_directory, clean_name.to_s, (extension.nil? ? type.to_s : type.to_s + ".#{extension}"))
+        if Open.exists?(file1)
+          data_file = file1
+        elsif Open.exists?(file2)
+          data_file = file2
+        else
+          raise DataNotFound, "Data type #{type} not found for #{clean_name} in #{@@data_directory}"
+        end
+      end
+
+      if Path === @@data_directory
+        @@data_directory.annotate data_file
+      else
+        Path.setup(data_file)
+      end
+    end
+
     def data_task(name, workflow, oname, *rest, &block)
-      dep_task(name, workflow, oname, *rest) do |jobname, options|
-        data_dir = self.get_datadir(jobname)
-        task_info = workflow.task_info(oname)
-        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
-        data_options = block.call data_dir, dir_options, task_info
-        if data_options.include?(:inputs)
-          data_options
+      dep_task(name, workflow, oname, *rest) do |jobname, options,dependencies|
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          options[:extension] ||= task_info[:extension]
+          path = get_file jobname, name, options[:extension]
+          job = Step.new path
+          job.task_name = name
+          job.result_type = options[:result_type] || options[:type] || options[:extension]
+          job
+        rescue DataNotFound
+        end
+
+        next job if job
+
+        data_options = {}
+        data_options = data_options.merge(Workflow.load_inputs(File.join(@@data_directory,"options"), task_info[:inputs], task_info[:input_types]))
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_file(jobname, :options), task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_datadir(jobname).options, task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          data_options = block.call get_datadir(jobname), data_options, task_info
+        rescue
+          Log.exception $!
+        end if block_given?
+
+        case data_options
+        when Step
+          next data_options
+        when Hash
+          if data_options.include?(:inputs)
+            data_options = data_options.merge(options)
+            workflow = data_options[:workflow] if data_options[:workflow]
+            oname = data_options[:task] if data_options[:task]
+            inputs = options.merge(data_options[:inputs])
+          else
+            inputs = options.merge(data_options)
+          end
+
+          {:workflow => workflow, :task => oname, :jobname => jobname, :inputs => inputs}
         else
-          {:inputs => data_options.merge(options)}
+          raise "Cannot understand data_options: #{data_options}"
         end
+
       end
     end
-  end
 
+    def data(name, options = {}, &block)
+      dep do |jobname, job_options, dependencies|
+        if block_given?
+          block.call jobname, job_options.merge(options), dependencies
+        else
+          begin
+            path = get_file jobname, name, options[:extension]
+            job = Step.new path
+            job.task_name = name
+            job.result_type = options[:result_type] || options[:type] || options[:extension]
+            job
+          rescue DataNotFound
+            {:task => name, :options => options, :jobname => jobname}
+          end
+        end
+      end
+    end
+  end
 end
+#module Workflow
+#  module Data
+#    def data(directory)
+#      @@data_directory = directory
+#    end
+#
+#    def get_datadir(clean_name)
+#      data_dir = File.join(@@data_directory, clean_name)
+#      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+#      if Path === @@data_directory
+#        @@data_directory.annotate data_dir
+#      else
+#        Path.setup(data_dir)
+#      end
+#    end
+#
+#    def data_task(name, workflow, oname, *rest, &block)
+#      dep_task(name, workflow, oname, *rest) do |jobname, options|
+#        data_dir = self.get_datadir(jobname)
+#        task_info = workflow.task_info(oname)
+#
+#        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+#        data_options = block.call data_dir, dir_options, task_info
+#
+#        case data_options
+#        when Step
+#          job = data_options
+#        when Hash
+#          if data_options.include?(:inputs)
+#            data_options = data_options.merge(options)
+#            workflow = data_options[:workflow] if data_options[:workflow]
+#            oname = data_options[:task] if data_options[:task]
+#            inputs = options.merge(data_options[:inputs])
+#          else
+#            inputs = options.merge(data_options)
+#          end
+#
+#          job = workflow.job(oname, jobname, inputs)
+#        else
+#          raise "Cannot understand data_options: #{data_options}"
+#        end
+#
+#        job
+#      end
+#    end
+#  end
+#
+#end

data/lib/rbbt/workflow.rb

@@ -45,14 +45,10 @@ module Workflow
     load_remote_tasks(Rbbt.root.etc.remote_tasks.find) if Rbbt.root.etc.remote_tasks.exists?
   end
 
-  def self.require_remote_workflow(wf_name, url)
-    require 'rbbt/workflow/remote_workflow'
-    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
-  end
 
   def self.require_remote_workflow(wf_name, url)
     require 'rbbt/workflow/remote_workflow'
-    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
+    eval "Object::#{wf_name.split("+").first} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
   end
 
   def self.load_workflow_libdir(filename)
@@ -134,9 +130,10 @@ module Workflow
   end
 
   def self.require_local_workflow(wf_name)
+
     filename = local_workflow_filename(wf_name)
 
-    if filename and File.exist? filename
+    if filename and File.exist?(filename)
       load_workflow_file filename
     else
       return false
@@ -194,14 +191,23 @@ module Workflow
     end
 
     Log.high{"Loading workflow #{wf_name}"}
-    require_local_workflow(wf_name) or 
-    (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
-    workflow = begin
-                 Misc.string2const Misc.camel_case(wf_name)
-               rescue
-                 Workflow.workflows.last || true
-               end
-    workflow.load_documentation
+
+    first = nil
+    wf_name.split("+").each do |wf_name|
+      require_local_workflow(wf_name) or 
+        (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
+
+      workflow = begin
+                   Misc.string2const Misc.camel_case(wf_name.split("+").first)
+                 rescue
+                   Workflow.workflows.last || true
+                 end
+      workflow.load_documentation
+
+      first ||= workflow
+    end
+    return first
+
     workflow
   end
 

data/share/rbbt_commands/workflow/task

@@ -287,8 +287,8 @@ else
     puts
     puts $!.message
     puts
-    
-    exit 0
+
+    exit -1
   end
 end
 
@@ -572,7 +572,9 @@ when Step
     exit! 0
   else
     res.join
-    out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    Open.open(res.path, :mode => 'rb') do |io|
+      Misc.consume_stream(io, false, out)
+    end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
   end
 else
   if Array === res

data/test/rbbt/workflow/util/test_data.rb

@@ -1,6 +1,16 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
 require 'rbbt/workflow/util/data'
 
+module TestDataWF2
+  extend Workflow
+  extend Workflow::Data
+
+  input :salutation, :string
+  task :salute_antonio => :string do |name,salutation|
+    "Hi Antonio: #{salutation}"
+  end
+end
+
 module TestDataWF
   extend Workflow
   extend Workflow::Data
@@ -23,6 +33,11 @@ module TestDataWF
   data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
     {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
   end
+
+  data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
+
 end
 
 class TestWorkflowData < Test::Unit::TestCase
@@ -33,15 +48,19 @@ class TestWorkflowData < Test::Unit::TestCase
       Open.write(tmpdir.TestDir.options.name, "Miguel")
       Open.write(tmpdir.TestDir.salutation, "My salutations")
 
-      TestDataWF.data tmpdir
+      TestDataWF.data_dir tmpdir
 
       job = TestDataWF.job(:salute_data, "TestDir")
       job.recursive_clean.run
       assert job.run.include? "Miguel"
 
       job = TestDataWF.job(:salute_data2, "TestDir")
-      job.recursive_clean.run
+      puts job.recursive_clean.run
       assert job.run.include? "Luis"
+
+      job = TestDataWF.job(:salute_data3, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Antonio"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.21
+  version: 5.32.25
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-15 00:00:00.000000000 Z
+date: 2021-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -172,6 +172,7 @@ executables:
 - rbbt_Rutil.rb
 - rbbt
 - rbbt_dangling_locks.rb
+- rbbt_find.rb
 extensions: []
 extra_rdoc_files:
 - LICENSE
@@ -183,6 +184,7 @@ files:
 - bin/rbbt_Rutil.rb
 - bin/rbbt_dangling_locks.rb
 - bin/rbbt_exec.rb
+- bin/rbbt_find.rb
 - bin/rbbt_query.rb
 - etc/app.d/base.rb
 - etc/app.d/entities.rb