rbbt-util 5.32.22 → 5.32.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bff9b30f50c873f7db46003571a8557c0ea756aaf342041119d4737da78e461b
-  data.tar.gz: 0765a797be1c16f90cd5ebf4f18fe5adb3d154aa69d07ab666cfe464fcb0a6bd
+  metadata.gz: 900a7061d40ec2cbffd38886ef9629aa0c0ed6d6ee1aef6e0c95c51656230ef0
+  data.tar.gz: 13b965a0ec134a932c6ae0ae6a109fbc2bc83b7ffd855a1ef05ed4e1e51b9d88
 SHA512:
-  metadata.gz: 9ba97c967d615145553d0f73bd84debbceee41c7ce1ca0c4fda512eeaf54880c256d36a4cf9d215ddce97b43a44a2cf19aa17f7e3b086512633210401cdfc085
-  data.tar.gz: c58f507abee878f56945886b06582271ace368c00808ff6fc7d0f0343373070a40dac797aff440a8abffd8cce3647fec9affeff078150bab8b7f52625754c91b
+  metadata.gz: 3b890e2b6fb3675df4319d0541d9057a54b0cb37286a814822915f5b5b061d4f7e02deab0afdd7cd0833cb741aeded7454a9585f93ff3486e595b99db4da00f6
+  data.tar.gz: 105072ca6c7b5c43c6a60455eaa9c7167feeb93f505c61c6aeffe6305de3166a6cb440c262abf025407306ba82c2fe3167f5f9e743e086d99fa936e44ea8e448

data/lib/rbbt/tsv/parser.rb

@@ -133,7 +133,7 @@ module TSV
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| value.nil? ? [] : value.split(@sep2, -1) }
+               end.collect{|value| (value.empty? || value.nil?) ? [nil] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/workflow/accessor.rb

@@ -172,50 +172,6 @@ module Workflow
     }.tap{|h| IndiferentHash.setup(h)}
   end
 
-  def assign_dep_inputs(_inputs, options, all_d, task_info)
-    IndiferentHash.setup(_inputs)
-
-    options.each{|i,v|
-      next if i == :compute or i == "compute"
-      case v
-      when :compute
-        compute = v
-      when Symbol
-        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
-
-        if rec_dependency.nil?
-          if _inputs.include?(v)
-            #_inputs[i] = _inputs.delete(v)
-            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
-          else
-            _inputs[i] = v unless _inputs.include? i
-          end
-        else
-          input_options = task_info[:input_options][i] || {}
-
-          #ToDo why was this always true?
-          if input_options[:stream] or true
-            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
-            _inputs[i] = rec_dependency
-          else
-            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
-            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
-            if rec_dependency.streaming? and rec_dependency.running?
-              _inputs[i] = rec_dependency.join.load
-            else
-              rec_dependency.run(true)
-              rec_dependency.join
-              _inputs[i] = rec_dependency.load
-            end
-          end
-        end
-      else
-        _inputs[i] = v
-      end
-    } if options
-
-    _inputs
-  end
 
   def task_from_dep(dep)
     task = case dep

data/lib/rbbt/workflow/dependencies.rb

@@ -92,6 +92,51 @@ module Workflow
     dep
   end
 
+  def assign_dep_inputs(_inputs, options, all_d, task_info)
+    IndiferentHash.setup(_inputs)
+
+    options.each{|i,v|
+      next if i == :compute or i == "compute"
+      case v
+      when :compute
+        compute = v
+      when Symbol
+        rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
+
+        if rec_dependency.nil?
+          if _inputs.include?(v)
+            #_inputs[i] = _inputs.delete(v)
+            _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
+          else
+            _inputs[i] = v unless _inputs.include? i
+          end
+        else
+          input_options = task_info[:input_options][i] || {}
+
+          #ToDo why was this always true?
+          if input_options[:stream] or true
+            #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
+            _inputs[i] = rec_dependency
+          else
+            rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
+            rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
+            if rec_dependency.streaming? and rec_dependency.running?
+              _inputs[i] = rec_dependency.join.load
+            else
+              rec_dependency.run(true)
+              rec_dependency.join
+              _inputs[i] = rec_dependency.load
+            end
+          end
+        end
+      else
+        _inputs[i] = v
+      end
+    } if options
+
+    _inputs
+  end
+
   def real_dependencies(task, orig_jobname, inputs, dependencies)
     real_dependencies = []
     path_deps = {}

data/lib/rbbt/workflow/step.rb

@@ -179,6 +179,10 @@ class Step
                      end
   end
 
+  def result_type=(type)
+    @result_type = type
+  end
+
   def result_description
     @result_description ||= if @task.nil?
                        info[:result_description]

data/lib/rbbt/workflow/util/data.rb

@@ -3,13 +3,15 @@ require 'rbbt/workflow/examples'
 
 module Workflow
   module Data
-    def data(directory)
+    class DataNotFound < RbbtException; end
+
+    def data_dir(directory)
       @@data_directory = directory
     end
 
     def get_datadir(clean_name)
       data_dir = File.join(@@data_directory, clean_name)
-      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      raise DataNotFound, "Data dir not found for #{clean_name} in #{@@data_directory}" unless File.directory?(data_dir)
       if Path === @@data_directory
         @@data_directory.annotate data_dir
       else
@@ -17,19 +19,147 @@ module Workflow
       end
     end
 
+    def get_file(clean_name, type = nil, extension = nil)
+      begin
+        extension = nil if String === extension && extension.empty?
+        file1 = File.join(@@data_directory, type.to_s, (extension.nil? ? clean_name.to_s : clean_name.to_s + ".#{extension}"))
+        file2 = File.join(@@data_directory, clean_name.to_s, (extension.nil? ? type.to_s : type.to_s + ".#{extension}"))
+        if Open.exists?(file1)
+          data_file = file1
+        elsif Open.exists?(file2)
+          data_file = file2
+        else
+          raise DataNotFound, "Data type #{type} not found for #{clean_name} in #{@@data_directory}"
+        end
+      end
+
+      if Path === @@data_directory
+        @@data_directory.annotate data_file
+      else
+        Path.setup(data_file)
+      end
+    end
+
     def data_task(name, workflow, oname, *rest, &block)
-      dep_task(name, workflow, oname, *rest) do |jobname, options|
-        data_dir = self.get_datadir(jobname)
-        task_info = workflow.task_info(oname)
-        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
-        data_options = block.call data_dir, dir_options, task_info
-        if data_options.include?(:inputs)
-          data_options
+      dep_task(name, workflow, oname, *rest) do |jobname, options,dependencies|
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          options[:extension] ||= task_info[:extension]
+          path = get_file jobname, name, options[:extension]
+          job = Step.new path
+          job.task_name = name
+          job.result_type = options[:result_type] || options[:type] || options[:extension]
+          job
+        rescue DataNotFound
+        end
+
+        next job if job
+
+        data_options = {}
+        data_options = data_options.merge(Workflow.load_inputs(File.join(@@data_directory,"options"), task_info[:inputs], task_info[:input_types]))
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_file(jobname, :options), task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          data_options = data_options.merge(Workflow.load_inputs(get_datadir(jobname).options, task_info[:inputs], task_info[:input_types]))
+        rescue DataNotFound
+        end
+
+        begin
+          task_info = workflow.nil? ? nil : workflow.task_info(oname) 
+          data_options = block.call get_datadir(jobname), data_options, task_info
+        rescue
+          Log.exception $!
+        end if block_given?
+
+        case data_options
+        when Step
+          next data_options
+        when Hash
+          if data_options.include?(:inputs)
+            data_options = data_options.merge(options)
+            workflow = data_options[:workflow] if data_options[:workflow]
+            oname = data_options[:task] if data_options[:task]
+            inputs = options.merge(data_options[:inputs])
+          else
+            inputs = options.merge(data_options)
+          end
+
+          {:workflow => workflow, :task => oname, :jobname => jobname, :inputs => inputs}
         else
-          {:inputs => data_options.merge(options)}
+          raise "Cannot understand data_options: #{data_options}"
         end
+
       end
     end
-  end
 
+    def data(name, options = {}, &block)
+      dep do |jobname, job_options, dependencies|
+        if block_given?
+          block.call jobname, job_options.merge(options), dependencies
+        else
+          begin
+            path = get_file jobname, name, options[:extension]
+            job = Step.new path
+            job.task_name = name
+            job.result_type = options[:result_type] || options[:type] || options[:extension]
+            job
+          rescue DataNotFound
+            {:task => name, :options => options, :jobname => jobname}
+          end
+        end
+      end
+    end
+  end
 end
+#module Workflow
+#  module Data
+#    def data(directory)
+#      @@data_directory = directory
+#    end
+#
+#    def get_datadir(clean_name)
+#      data_dir = File.join(@@data_directory, clean_name)
+#      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+#      if Path === @@data_directory
+#        @@data_directory.annotate data_dir
+#      else
+#        Path.setup(data_dir)
+#      end
+#    end
+#
+#    def data_task(name, workflow, oname, *rest, &block)
+#      dep_task(name, workflow, oname, *rest) do |jobname, options|
+#        data_dir = self.get_datadir(jobname)
+#        task_info = workflow.task_info(oname)
+#
+#        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+#        data_options = block.call data_dir, dir_options, task_info
+#
+#        case data_options
+#        when Step
+#          job = data_options
+#        when Hash
+#          if data_options.include?(:inputs)
+#            data_options = data_options.merge(options)
+#            workflow = data_options[:workflow] if data_options[:workflow]
+#            oname = data_options[:task] if data_options[:task]
+#            inputs = options.merge(data_options[:inputs])
+#          else
+#            inputs = options.merge(data_options)
+#          end
+#
+#          job = workflow.job(oname, jobname, inputs)
+#        else
+#          raise "Cannot understand data_options: #{data_options}"
+#        end
+#
+#        job
+#      end
+#    end
+#  end
+#
+#end

data/test/rbbt/workflow/util/test_data.rb

@@ -1,6 +1,16 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
 require 'rbbt/workflow/util/data'
 
+module TestDataWF2
+  extend Workflow
+  extend Workflow::Data
+
+  input :salutation, :string
+  task :salute_antonio => :string do |name,salutation|
+    "Hi Antonio: #{salutation}"
+  end
+end
+
 module TestDataWF
   extend Workflow
   extend Workflow::Data
@@ -23,6 +33,11 @@ module TestDataWF
   data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
     {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
   end
+
+  data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
+    {:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
+  end
+
 end
 
 class TestWorkflowData < Test::Unit::TestCase
@@ -33,15 +48,19 @@ class TestWorkflowData < Test::Unit::TestCase
       Open.write(tmpdir.TestDir.options.name, "Miguel")
       Open.write(tmpdir.TestDir.salutation, "My salutations")
 
-      TestDataWF.data tmpdir
+      TestDataWF.data_dir tmpdir
 
       job = TestDataWF.job(:salute_data, "TestDir")
       job.recursive_clean.run
       assert job.run.include? "Miguel"
 
       job = TestDataWF.job(:salute_data2, "TestDir")
-      job.recursive_clean.run
+      puts job.recursive_clean.run
       assert job.run.include? "Luis"
+
+      job = TestDataWF.job(:salute_data3, "TestDir")
+      job.recursive_clean.run
+      assert job.run.include? "Antonio"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.22
+  version: 5.32.23
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-16 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake