rbbt-util 5.20.7 → 5.20.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6cd36b953fe4aad10752458b021f57a36ec8ed6a
-  data.tar.gz: ae1bb7eeec2b45423a6b68359452fa32286796d1
+  metadata.gz: 72d3825a7d4644c54b8498b2649e8820d610bef2
+  data.tar.gz: 02157c505e0f582a7b3cf4517ec590b124734602
 SHA512:
-  metadata.gz: d3161469951636f056f68fef3e2e4233111effa97cffd48cdb5e7f07c7ff3cf87b3eee9e67d8a8aa19286e5d03e47c5cb3ce5372ce25dbb34e64b229f065675e
-  data.tar.gz: 8ff9bfd9770e3b0cf7b1c65377eed457348ce83889d04369642210d2726dfc023a4593213512c282842ec20e3cbbaba0b6dae84c07e2d57e72acd27296c8bd8b
+  metadata.gz: e04bcbe557031df6d184336aa9ac8c2dfab9c33f509f4b81d8cf8fd6264a5fb01fc9c360fa9c35a5855a0a485642acdd29e201956f1ffe38f1a2af7f23b19c42
+  data.tar.gz: 040f5b2771c9b0037e64fc2b31f26e77855cab32c40406f1a978ec52cd8379df348b3feef701dc9e051db65b931bc8764a7e3067c9491547298365f34ca448ce

data/lib/rbbt/persist.rb

@@ -45,7 +45,7 @@ module Persist
       if Array === check
         newer = check.select{|file| newer? path, file}
         if newer.any?
-          Log.medium "Persistence check for #{path} failed in: #{ newer * ", "}"
+          Log.medium "Persistence check for #{path} failed in: #{ Misc.fingerprint(newer)}"
           return false 
         end
      else

data/lib/rbbt/tsv/stream.rb

@@ -156,7 +156,7 @@ module TSV
                   parts[i] = p.collect{|e| e.nil? ? "" : e }
                 end
               rescue TryAgain
-                Log.warn "Skipping repeated key in stream #{i}: #{keys[i]}"
+                Log.debug "Skipping repeated key in stream #{i}: #{keys[i]}"
                 retry
               end
             else

data/lib/rbbt/util/misc/inspect.rb

@@ -227,7 +227,7 @@ module Misc
           when File 
             "<File:" << obj.path << ">"
           when (defined? Step and Step)
-            "<Step #{obj.path}>"
+            "<IO:" << obj.path << ">"
           else
             if obj.respond_to? :filename and obj.filename
               "<IO:" << obj.filename << ">"

data/lib/rbbt/workflow.rb

@@ -302,6 +302,7 @@ module Workflow
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
 
     real_inputs = {}
+    recursive_inputs = rec_inputs(taskname)
 
     inputs.each do |k,v|
       default = defaults[k]

data/lib/rbbt/workflow/accessor.rb

@@ -493,30 +493,45 @@ module Workflow
     @rec_dependencies ||= {}
     @rec_dependencies[taskname] ||= begin
                             if task_dependencies.include? taskname
+
                               deps = task_dependencies[taskname]
-                              all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
+
+                              #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
+
+                              all_deps = []
                               deps.each do |dep| 
+                                if DependencyBlock === dep
+                                  all_deps << dep.dependency if dep.dependency
+                                else
+                                  all_deps << dep unless Proc === dep
+                                end
                                 case dep
                                 when Array
                                   wf, t, o = dep
+
                                   wf.rec_dependencies(t).each do |d|
                                     if Array === d
                                       new = d.dup
                                     else
                                       new = [dep.first, d]
                                     end
+
                                     if Hash === o and not o.empty? 
                                       if Hash === new.last
-                                        hash = new.last
+                                        hash = new.last.dup
                                         o.each{|k,v| hash[k] ||= v}
+                                        new[new.length-1] = hash
                                       else
-                                        new.push o
+                                        new.push o.dup
                                       end
                                     end
+
                                     all_deps << new
                                   end
+
                                 when String, Symbol
-                                  all_deps.concat rec_dependencies(dep.to_sym)
+                                  rec_deps = rec_dependencies(dep.to_sym)
+                                  all_deps.concat rec_deps
                                 when DependencyBlock
                                   all_deps << dep.dependency if dep.dependency
                                   case dep.dependency
@@ -558,14 +573,15 @@ module Workflow
 
   def rec_inputs(taskname)
     task = task_from_dep(taskname)
-    dep_inputs = task.dep_inputs rec_dependencies(taskname), self
+    deps = rec_dependencies(taskname)
+    dep_inputs = task.dep_inputs deps, self
     task.inputs + dep_inputs.values.flatten
   end
 
   def rec_input_defaults(taskname)
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject(IndiferentHash.setup({})){|acc, tn|
-      if Array === tn and tn.first
+      if Array === tn and tn[0] and tn[1]
         new = tn.first.tasks[tn[1].to_sym].input_defaults
       elsif Symbol === tn
         new = tasks[tn.to_sym].input_defaults
@@ -581,7 +597,7 @@ module Workflow
   def rec_input_types(taskname)
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
-      if Array === tn and tn.first
+      if Array === tn and tn[0] and tn[1]
         new = tn.first.tasks[tn[1].to_sym].input_types
       elsif Symbol === tn
         new = tasks[tn.to_sym].input_types
@@ -597,7 +613,7 @@ module Workflow
   def rec_input_descriptions(taskname)
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
-      if Array === tn and tn.first
+      if Array === tn and tn[0] and tn[1]
         new = tn.first.tasks[tn[1].to_sym].input_descriptions
       elsif Symbol === tn
         new = tasks[tn.to_sym].input_descriptions
@@ -613,7 +629,7 @@ module Workflow
   def rec_input_options(taskname)
     rec_inputs = rec_inputs(taskname)
     [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
-      if Array === tn and tn.first
+      if Array === tn and tn[0] and tn[1]
         new = tn.first.tasks[tn[1].to_sym].input_options
       elsif Symbol === tn
         new = tasks[tn.to_sym].input_options
@@ -646,10 +662,14 @@ module Workflow
             rec_dependency = all_d.select{|d| d.task_name.to_sym == v }.first
 
             if rec_dependency.nil?
-              _inputs[i] = v unless _inputs.include? i
+              if inputs.include? v
+                _inputs[i] = _inputs.delete(v)
+              else
+                _inputs[i] = v unless _inputs.include? i
+              end
             else
               input_options = workflow.task_info(dep_task)[:input_options][i] || {}
-              if true or input_options[:stream]
+              if input_options[:stream]
                 #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
                 _inputs[i] = rec_dependency
               else

data/lib/rbbt/workflow/definition.rb

@@ -38,11 +38,13 @@ module Workflow
     @dependencies ||= []
     if block_given?
       if dependency.any?
-        wf, task, opt = dependency
 
-        opt, wf = wf, nil if Hash === wf
-        wf, task = self, wf if task.nil? and wf
-        DependencyBlock.setup block, [wf, task, opt] 
+        wf, task_name, options = dependency
+        options, task_name = task_name, nil if Hash === task_name
+        options, wf = wf, nil if Hash === wf
+        task_name, wf = wf, self if task_name.nil?
+
+        DependencyBlock.setup block, [wf, task_name, options] 
       end
       @dependencies << block
     else

data/lib/rbbt/workflow/step/dependencies.rb

@@ -77,7 +77,9 @@ class Step
     (job.streaming? and job.running?) or 
     (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
 
-    job.clean if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
+    if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
+      job.clean 
+    end
 
     job.dup_inputs unless job.done? or job.started?
 
@@ -165,9 +167,10 @@ class Step
         stream = step.result
         other_steps = dep_step[step.path]
         return unless other_steps.length > 1
+        log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
-        log_dependency_exec(step, "duplicating #{copies.length}") 
         other_steps.zip(copies).each do |other,dupped_stream|
+          stream.annotate(dupped_stream) if stream.respond_to?(:annotate)
           other.instance_variable_set("@result", dupped_stream)
         end
       end

data/lib/rbbt/workflow/task.rb

@@ -75,10 +75,15 @@ module Task
     seen = []
     task_inputs = {}
     deps.each do |dep|
-      if Array === dep and dep.first
-        wf, task = (Array === dep ? [dep.first, dep.first.tasks[dep[1].to_sym]] : [workflow, workflow.tasks[dep.to_sym]])
-      elsif Symbol === dep 
+      if Symbol === dep
         wf, task = [workflow, workflow.tasks[dep.to_sym]]
+      elsif Array === dep and dep.first
+        wf, task_name, options = dep
+        options, task_name = task_name, nil if Hash === task_name
+        options, wf = wf, nil if Hash === wf
+        task_name, wf = wf, workflow if task_name.nil? and Symbol === wf or String === wf
+        next if task_name.nil?
+        task = wf.tasks[task_name.to_sym]
       else 
         next
       end
@@ -88,12 +93,17 @@ module Task
       seen << [wf, task.name]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
-      task_inputs[task] = new_inputs
+      if task_inputs[task].nil?
+        task_inputs[task] = new_inputs
+      else
+        task_inputs[task] = (task_inputs[task] + new_inputs).uniq
+      end
     end
     task_inputs
   end
 
   def dep_inputs(deps, workflow = nil)
+    return {} if deps.empty?
     task_inputs = Task.dep_inputs deps, workflow
     task_inputs.each do |task, inputs|
       inputs.replace (inputs - self.inputs)

data/test/rbbt/test_workflow.rb

@@ -57,6 +57,25 @@ Returns numer * 2 lines containing TEST
 
   export_synchronous :double
   export_asynchronous :repeat2
+
+  input :letter, :string, "Letter", "D"
+  task :letter => :string do |l|
+    l
+  end
+
+  dep :letter
+  task :letter_repeat => :string do |l|
+    ([step(:letter).load] * 2) * ""
+  end
+
+  dep :letter
+  dep :letter_repeat, :letter => "A"
+  task :two_letters => :string do
+    dependencies.collect{|d| d.load } * ":"
+  end
+
+
+
 end
 
 TestWF.workdir = Rbbt.tmp.test.workflow
@@ -125,4 +144,12 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal 'bar', job.exec
   end
 
+  def test_letter
+    assert_equal "D", TestWF.job(:letter).run
+    assert_equal "B", TestWF.job(:letter, nil, :letter => "B").run
+    assert_equal "BB", TestWF.job(:letter_repeat, nil, :letter => "B").run
+    job = TestWF.job(:two_letters, nil, :letter => "V")
+    assert_equal "V:AA", job.run
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.20.7
+  version: 5.20.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-07 00:00:00.000000000 Z
+date: 2016-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake