rbbt-util 5.35.4 → 5.37.0

data/lib/rbbt/workflow/step/dependencies.rb

@@ -3,7 +3,7 @@ class Step
   STREAM_CACHE = {}
   STREAM_CACHE_MUTEX = Mutex.new
   def self.purge_stream_cache
-    Log.debug "Purging dup. stream cache"
+    # Log.debug "Purging dup. stream cache"
     STREAM_CACHE_MUTEX.synchronize do
       STREAM_CACHE.clear
     end
@@ -145,11 +145,12 @@ class Step
   end
 
   def input_dependencies
-    @input_dependencies ||= (recursive_inputs.flatten.select{|i| Step === i } + 
-                             recursive_inputs.flatten.
-                              select{|dep| Path === dep && Step === dep.resource }.
-                              select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
-                              collect{|dep| dep.resource })
+    @input_dependencies ||= recursive_inputs.flatten.
+      select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
+      recursive_inputs.flatten.
+      select{|dep| Path === dep && Step === dep.resource }.
+      select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+      collect{|dep| dep.resource }
   end
 
   def execute_dependency(dependency, log = true)
@@ -209,7 +210,7 @@ class Step
       end
 
     rescue TryAgain
-      Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
+      #Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
       retry
     rescue Aborted, Interrupt
       Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
@@ -260,6 +261,9 @@ class Step
           dep.produce
         rescue RbbtException
           Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last if dep.messages}"
+        rescue Exception
+          Log.warn "Not Allowing failing of #{dep.path} because #{$!.class} not RbbtException"
+          raise $!
         end
         nil
       end
@@ -430,7 +434,6 @@ class Step
 
     required_dep_paths = []
     dep_step.each do |path,list|
-      #required_dep_paths << path if list.length > 1
       required_dep_paths << path if (list & dependencies).any?
     end
 
@@ -442,42 +445,48 @@ class Step
       [dep.path] + dep.input_dependencies
     end.flatten)
 
-    log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
 
     pre_deps = []
-    compute_pre_deps = {}
-    last_deps = []
+    simple_dependencies = []
+    compute_simple_dependencies = {}
     compute_last_deps = {}
     seen_paths = Set.new
-    rec_dependencies.uniq.each do |step| 
+    rec_dependencies.uniq.reverse.each do |step| 
       next if seen_paths.include? step.path
       seen_paths << step.path
       next unless required_dep_paths.include? step.path
-      if step.inputs.flatten.select{|i| Step === i}.any?
-        if ComputeDependency === step
-          next if produced.include? step.path 
+      required_seen_paths = seen_paths & required_dep_paths
+
+      inputs = step.inputs
+      inputs = inputs.values if Hash === inputs
+      internal = inputs.select{|i| i.respond_to?(:path) && required_seen_paths.include?(i.path) }.any?
+
+      if ComputeDependency === step 
+        next if produced.include? step.path
+        if internal
           compute_last_deps[step.compute] ||= []
           compute_last_deps[step.compute] << step
         else
-          last_deps << step
+          compute_simple_dependencies[step.compute] ||= []
+          compute_simple_dependencies[step.compute] << step
         end
       else
-        if ComputeDependency === step
-          next if produced.include? step.path 
-          compute_pre_deps[step.compute] ||= []
-          compute_pre_deps[step.compute] << step
+        if internal
+          simple_dependencies << step
         else
-          pre_deps << step #if dependencies.include?(step)
+          simple_dependencies.prepend(step)
         end
       end
     end
 
-    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
-    compute_pre_deps.each do |type,list|
+    log :dependencies, "Processing dependencies for #{Log.color :yellow, task_name.to_s || ""}" if compute_simple_dependencies.any? || simple_dependencies.any? || compute_last_deps.any?
+
+    Log.debug "compute_simple_dependencies: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
+    compute_simple_dependencies.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
 
-    Log.medium "Processing pre dependencies: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
+    Log.low "pre_deps: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     pre_deps.each do |step|
       next if compute_deps.include? step
       begin
@@ -487,8 +496,8 @@ class Step
       end
     end
 
-    Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if last_deps.any?
-    last_deps.each do |step|
+    Log.debug "simple_dependencies: #{Misc.fingerprint(simple_dependencies)} - #{Log.color :blue, self.path}" if simple_dependencies.any?
+    simple_dependencies.each do |step|
       next if compute_deps.include? step
       begin Exception
         execute_and_dup(step, dep_step) 
@@ -497,8 +506,8 @@ class Step
       end
     end
 
-    Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if compute_last_deps.any?
-    compute_last_deps.each do |type,list|
+    Log.low "compute_last_deps: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
+    compute_simple_dependencies.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
 

data/lib/rbbt/workflow/step/{prepare.rb → produce.rb}

@@ -1,5 +1,5 @@
 class Step
-  def self.prepare_dependencies(jobs, tasks, cpus)
+  def self.produce_dependencies(jobs, tasks, cpus)
     deps = []
 
     jobs = [jobs] unless Array === jobs

data/lib/rbbt/workflow/step/run.rb

@@ -96,7 +96,13 @@ class Step
     @exec = true if @exec.nil?
     begin
       old = Signal.trap("INT"){ Thread.current.raise Aborted }
-      @task.exec_in((bindings ? bindings : self), *@inputs)
+      if @task.respond_to?(:exec_in)
+        @task.exec_in((bindings || self), *@inputs)
+      elsif @task
+        (bindings || self).instance_exec *@inputs, &@task
+      else
+        raise DependencyError, "Dependency #{self.path} cannot be produced"
+      end
     ensure
       Signal.trap("INT", old)
     end
@@ -223,14 +229,15 @@ class Step
     end
 
     begin
-      time_elapsed = total_time_elapsed = nil
-      res = @mutex.synchronize do
-        no_load = :stream if no_load
+      no_load = :stream if no_load
+      result_type = self.result_type || info[:result_type]
 
+      res = @mutex.synchronize do
+        time_elapsed = total_time_elapsed = nil
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result_type = @task.result_type if @task
-        result_type = info[:result_type] if result_type.nil?
+
         result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do 
+
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
           end
@@ -240,26 +247,30 @@ class Step
           @exec = false
           init_info(true)
 
-          log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
+          workflow           = @workflow || @task.respond_to?(:workflow)  ?  @task.workflow : nil
+          result_type        = @task.respond_to?(:result_type)            ?  @task.result_type : nil
+          result_description = @task.respond_to?(:result_description)     ?  @task.result_description : nil
+
+          log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task_name}"
 
           merge_info({
-            :issued => (issue_time = Time.now),
-            :name => name,
-            :pid => Process.pid.to_s,
-            :pid_hostname => Socket.gethostname,
-            :clean_name => clean_name,
-            :workflow => (@workflow || @task.workflow).to_s,
-            :task_name => @task.name,
-            :result_type => @task.result_type,
-            :result_description => @task.result_description,
-            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
-            :versions => Rbbt.versions
+            :issued             => (issue_time = Time.now),
+            :name               => name,
+            :pid                => Process.pid.to_s,
+            :pid_hostname       => Socket.gethostname,
+            :clean_name         => clean_name,
+            :workflow           => workflow.to_s,
+            :task_name          => task_name,
+            :result_type        => result_type,
+            :result_description => result_description,
+            :dependencies       => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
+            :versions           => Rbbt.versions
           })
 
           new_inputs = []
           @inputs.each_with_index do |input,i|
-            name = @task.inputs[i]
-            type = @task.input_types[name]
+            name = @task.respond_to?(:inputs)       ?  @task.inputs[i] : nil
+            type = @task.respond_to?(:input_types)  ?  @task.input_types[i] : nil
 
             if type == :directory
               directory_inputs = file('directory_inputs')
@@ -303,7 +314,7 @@ class Step
 
           @inputs = new_inputs if @inputs
 
-          if @inputs && ! task.inputs.nil?
+          if @inputs && task.respond_to?(:inputs) && ! task.inputs.nil?
             info_inputs = @inputs.collect do |i| 
               if Path === i 
                 i.to_s
@@ -323,7 +334,7 @@ class Step
           end
 
           set_info :started, (start_time = Time.now)
-          log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
+          log :started, "Starting step #{Log.color :yellow, task_name}"
 
           config_keys_pre = Rbbt::Config::GOT_KEYS.dup
           begin
@@ -357,7 +368,7 @@ class Step
                    end
 
           if stream
-            log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
+            log :streaming, "Streaming step #{Log.color :yellow, task_name.to_s || ""}"
 
             callback = Proc.new do
               if AbortedStream === stream
@@ -377,7 +388,7 @@ class Step
                       :time_elapsed => (time_elapsed = done_time - start_time),
                       :versions => Rbbt.versions
                     })
-                    log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
+                    log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
                   end
                 end
               rescue
@@ -393,7 +404,7 @@ class Step
                 if exception
                   self.exception exception
                 else
-                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task_name.to_s || ""}" if status == :streaming
                 end
                 _clean_finished
               rescue
@@ -419,7 +430,6 @@ class Step
             })
             log :ending
             Step.purge_stream_cache
-            Open.rm pid_file if Open.exist?(pid_file)
           end
 
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
@@ -430,16 +440,17 @@ class Step
           if result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
             Open.mv self.tmp_path, self.path
           end
+          Open.rm pid_file if Open.exist?(pid_file) unless stream
           result
         end # END PERSIST
-        log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
+        log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
 
         if no_load
           @result ||= result
           self
         else
           Step.purge_stream_cache
-          @result = prepare_result result, @task.result_description
+          @result = prepare_result result, result_description
         end
       end # END SYNC
       res

data/lib/rbbt/workflow/step/status.rb

@@ -98,7 +98,7 @@ class Step
   end
 
   def done?
-    @done ||= path and Open.exists? path
+    @done ||= path and Open.exists?(path)
   end
 
   def streaming?

data/lib/rbbt/workflow/step.rb

@@ -3,20 +3,12 @@ require 'rbbt/persist/tsv'
 require 'rbbt/util/log'
 require 'rbbt/util/semaphore'
 require 'rbbt/workflow/step/accessor'
-require 'rbbt/workflow/step/prepare'
+require 'rbbt/workflow/step/produce'
 require 'rbbt/workflow/step/status'
 require 'rbbt/workflow/step/info'
 require 'rbbt/workflow/step/save_load_inputs'
 
 class Step
-  attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
-  attr_accessor :task_name, :overriden
-  attr_accessor :pid
-  attr_accessor :exec
-  attr_accessor :relocated
-  attr_accessor :result, :mutex, :seen
-  attr_accessor :real_inputs, :original_task_name, :original_workflow
-
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
   class << self
@@ -38,10 +30,12 @@ class Step
   end
 
 
-  def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
+  def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil, &block)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
     path = path.call if Proc === path
 
+    task = block if block_given?
+
     @path = path
     @task = task
     @bindings = bindings
@@ -73,7 +67,8 @@ class Step
 
     load_inputs_from_info if @inputs.nil? 
 
-    NamedArray.setup(@inputs, task.inputs) if task && task.inputs && !(NamedArray === @inputs)
+    task_inputs = task.respond_to?(:inputs) ? task.inputs : nil
+    NamedArray.setup(@inputs, task_inputs) unless NamedArray === @inputs
 
     @inputs || []
   end
@@ -133,15 +128,6 @@ class Step
     v
   end
 
-  def task_name
-    @task_name ||= begin
-                     if @task.nil?
-                        @path.split("/")[-2]
-                     else
-                       @task.name
-                     end
-                   end
-  end
 
   def path
     if Proc === @path
@@ -155,8 +141,11 @@ class Step
     attr_accessor :log_relay_step
   end
 
+  
+  # May be deprecated: This is no loger used
   def relay_log(step)
-    return self unless Task === self.task and not self.task.name.nil?
+    return self if self.task_name.nil?
+
     if not self.respond_to? :original_log
       class << self
         attr_accessor :relay_step
@@ -164,77 +153,58 @@ class Step
         def log(status, message = nil)
           self.status = status
           message Log.uncolor message
-          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
+          relay_step.log([self.task_name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
         end
       end
     end
     @relay_step = step
-    self
-  end
 
-  def result_type
-    @result_type ||= if @task.nil?
-                       info[:result_type] || :binary
-                     else
-                       @task.result_type || info[:result_type] || :string
-                     end
-  end
-
-  def result_type=(type)
-    @result_type = type
-  end
-
-  def result_description
-    @result_description ||= if @task.nil?
-                       info[:result_description]
-                     else
-                       @task.result_description
-                     end
+    self
   end
 
   def prepare_result(value, description = nil, entity_info = nil)
     res = case 
-    when IO === value
-      begin
-        res = case result_type
-              when :array
-                array = []
-                while line = value.gets
-                  array << line.chomp
-                end
-                array
-              when :tsv
-                begin
-                  TSV.open(value)
-                rescue IOError
-                  TSV.setup({})
-                end
-              else
-                value.read
-              end
-        value.join if value.respond_to? :join
-        res
-      rescue Exception
-        value.abort if value.respond_to? :abort
-        self.abort
-        raise $!
-      end
-    when (not defined? Entity or description.nil? or not Entity.formats.include? description)
-      value
-    when (Annotated === value and info.empty?)
-      value
-    when Annotated === value
-      annotations = value.annotations
-      entity_info ||= begin 
-                        entity_info = info.dup
-                        entity_info.merge! info[:inputs] if info[:inputs]
-                        entity_info
+          when IO === value
+            begin
+              res = case result_type
+                    when :array
+                      array = []
+                      while line = value.gets
+                        array << line.chomp
                       end
-      entity_info.each do |k,v|
-        value.send("#{h}=", v) if annotations.include? k
-      end
-                        
-      value
+                      array
+                    when :tsv
+                      begin
+                        TSV.open(value)
+                      rescue IOError
+                        TSV.setup({})
+                      end
+                    else
+                      value.read
+                    end
+              value.join if value.respond_to? :join
+              res
+            rescue Exception
+              value.abort if value.respond_to? :abort
+              self.abort
+              raise $!
+            end
+          when (not defined? Entity or description.nil? or not Entity.formats.include? description)
+            value
+          when (Annotated === value and info.empty?)
+            value
+          when Annotated === value
+            annotations = value.annotations
+            entity_info ||= begin 
+                              entity_info = info.dup
+                              entity_info.merge! info[:inputs] if info[:inputs]
+                              entity_info
+                            end
+            entity_info.each do |k,v|
+              value.send("#{h}=", v) if annotations.include? k
+            end
+
+            value
     else
       entity_info ||= begin 
                         entity_info = info.dup
@@ -371,8 +341,9 @@ class Step
   def step(name)
     @steps ||= {}
     @steps[name] ||= begin
+                       name = name.to_sym
                        deps = rec_dependencies.select{|step| 
-                         step.task_name.to_sym == name.to_sym
+                         step.task_name && step.task_name.to_sym == name
                        }
                        raise "Dependency step not found: #{ name }" if deps.empty?
                        if (deps & self.dependencies).any?
@@ -385,3 +356,4 @@ class Step
 end
 
 require 'rbbt/workflow/step/run'
+require 'rbbt/workflow/step/accessor'

data/lib/rbbt/workflow/task.rb

@@ -88,13 +88,15 @@ module Task
         next
       end
 
+      task_name ||= task.name
+
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task.name, maps]
+      next if seen.include? [wf, task_name, maps]
 
       task.workflow = wf if wf
 
-      seen << [wf, task.name, maps]
+      seen << [wf, task_name, maps]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
       if task_inputs[task].nil?

data/lib/rbbt/workflow/util/archive.rb

@@ -160,44 +160,50 @@ puts files * "\n"
     end
   end
 
-  def self.migrate(path, search_path, options = {})
-    if Step === path
-      if options[:source]
-        path = Rbbt.identify(path.path)
-      else
-        path = path.path
+  def self.migrate(paths, search_path, options = {})
+    subpath_files = {}
+    target_paths = []
+    resource = options[:resource] || Rbbt
+
+    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
+
+    (Array === paths ? paths : [paths]).each do |path|
+      if Step === path
+        if options[:source]
+          path = Rbbt.identify(path.path)
+        else
+          path = path.path
+        end
       end
-    end
-    search_path = 'user' if search_path.nil?
+      search_path = 'user' if search_path.nil?
 
-    resource = Rbbt
 
-    path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
+      path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
 
-    subpath_files = {}
-    real_paths.sort.each do |path|
-      parts = path.split("/")
-      subpath = parts[0..-4] * "/" + "/"
+      real_paths.sort.each do |path|
+        parts = path.split("/")
+        subpath = parts[0..-4] * "/" + "/"
+
+        if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
+          subpath = subpath_files.keys.last
+        end
 
-      if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
-        subpath = subpath_files.keys.last
+        source = path.chars[subpath.length..-1] * ""
+
+        subpath_files[subpath] ||= []
+        subpath_files[subpath] << source
       end
 
-      source = path.chars[subpath.length..-1] * ""
 
-      subpath_files[subpath] ||= []
-      subpath_files[subpath] << source
+      target_paths << File.join(target, *path.split("/")[-3..-1])
     end
 
-    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
-
-    target_path = File.join(target, *path.split("/")[-3..-1])
 
     subpath_files.each do |subpath, files|
       Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
 
-    target_path
+    target_paths
   end
 
   def self.purge(path, recursive = false, skip_overriden = true)

data/lib/rbbt/workflow.rb

@@ -212,11 +212,13 @@ module Workflow
                  end
       workflow.load_documentation
 
+
       first ||= workflow
     end
-    return first
 
-    workflow
+    first.complete_name = wf_name
+
+    return first
   end
 
   attr_accessor :description
@@ -230,6 +232,12 @@ module Workflow
   attr_accessor :relay_tasks
 
   #{{{ ATTR DEFAULTS
+  #
+  attr_accessor :complete_name
+
+  def self.complete_name
+    @complete_name ||= self.to_s
+  end
   
   def self.workdir=(path)
     path = Path.setup path.dup unless Path === path
@@ -482,8 +490,10 @@ module Workflow
       end
     end
 
+    overriden = true if dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
+
     input_values = task.take_input_values(inputs)
-    if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
+    if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden #&& ! dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
       step_path = step_path taskname, jobname, [], [], extension
     else
       step_path = step_path taskname, jobname, input_values, dependencies, extension