rbbt-util 5.37.16 → 5.38.1

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -1,6 +1,6 @@
 class RemoteStep
   module SSH
-    attr_accessor :override_dependencies, :run_type, :slurm_options
+    attr_accessor :override_dependencies, :run_type, :slurm_options, :produce_dependencies
 
     def init_job(cache_type = nil, other_params = {})
       return self if @url
@@ -52,11 +52,12 @@ class RemoteStep
     end
     
     def _run
-      RemoteWorkflow::SSH.upload_dependencies(self, @server)
+      RemoteWorkflow::SSH.upload_dependencies(self, @server, 'user', @produce_dependencies)
       RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
     end
 
     def _run_slurm
+      RemoteWorkflow::SSH.upload_dependencies(self, @server, 'user', @produce_dependencies)
       RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
     end
 
@@ -64,7 +65,7 @@ class RemoteStep
       RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
     end
 
-    def produce(*args)
+    def issue
       input_types = {}
       init_job
       @remote_path = case @run_type
@@ -76,6 +77,10 @@ class RemoteStep
                        _orchestrate_slurm
                      end
       @started = true
+    end
+
+    def produce(*args)
+      issue
       while ! (done? || error? || aborted?)
         sleep 1
       end
@@ -87,9 +92,13 @@ class RemoteStep
       load_res Open.open(path)
     end
 
-    def run(*args)
-      produce(*args)
-      self.load unless args.first
+    def run(stream = nil)
+      if stream
+        issue
+      else
+        produce
+        self.load unless args.first
+      end
     end
 
     def clean
@@ -107,7 +116,7 @@ class RemoteStep
         select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
         inputs.values.flatten.
         select{|dep| Path === dep && Step === dep.resource }.
-        select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+        #select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
         collect{|dep| dep.resource }
     end
   end

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -188,6 +188,7 @@ class RemoteStep < Step
   end
 
   def file(file)
+    init_job
     @adaptor.get_raw(File.join(url, 'file', file.to_s))
   end
 

data/lib/rbbt/workflow/step/dependencies.rb

@@ -107,7 +107,7 @@ class Step
       canfail = ComputeDependency === job && job.canfail?
     end
 
-    raise_dependency_error(job) if job.error? and not canfail
+    Step.raise_dependency_error(job) if job.error? and not canfail
   end
 
   def self.raise_dependency_error(job)
@@ -149,7 +149,7 @@ class Step
       select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
       recursive_inputs.flatten.
       select{|dep| Path === dep && Step === dep.resource }.
-      select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+      #select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
       collect{|dep| dep.resource }
   end
 
@@ -193,7 +193,7 @@ class Step
 
       if dependency.error?
         log_dependency_exec(dependency, :error)
-        raise_dependency_error dependency 
+        Step.raise_dependency_error dependency 
       end
 
       if dependency.streaming?
@@ -382,7 +382,7 @@ class Step
 
   def run_dependencies
 
-    rec_dependencies = self.rec_dependencies(true) + input_dependencies
+    rec_dependencies = self.rec_dependencies(true) + input_dependencies.reject{|d| d.started? }
 
     return if rec_dependencies.empty?
 
@@ -402,7 +402,7 @@ class Step
       seen_paths << step.path
 
       begin
-        Step.prepare_for_execution(step) unless step == self
+        Step.prepare_for_execution(step) unless step == self 
       rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
@@ -423,6 +423,7 @@ class Step
 
     produced = []
     (dependencies + input_dependencies).each do |dep|
+      next if dep.started?
       next unless ComputeDependency === dep
       if dep.compute == :produce
         dep.produce 

data/lib/rbbt/workflow/step/info.rb

@@ -132,7 +132,7 @@ class Step
       messages
     else
       set_info(:messages, []) if self.respond_to?(:set_info)
-    end
+    end || []
   end
 
   def message(message)

data/lib/rbbt/workflow/step/run.rb

@@ -247,7 +247,7 @@ class Step
           @exec = false
           init_info(true)
 
-          workflow           = @workflow || @task.respond_to?(:workflow)  ?  @task.workflow : nil
+          #workflow           = @workflow || @task.respond_to?(:workflow)  ?  @task.workflow : nil
           result_type        = @task.respond_to?(:result_type)            ?  @task.result_type : nil
           result_description = @task.respond_to?(:result_description)     ?  @task.result_description : nil
 

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -49,9 +49,9 @@ module Workflow
         when :nofile
           inputs[input.to_sym]  = Open.realpath(file)
         when :path_array
-          inputs[input.to_sym]  = Open.read(file).strip.split("\n")
+          inputs[input.to_sym]  = Open.read(file).strip.split("\n").collect{|p| Path.setup(p) }
         when :path
-          inputs[input.to_sym]  = Open.read(file).strip.split("\n").first
+          inputs[input.to_sym]  = Path.setup(Open.read(file).strip.split("\n").first)
         when :io
           inputs[input.to_sym] = Open.open(Open.realpath(file))
         when :io_array
@@ -67,9 +67,9 @@ module Workflow
           inputs[input.to_sym] = steps.first
         when :step_file
           path = Open.read(file).strip
-          path.extend Path
-          step_path = path.match(/(.*)\.files/)[1]
-          path.resource = Step.new step_path
+          step_path, relative = path.match(/(.*)\.files\/(.*)/).values_at 1, 2
+          step = Step.new Path.setup(step_path).find
+          path = step.file(relative)
           inputs[input.to_sym] = path
         when :step_file_array
           paths = Open.read(file).split("\n")
@@ -149,12 +149,14 @@ class Step
     case value
     when Path
       if Step === value.resource
+        step = value.resource
+        value = File.join('var/jobs', step.workflow.to_s, step.short_path + '.files', Misc.path_relative_to(step.files_dir, value))
         path = path + '.as_step_file'
       else
         path = path + '.as_path'
       end
     when String
-      if Misc.is_filename?(value, false)
+      if Misc.is_filename?(value, true)
         value = value.dup
         value.extend Path
         return save_input(name, value, type, dir)
@@ -173,7 +175,7 @@ class Step
           path = path + '.as_path_array'
         end
       when String
-        if Misc.is_filename?(value.first, false)
+        if Misc.is_filename?(value.first, true)
           path = path + '.as_path_array'
         end
       when IO
@@ -249,5 +251,4 @@ class Step
 
     inputs.keys
   end
-
 end

data/lib/rbbt/workflow/step.rb

@@ -57,7 +57,12 @@ class Step
   end
 
   def workflow
-    @workflow || info[:workflow]
+    @workflow ||= begin
+                    wf = info[:workflow] 
+                    wf = nil if wf == ""
+                    wf ||= @task.workflow if @task && @task.respond_to?(:workflow)
+                    wf ||= path.split("/")[-3]
+                  end
   end
 
 
@@ -296,7 +301,7 @@ class Step
               res = @result
             else
               join if not done?
-              res = @path.exists? ? Persist.load_file(@path, result_type) : exec
+              res = @path.exists? ? Persist.load_file(@path, result_type) : run
             end
 
             if result_description

data/lib/rbbt/workflow/usage.rb

@@ -128,7 +128,7 @@ module Workflow
       break if child
 
       if child
-        description << "-> " << task_name.to_s
+        description << "->" << task_name.to_s
       elsif first
         description << "" << task_name.to_s
       else
@@ -201,10 +201,10 @@ module Workflow
         description = description.split("\n\n").first
 
         next if abridge && ! final.include?(name)
-        puts Misc.format_definition_list_item(name.to_s, description, Log.tty_size || 80, 20, :yellow)
+        puts Misc.format_definition_list_item(name.to_s, description, Log.tty_size || 80, 30, :yellow)
 
         prov_string = prov_string(dep_tree(name))
-        puts Misc.format_paragraph Log.color(:blue, "-> " + prov_string) if prov_string && ! prov_string.empty?
+        puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
       end
 
     else

data/lib/rbbt/workflow.rb

@@ -486,7 +486,7 @@ module Workflow
         if dep_basename.include? "."
           parts = dep_basename.split(".")
           extension = [parts.pop]
-          while parts.last.length <= 4
+          while parts.length > 1 && parts.last.length <= 4
             extension << parts.pop
           end
           extension = extension.reverse * "."