rbbt-util 5.35.4 → 5.36.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3a214b340c407e61c0cc227b8872c4b0c71463affcf6bf84f9d4ed66486435c
-  data.tar.gz: d62735c060afe4507cc69363280d76e24c88ed89063531e526fe5a03ecfd6386
+  metadata.gz: d105d6c21ad018f28acf6a234feae8d81f327f3f94f228c01f118abed6c386f6
+  data.tar.gz: 5c883e0611f02203d394ec02e448664efdc42361be04e9d873e9ba3f5e6cc92e
 SHA512:
-  metadata.gz: f8fd64ecb9abc0a5bf519fa294196e8b1082a4ef358106806bc7c18dadfb361bd22b2b14aca96f3dd32c276461fa6616d3613a7de7aca87ceaddc7db74d57e23
-  data.tar.gz: baa4182336b92c09eead8fed9a28b2bc6d84aea7e2e9e98c4597ae3373ac31ff27feb62e1dbd73817ac69f7986a00ceb2bf404d38fb739919882fea684053bfa
+  metadata.gz: 20a4bccdeb0c587041b7f20f09bda19949713fcb362911a249b08112399c09178f8e50b5ae90f390bd6cd1389b3c9a47b7374c4fc05a83b073f1d43bdfdc6611
+  data.tar.gz: 1b3b112106b4ac1e261bcf1630ecaeea0ed27f1975aeead9694708241e9268fdac36da4f5b4be25c12288cb7f97b8b4163815b8d9010aba40f6f2434e654f99c

data/lib/rbbt/persist.rb

@@ -44,6 +44,8 @@ module Persist
   end
 
   def self.is_persisted?(path, persist_options = {})
+    return true if Open.remote?(path)
+    return true if Open.ssh?(path)
     return false if not Open.exists? path
     return false if TrueClass === persist_options[:update]
 

data/lib/rbbt/resource/path.rb

@@ -319,7 +319,8 @@ module Path
 
 
   def open(options = {}, &block)
-    Open.open(self.produce.find, options, &block)
+    file = Open.remote?(self) || Open.ssh?(self) ? self : self.produce.find
+    Open.open(file, options, &block)
   end
 
   def to_s
@@ -331,7 +332,16 @@ module Path
   end
 
   def tsv(*args)
-    TSV.open(self.produce, *args)
+    begin
+      path = self.produce
+    rescue Resource::ResourceNotFound => e
+      begin
+        path = self.set_extension('tsv').produce
+      rescue Resource::ResourceNotFound 
+        raise e
+      end
+    end
+    TSV.open(path, *args)
   end
 
   def tsv_options(options = {})

data/lib/rbbt/util/python.rb

@@ -59,12 +59,12 @@ module RbbtPython
   end
 
   def self.get_class(module_name, class_name)
-    save_module_name = module_name.gsub(".", "_")
+    save_module_name = module_name.to_s.gsub(".", "_")
     RbbtPython.pyimport(module_name, as: save_module_name)
     RbbtPython.send(save_module_name).send(class_name)
   end
 
-  def self.class_new_obj(module_name, class_name, args)
+  def self.class_new_obj(module_name, class_name, args={})
     RbbtPython.get_class(module_name, class_name).new(**args)
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -175,6 +175,8 @@ module Workflow
 
     tasks[name] = task
     task_dependencies[name] = consume_dependencies
+
+    task
   end
 
   def unexport(*names)

data/lib/rbbt/workflow/doc.rb

@@ -37,6 +37,7 @@ module Workflow
   def documentation_markdown
     return "" if @libdir.nil?
     file = @libdir['workflow.md'].find
+    file = @libdir['README.md'].find unless file.exists?
     if file.exists?
       file.read
     else

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -1,4 +1,6 @@
 class RemoteWorkflow
+  RBBT_DEBUG_REMOTE_JSON = ENV["RBBT_DEBUG_REMOTE_JSON"] == 'true'
+
   module SSH
     #def self.run(server, script)
     #  Log.debug "Run ssh script in #{server}:\n#{script}"
@@ -21,9 +23,16 @@ class RemoteWorkflow
 
       workflow, task, job, *rest = path.split("/")
 
+      workflow_name = begin
+                        workflow = Kernel.const_get(workflow) if String === workflow
+                        workflow.respond_to?(:complete_name) ? workflow.complete_name : workflow
+                      rescue
+                        workflow
+                      end
+
       script =<<-EOF
 require 'rbbt/workflow'
-wf = Workflow.require_workflow "#{workflow}"
+wf = Workflow.require_workflow "#{workflow_name}"
       EOF
 
       case task
@@ -86,7 +95,7 @@ STDOUT.write res.to_json
       EOF
 
       json = Misc.ssh_run(server, script)
-      Log.debug "JSON (#{ url }): #{json}"
+      Log.debug "JSON (#{ url }): #{json}" if RBBT_DEBUG_REMOTE_JSON
       JSON.parse(json)
     end
 
@@ -172,11 +181,17 @@ job.clean
     def self.upload_inputs(server, inputs, input_types, input_id)
       TmpFile.with_file do |dir|
         if Step.save_inputs(inputs, input_types, dir)
-          Dir.glob(File.join(dir, "*.as_step")).each do |file|
-            path = Open.read(file).strip
-            new = Step.migrate(path, :user, :target => server)
-            Open.write(file, new)
-          end
+          # Dir.glob(File.join(dir, "*.as_step")).each do |file|
+          #   Log.medium "Migrating Step input #{file} #{ server }" 
+          #   path = Open.read(file).strip
+          #   new = Step.migrate(path, :user, :target => server)
+          #   Open.write(file, new)
+          # end
+
+          paths = Dir.glob(File.join(dir, "*.as_step")).collect{|f| Open.read(f).strip }
+          new = Step.migrate(paths, :user, :target => server)
+          paths.zip(new).each{|file,new| Open.write(file, new) }
+
           CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end
@@ -204,31 +219,56 @@ job.clean
     #  rjob.run
     #end
 
+    def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
+      server, path = parse_url(server) if server =~ /^ssh:\/\//
+      job.dependencies.each do |dep|
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
+        dep.produce
+      end if produce_dependencies
+
+      job.input_dependencies.each do |dep|
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
+        dep.produce
+      end if produce_dependencies
+
+      migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
+      Log.medium "Migrating #{migrate_dependencies.length} dependencies to #{ server }" 
+      Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
+    end
+
+    def self.missing_dep_inputs(job)
+      inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
+      job.dependencies.each do |dep|
+        next if dep.done?
+        iif [dep, dep.inputs, dep.real_inputs]
+        inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
+        inputs = missing_dep_inputs(dep).merge(inputs)
+      end
+      inputs
+    end
+
     def self.relay_job(job, server, options = {})
-      migrate, produce, produce_dependencies, search_path = Misc.process_options options.dup,
-        :migrate, :produce, :produce_dependencies, :search_path
+      migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
+        :migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
 
       search_path ||= 'user'
 
       produce = true if migrate
 
       workflow_name = job.workflow.to_s
-      inputs = job.recursive_inputs.to_hash
-
-      job.dependencies.each do |dep|
-        dep.produce 
-      end if options[:produce_dependencies]
-
-      job.rec_dependencies.each do |dep|
-        Step.migrate(dep.path, search_path, :target => server) if dep.done?
-      end
+      remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
+      inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
+      Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
 
-      remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{job.workflow.to_s}", "#{job.workflow.to_s}")
+      upload_dependencies(job, server, search_path, options[:produce_dependencies])
       rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
 
       override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
       rjob.override_dependencies = override_dependencies
 
+      rjob.run_type = run_type
+      rjob.slurm_options = slurm_options || {}
+
       if options[:migrate]
         rjob.produce
         Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
@@ -255,7 +295,7 @@ job.clean
       @task_info ||= IndiferentHash.setup({})
 
       if @task_info[task].nil?
-        task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url, task.to_s))
+        task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url || @url, task.to_s))
         task_info = RemoteWorkflow::SSH.fix_hash(task_info)
 
         task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -1,6 +1,6 @@
 class RemoteStep
   module SSH
-    attr_accessor :override_dependencies
+    attr_accessor :override_dependencies, :run_type, :slurm_options
 
     def init_job(cache_type = nil, other_params = {})
       return self if @url
@@ -55,12 +55,27 @@ class RemoteStep
       RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
     end
 
+    def _run_slurm
+      RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
+    end
+
+    def _orchestrate_slurm
+      RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
+    end
+
     def produce(*args)
       input_types = {}
       init_job
-      @remote_path = _run
+      @remote_path = case @run_type
+                     when 'run', :run, nil
+                       _run
+                     when 'slurm', :slurm
+                       _run_slurm
+                     when 'orchestrate', :orchestrate
+                       _orchestrate_slurm
+                     end
       @started = true
-      while ! (done? || error?)
+      while ! (done? || error? || aborted?)
         sleep 1
       end
       raise self.get_exception if error?

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -163,6 +163,7 @@ class RemoteStep < Step
     begin
       status = info[:status]
       @done = true if status and status.to_sym == :done
+      Log.low "RemoteStep status '#{status}' #{self.url}"
       status
     rescue
       Log.exception $!

data/lib/rbbt/workflow/step/accessor.rb

@@ -1,4 +1,12 @@
 class Step
+  attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
+  attr_accessor :task_name, :overriden
+  attr_accessor :pid
+  attr_accessor :exec
+  attr_accessor :relocated
+  attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name, :original_workflow
+
 
   INFO_SERIALIZER = begin
                        if ENV["RBBT_INFO_SERIALIZER"]
@@ -86,12 +94,45 @@ class Step
     end
   end
 
+  def task_name
+    @task_name ||= begin
+                     if @task.respond_to?(:name)
+                     
+                     @task.name
+                     else
+                       @path.split("/")[-2]
+                     end
+                   end
+  end
+
+  def result_type
+    @result_type ||= if @task.respond_to?(:result_type)
+                       @task.result_type || info[:result_type]
+                     else
+                       info[:result_type]
+                     end
+  end
+
+  def result_type=(type)
+    @result_type = type
+  end
+
+  def result_description
+    @result_description ||= if @task.respond_to?(:result_description)
+                              @task.result_description
+                            else
+                              nil
+                            end
+  end
+
+  def result_description=(description)
+    @result_description = description
+  end
 
   def name
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
   end
 
-
   def short_path
     [task_name, name] * "/"
   end
@@ -105,10 +146,6 @@ class Step
     workflow.to_s + "#" + short_path
   end
 
-  def task_name
-    @task_name ||= task.name
-  end
-
   def task_signature
     [workflow.to_s, task_name] * "#"
   end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -3,7 +3,7 @@ class Step
   STREAM_CACHE = {}
   STREAM_CACHE_MUTEX = Mutex.new
   def self.purge_stream_cache
-    Log.debug "Purging dup. stream cache"
+    # Log.debug "Purging dup. stream cache"
     STREAM_CACHE_MUTEX.synchronize do
       STREAM_CACHE.clear
     end
@@ -209,7 +209,7 @@ class Step
       end
 
     rescue TryAgain
-      Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
+      #Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
       retry
     rescue Aborted, Interrupt
       Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
@@ -430,7 +430,6 @@ class Step
 
     required_dep_paths = []
     dep_step.each do |path,list|
-      #required_dep_paths << path if list.length > 1
       required_dep_paths << path if (list & dependencies).any?
     end
 
@@ -442,42 +441,46 @@ class Step
       [dep.path] + dep.input_dependencies
     end.flatten)
 
-    log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
 
     pre_deps = []
-    compute_pre_deps = {}
-    last_deps = []
+    simple_dependencies = []
+    compute_simple_dependencies = {}
     compute_last_deps = {}
     seen_paths = Set.new
     rec_dependencies.uniq.each do |step| 
       next if seen_paths.include? step.path
       seen_paths << step.path
       next unless required_dep_paths.include? step.path
-      if step.inputs.flatten.select{|i| Step === i}.any?
-        if ComputeDependency === step
-          next if produced.include? step.path 
+      required_seen_paths = seen_paths & required_dep_paths
+
+      internal = step.inputs.select{|i| Step == i && required_paths.include?(i.path) && seen_paths.include?(i.path) }
+
+      if ComputeDependency === step 
+        next if produced.include? step.path
+        if internal
           compute_last_deps[step.compute] ||= []
           compute_last_deps[step.compute] << step
         else
-          last_deps << step
+          compute_simple_dependencies[step.compute] ||= []
+          compute_simple_dependencies[step.compute] << step
         end
       else
-        if ComputeDependency === step
-          next if produced.include? step.path 
-          compute_pre_deps[step.compute] ||= []
-          compute_pre_deps[step.compute] << step
+        if internal
+          simple_dependencies.prepend(step)
         else
-          pre_deps << step #if dependencies.include?(step)
+          simple_dependencies << step
         end
       end
     end
 
-    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
-    compute_pre_deps.each do |type,list|
+    log :dependencies, "Processing dependencies for #{Log.color :yellow, task_name.to_s || ""}" if compute_simple_dependencies.any? || simple_dependencies.any? || compute_last_deps.any?
+
+    Log.debug "compute_simple_dependencies: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
+    compute_simple_dependencies.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
 
-    Log.medium "Processing pre dependencies: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
+    Log.low "pre_deps: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     pre_deps.each do |step|
       next if compute_deps.include? step
       begin
@@ -487,8 +490,8 @@ class Step
       end
     end
 
-    Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if last_deps.any?
-    last_deps.each do |step|
+    Log.debug "simple_dependencies: #{Misc.fingerprint(simple_dependencies)} - #{Log.color :blue, self.path}" if simple_dependencies.any?
+    simple_dependencies.each do |step|
       next if compute_deps.include? step
       begin Exception
         execute_and_dup(step, dep_step) 
@@ -497,8 +500,8 @@ class Step
       end
     end
 
-    Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if compute_last_deps.any?
-    compute_last_deps.each do |type,list|
+    Log.low "compute_last_deps: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
+    compute_simple_dependencies.each do |type,list|
       run_compute_dependencies(type, list, dep_step)
     end
 

data/lib/rbbt/workflow/step/{prepare.rb → produce.rb}

@@ -1,5 +1,5 @@
 class Step
-  def self.prepare_dependencies(jobs, tasks, cpus)
+  def self.produce_dependencies(jobs, tasks, cpus)
     deps = []
 
     jobs = [jobs] unless Array === jobs

data/lib/rbbt/workflow/step/run.rb

@@ -96,7 +96,11 @@ class Step
     @exec = true if @exec.nil?
     begin
       old = Signal.trap("INT"){ Thread.current.raise Aborted }
-      @task.exec_in((bindings ? bindings : self), *@inputs)
+      if @task.respond_to?(:exec_in)
+        @task.exec_in((bindings || self), *@inputs)
+      else
+        (bindings || self).instance_exec *@inputs, &@task
+      end
     ensure
       Signal.trap("INT", old)
     end
@@ -223,14 +227,15 @@ class Step
     end
 
     begin
-      time_elapsed = total_time_elapsed = nil
-      res = @mutex.synchronize do
-        no_load = :stream if no_load
+      no_load = :stream if no_load
+      result_type = self.result_type || info[:result_type]
 
+      res = @mutex.synchronize do
+        time_elapsed = total_time_elapsed = nil
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result_type = @task.result_type if @task
-        result_type = info[:result_type] if result_type.nil?
+
         result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do 
+
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
           end
@@ -240,26 +245,30 @@ class Step
           @exec = false
           init_info(true)
 
-          log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
+          workflow           = @workflow || @task.respond_to?(:workflow)  ?  @task.workflow : nil
+          result_type        = @task.respond_to?(:result_type)            ?  @task.result_type : nil
+          result_description = @task.respond_to?(:result_description)     ?  @task.result_description : nil
+
+          log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task_name}"
 
           merge_info({
-            :issued => (issue_time = Time.now),
-            :name => name,
-            :pid => Process.pid.to_s,
-            :pid_hostname => Socket.gethostname,
-            :clean_name => clean_name,
-            :workflow => (@workflow || @task.workflow).to_s,
-            :task_name => @task.name,
-            :result_type => @task.result_type,
-            :result_description => @task.result_description,
-            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
-            :versions => Rbbt.versions
+            :issued             => (issue_time = Time.now),
+            :name               => name,
+            :pid                => Process.pid.to_s,
+            :pid_hostname       => Socket.gethostname,
+            :clean_name         => clean_name,
+            :workflow           => workflow.to_s,
+            :task_name          => task_name,
+            :result_type        => result_type,
+            :result_description => result_description,
+            :dependencies       => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
+            :versions           => Rbbt.versions
           })
 
           new_inputs = []
           @inputs.each_with_index do |input,i|
-            name = @task.inputs[i]
-            type = @task.input_types[name]
+            name = @task.respond_to?(:inputs)       ?  @task.inputs[i] : nil
+            type = @task.respond_to?(:input_types)  ?  @task.input_types[i] : nil
 
             if type == :directory
               directory_inputs = file('directory_inputs')
@@ -303,7 +312,7 @@ class Step
 
           @inputs = new_inputs if @inputs
 
-          if @inputs && ! task.inputs.nil?
+          if @inputs && task.respond_to?(:inputs) && ! task.inputs.nil?
             info_inputs = @inputs.collect do |i| 
               if Path === i 
                 i.to_s
@@ -323,7 +332,7 @@ class Step
           end
 
           set_info :started, (start_time = Time.now)
-          log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
+          log :started, "Starting step #{Log.color :yellow, task_name}"
 
           config_keys_pre = Rbbt::Config::GOT_KEYS.dup
           begin
@@ -357,7 +366,7 @@ class Step
                    end
 
           if stream
-            log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
+            log :streaming, "Streaming step #{Log.color :yellow, task_name.to_s || ""}"
 
             callback = Proc.new do
               if AbortedStream === stream
@@ -377,7 +386,7 @@ class Step
                       :time_elapsed => (time_elapsed = done_time - start_time),
                       :versions => Rbbt.versions
                     })
-                    log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
+                    log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
                   end
                 end
               rescue
@@ -393,7 +402,7 @@ class Step
                 if exception
                   self.exception exception
                 else
-                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
+                  log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task_name.to_s || ""}" if status == :streaming
                 end
                 _clean_finished
               rescue
@@ -419,7 +428,6 @@ class Step
             })
             log :ending
             Step.purge_stream_cache
-            Open.rm pid_file if Open.exist?(pid_file)
           end
 
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
@@ -430,16 +438,17 @@ class Step
           if result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
             Open.mv self.tmp_path, self.path
           end
+          Open.rm pid_file if Open.exist?(pid_file) unless stream
           result
         end # END PERSIST
-        log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
+        log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
 
         if no_load
           @result ||= result
           self
         else
           Step.purge_stream_cache
-          @result = prepare_result result, @task.result_description
+          @result = prepare_result result, result_description
         end
       end # END SYNC
       res

data/lib/rbbt/workflow/step/status.rb

@@ -98,7 +98,7 @@ class Step
   end
 
   def done?
-    @done ||= path and Open.exists? path
+    @done ||= path and Open.exists?(path)
   end
 
   def streaming?

data/lib/rbbt/workflow/step.rb

@@ -3,20 +3,12 @@ require 'rbbt/persist/tsv'
 require 'rbbt/util/log'
 require 'rbbt/util/semaphore'
 require 'rbbt/workflow/step/accessor'
-require 'rbbt/workflow/step/prepare'
+require 'rbbt/workflow/step/produce'
 require 'rbbt/workflow/step/status'
 require 'rbbt/workflow/step/info'
 require 'rbbt/workflow/step/save_load_inputs'
 
 class Step
-  attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
-  attr_accessor :task_name, :overriden
-  attr_accessor :pid
-  attr_accessor :exec
-  attr_accessor :relocated
-  attr_accessor :result, :mutex, :seen
-  attr_accessor :real_inputs, :original_task_name, :original_workflow
-
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
   class << self
@@ -38,10 +30,12 @@ class Step
   end
 
 
-  def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
+  def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil, &block)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
     path = path.call if Proc === path
 
+    task = block if block_given?
+
     @path = path
     @task = task
     @bindings = bindings
@@ -73,7 +67,8 @@ class Step
 
     load_inputs_from_info if @inputs.nil? 
 
-    NamedArray.setup(@inputs, task.inputs) if task && task.inputs && !(NamedArray === @inputs)
+    task_inputs = task.respond_to?(:inputs) ? task.inputs : nil
+    NamedArray.setup(@inputs, task_inputs) unless NamedArray === @inputs
 
     @inputs || []
   end
@@ -133,15 +128,6 @@ class Step
     v
   end
 
-  def task_name
-    @task_name ||= begin
-                     if @task.nil?
-                        @path.split("/")[-2]
-                     else
-                       @task.name
-                     end
-                   end
-  end
 
   def path
     if Proc === @path
@@ -155,8 +141,11 @@ class Step
     attr_accessor :log_relay_step
   end
 
+  
+  # May be deprecated: This is no loger used
   def relay_log(step)
-    return self unless Task === self.task and not self.task.name.nil?
+    return self if self.task_name.nil?
+
     if not self.respond_to? :original_log
       class << self
         attr_accessor :relay_step
@@ -164,77 +153,58 @@ class Step
         def log(status, message = nil)
           self.status = status
           message Log.uncolor message
-          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
+          relay_step.log([self.task_name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
         end
       end
     end
     @relay_step = step
-    self
-  end
 
-  def result_type
-    @result_type ||= if @task.nil?
-                       info[:result_type] || :binary
-                     else
-                       @task.result_type || info[:result_type] || :string
-                     end
-  end
-
-  def result_type=(type)
-    @result_type = type
-  end
-
-  def result_description
-    @result_description ||= if @task.nil?
-                       info[:result_description]
-                     else
-                       @task.result_description
-                     end
+    self
   end
 
   def prepare_result(value, description = nil, entity_info = nil)
     res = case 
-    when IO === value
-      begin
-        res = case result_type
-              when :array
-                array = []
-                while line = value.gets
-                  array << line.chomp
-                end
-                array
-              when :tsv
-                begin
-                  TSV.open(value)
-                rescue IOError
-                  TSV.setup({})
-                end
-              else
-                value.read
-              end
-        value.join if value.respond_to? :join
-        res
-      rescue Exception
-        value.abort if value.respond_to? :abort
-        self.abort
-        raise $!
-      end
-    when (not defined? Entity or description.nil? or not Entity.formats.include? description)
-      value
-    when (Annotated === value and info.empty?)
-      value
-    when Annotated === value
-      annotations = value.annotations
-      entity_info ||= begin 
-                        entity_info = info.dup
-                        entity_info.merge! info[:inputs] if info[:inputs]
-                        entity_info
+          when IO === value
+            begin
+              res = case result_type
+                    when :array
+                      array = []
+                      while line = value.gets
+                        array << line.chomp
                       end
-      entity_info.each do |k,v|
-        value.send("#{h}=", v) if annotations.include? k
-      end
-                        
-      value
+                      array
+                    when :tsv
+                      begin
+                        TSV.open(value)
+                      rescue IOError
+                        TSV.setup({})
+                      end
+                    else
+                      value.read
+                    end
+              value.join if value.respond_to? :join
+              res
+            rescue Exception
+              value.abort if value.respond_to? :abort
+              self.abort
+              raise $!
+            end
+          when (not defined? Entity or description.nil? or not Entity.formats.include? description)
+            value
+          when (Annotated === value and info.empty?)
+            value
+          when Annotated === value
+            annotations = value.annotations
+            entity_info ||= begin 
+                              entity_info = info.dup
+                              entity_info.merge! info[:inputs] if info[:inputs]
+                              entity_info
+                            end
+            entity_info.each do |k,v|
+              value.send("#{h}=", v) if annotations.include? k
+            end
+
+            value
     else
       entity_info ||= begin 
                         entity_info = info.dup
@@ -371,8 +341,9 @@ class Step
   def step(name)
     @steps ||= {}
     @steps[name] ||= begin
+                       name = name.to_sym
                        deps = rec_dependencies.select{|step| 
-                         step.task_name.to_sym == name.to_sym
+                         step.task_name && step.task_name.to_sym == name
                        }
                        raise "Dependency step not found: #{ name }" if deps.empty?
                        if (deps & self.dependencies).any?
@@ -385,3 +356,4 @@ class Step
 end
 
 require 'rbbt/workflow/step/run'
+require 'rbbt/workflow/step/accessor'

data/lib/rbbt/workflow/task.rb

@@ -88,13 +88,15 @@ module Task
         next
       end
 
+      task_name ||= task.name
+
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task.name, maps]
+      next if seen.include? [wf, task_name, maps]
 
       task.workflow = wf if wf
 
-      seen << [wf, task.name, maps]
+      seen << [wf, task_name, maps]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
       if task_inputs[task].nil?

data/lib/rbbt/workflow/util/archive.rb

@@ -160,44 +160,50 @@ puts files * "\n"
     end
   end
 
-  def self.migrate(path, search_path, options = {})
-    if Step === path
-      if options[:source]
-        path = Rbbt.identify(path.path)
-      else
-        path = path.path
+  def self.migrate(paths, search_path, options = {})
+    subpath_files = {}
+    target_paths = []
+    resource = options[:resource] || Rbbt
+
+    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
+
+    (Array === paths ? paths : [paths]).each do |path|
+      if Step === path
+        if options[:source]
+          path = Rbbt.identify(path.path)
+        else
+          path = path.path
+        end
       end
-    end
-    search_path = 'user' if search_path.nil?
+      search_path = 'user' if search_path.nil?
 
-    resource = Rbbt
 
-    path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
+      path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
 
-    subpath_files = {}
-    real_paths.sort.each do |path|
-      parts = path.split("/")
-      subpath = parts[0..-4] * "/" + "/"
+      real_paths.sort.each do |path|
+        parts = path.split("/")
+        subpath = parts[0..-4] * "/" + "/"
+
+        if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
+          subpath = subpath_files.keys.last
+        end
 
-      if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
-        subpath = subpath_files.keys.last
+        source = path.chars[subpath.length..-1] * ""
+
+        subpath_files[subpath] ||= []
+        subpath_files[subpath] << source
       end
 
-      source = path.chars[subpath.length..-1] * ""
 
-      subpath_files[subpath] ||= []
-      subpath_files[subpath] << source
+      target_paths << File.join(target, *path.split("/")[-3..-1])
     end
 
-    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
-
-    target_path = File.join(target, *path.split("/")[-3..-1])
 
     subpath_files.each do |subpath, files|
       Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
 
-    target_path
+    target_paths
   end
 
   def self.purge(path, recursive = false, skip_overriden = true)

data/lib/rbbt/workflow.rb

@@ -212,11 +212,13 @@ module Workflow
                  end
       workflow.load_documentation
 
+
       first ||= workflow
     end
-    return first
 
-    workflow
+    first.complete_name = wf_name
+
+    return first
   end
 
   attr_accessor :description
@@ -230,6 +232,12 @@ module Workflow
   attr_accessor :relay_tasks
 
   #{{{ ATTR DEFAULTS
+  #
+  attr_accessor :complete_name
+
+  def self.complete_name
+    @complete_name ||= self.to_s
+  end
   
   def self.workdir=(path)
     path = Path.setup path.dup unless Path === path

data/python/rbbt.py

@@ -1,7 +1,93 @@
 import warnings
 import sys
 import os
-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
 
 def rbbt():
     print("Rbbt")
+
+def path(subdir = None, base_dir = None):
+    from pathlib import Path
+    import os
+
+    if (base_dir == None):
+        base_dir = os.path.join(Path.home(), ".rbbt")
+    if (subdir == None):
+        return base_dir
+    else:
+        return os.path.join(base_dir, subdir)
+
+def inspect(obj):
+    print(dir(obj))
+
+def log_tsv(tsv):
+    print(tsv)
+    print(tsv.keys())
+
+def tsv_preamble(line, comment_char="#"):
+    import re
+    header = dict()
+    entries = re.sub(f"^{comment_char}:", '', line)
+    entries = re.sub(f"^{comment_char}:", '', line).split("#")
+    for entry in entries:
+        key, value = entry.split("=")
+        key = re.sub("^:","",key)
+        value = re.sub("^:","",value)
+        header[key] = value
+
+    return header
+
+
+def tsv_header(filename, sep="\t", comment_char="#"):
+    import re
+
+    f = open(filename)
+    line = f.readline().strip()
+
+    if (not line.startswith(comment_char)):
+        header = {"fields":None, "type":"list", "start": 0}
+    else:
+        header = dict()
+        start = 0
+        if (line.startswith(f"{comment_char}:")):
+            header["preamble"]=tsv_preamble(line, comment_char)
+            if ("type" in header["preamble"]):
+                header["type"] = header["preamble"]["type"]
+            line = f.readline().strip()
+            start = 1
+
+        if (line.startswith(comment_char)):
+            header["all_fields"] = re.sub(f"^{comment_char}", "", line).split(sep)
+            header["key_field"] = header["all_fields"][0]
+            header["fields"] = header["all_fields"][1:]
+
+        header["start"] = start
+
+    f.close()
+    return header
+
+
+def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
+    import pandas
+
+    if (comment_char == ""):
+        tsv = pandas.read_table(filename, sep=sep, index_col=index_col, **kwargs)
+    else:
+        header = tsv_header(filename, sep=sep, comment_char="#")
+
+        if ("type" in header and header["type"] == "flat"):
+            return None
+        else:
+            if ("sep" in header):
+                sep=header["sep"]
+
+            tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=header["start"], **kwargs)
+
+            if ("fields" in header):
+                tsv.columns = header["fields"]
+                tsv.index.name = header["key_field"]
+
+    return tsv
+
+def tsv(*args, **kwargs):
+    return tsv_pandas(*args, **kwargs)
+

data/share/rbbt_commands/tsv/write_excel

@@ -54,8 +54,9 @@ tsvfile, excelfile = ARGV
 
 excelfile, tsvfile = tsvfile, nil if excelfile.nil? and tsvfile =~ /\.(xls|xlsx)$/
 
-raise ParameterException, "No files given" if tsvfile.nil? and excelfile.nil?
-raise ParameterException, "No excel file given" if excelfile.nil?
+raise ParameterException, "No TSV file given" if tsvfile.nil?
+
+excelfile = tsvfile.sub(/\.tsv(\.gz)?$/, '.xlsx') if excelfile.nil?
 
 require 'rbbt/tsv/excel'
 TSV.open(tsvfile, options).excel(excelfile, options)

data/share/rbbt_commands/workflow/task

@@ -213,11 +213,15 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -prov--provenance Report the jobs provenance
 -W--workflows* Load a list of workflows
 -R--requires* Require a list of files
--pre--prepare* Prepare dependencies
--prec--prepare_cpus* Number of dependencies prepared in parallel
+-pro--produce* Prepare dependencies
+-proc--produce_cpus* Number of dependencies prepared in parallel
 -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
 -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
 -PERF--procpath_performance* Measure performance using procpath
+-r--relay* Relay job to SSH server
+-sr--slurm_relay* Relay job to SSH SLURM server
+-rdep--relay_dependencies* Relay dependencies instead of main job
+-pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
 EOF
 
 workflow = ARGV.shift
@@ -415,13 +419,53 @@ begin
     exit 0
   end
 
-  if tasks = options.delete(:prepare)
+  if tasks = options.delete(:produce)
     tasks = tasks.split(",")
-    prepare_cpus = (options[:prepare_cpus] || 1)
-    puts Step.prepare_dependencies(job, tasks,  prepare_cpus)
+    produce_cpus = (options[:produce_cpus] || 1)
+    puts Step.produce_dependencies(job, tasks,  prepare_cpus)
     exit 0
   end
 
+  def match_dependencies(queries, dependencies)
+    queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
+    dependencies.select do |dep|
+      queries.select do |q|
+        if Array === q
+          q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
+        else
+          q.to_s == dep.task_name.to_s
+        end
+      end.any?
+    end
+  end
+
+  if server = options.delete(:relay)
+    require 'rbbt/workflow/remote_workflow'
+    relay_dependencies = options.delete(:relay_dependencies).split(",")
+    produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
+    if relay_dependencies
+      match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
+        RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+      end
+    else
+      RemoteWorkflow::SSH.relay_job(job, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+    end
+  end
+
+  if server = options.delete(:slurm_relay)
+    require 'rbbt/workflow/remote_workflow'
+    relay_dependencies = options.delete(:relay_dependencies).split(",")
+    produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
+    if relay_dependencies
+      match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
+        RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+      end
+    else
+      RemoteWorkflow::SSH.relay_job(job, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
+    end
+  end
+
+
   if options[:procpath_performance]
     require 'rbbt/util/procpath'
     current_pid = job.info[:pid]

data/test/rbbt/test_workflow.rb

@@ -513,4 +513,92 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal "TLA", job.run
     assert job.overriden
   end
+
+  def test_anonymous_workflow
+    workflow = Module.new
+    workflow.extend Workflow
+    workflow.define_singleton_method(:to_s){"TestWorkflow"}
+
+    assert_equal "TestWorkflow", workflow.to_s
+
+    t1 = workflow.task :s1 => :string do
+      task_name.to_s
+    end
+
+    workflow.dep :s1
+    t2 = workflow.task :s2 => :string do
+      task_name.to_s
+    end
+
+    workflow.dep :s2
+    t3 = workflow.task :s3 => :string do
+      task_name.to_s
+    end
+
+    assert_equal "s3", workflow.job(:s3).run
+  end
+
+  def test_low_level_step
+    Log.with_severity 0 do
+      TmpFile.with_file do |tmpdir|
+        Path.setup tmpdir
+
+        s1 = Step.new tmpdir.s1 do "s1" end 
+        s1.task_name = "s1"
+
+        # run clean run
+        assert_equal "s1", s1.run
+        s1.clean
+        assert_equal "s1", s1.run
+        assert_equal "s1", s1.run
+
+        # is persisted because it raises RbbtException
+        s1.task = proc do raise RbbtException end
+        assert_equal "s1", s1.run
+        s1.clean
+        assert_raises RbbtException do s1.run end
+        assert RbbtException === s1.get_exception
+        assert s1.error?
+
+        s1.recursive_clean
+        s1.task = proc do "s1" end
+
+        # add dependencies
+        s2 = Step.new tmpdir.s2 do 
+          step(:s1).load + " -> s2"  
+        end 
+        s2.task_name = "s2"
+        s2.dependencies << s1
+
+        s3 = Step.new tmpdir.s3 do 
+          step(:s2).load + " -> s3"  
+        end 
+        s3.task_name = "s3"
+        s3.dependencies << s2
+
+        assert_equal "s1 -> s2", s2.run
+        assert_equal "s1 -> s2 -> s3", s3.run
+        
+        # Test recusive behaviour
+        s1.task = proc do raise RbbtException end
+        ComputeDependency.setup(s1, :produce)
+        s1.clean
+        s3.clean
+        s3.run
+        assert_equal "s1 -> s2 -> s3", s3.run
+
+        s1.task = proc do raise RbbtException end
+        ComputeDependency.setup(s1, :produce)
+        s3.recursive_clean
+        assert_raises RbbtException do s3.run end
+
+        s1.task = proc{|v| v }
+        s1.inputs = ["test"]
+        assert_equal "test", s1.run
+        s3.recursive_clean
+        assert_equal "test -> s2 -> s3", s3.run
+      end
+    end
+  end
+
 end

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -294,5 +294,6 @@ class TestWorkflowDependency < Test::Unit::TestCase
     assert job.dependencies.first.file('foo').exists?
     assert_equal 'done'.reverse, job.run
   end
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.35.4
+  version: 5.36.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-02-09 00:00:00.000000000 Z
+date: 2023-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -366,7 +366,7 @@ files:
 - lib/rbbt/workflow/step/accessor.rb
 - lib/rbbt/workflow/step/dependencies.rb
 - lib/rbbt/workflow/step/info.rb
-- lib/rbbt/workflow/step/prepare.rb
+- lib/rbbt/workflow/step/produce.rb
 - lib/rbbt/workflow/step/run.rb
 - lib/rbbt/workflow/step/save_load_inputs.rb
 - lib/rbbt/workflow/step/status.rb