rbbt-util 5.34.20 → 5.34.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4109d61e49c9573bc85f7c2b8358b1dbd0bdd032dd0376f01c39f3d1e15e2c46
-  data.tar.gz: 478773feffcb9b316244ac01a28ed975a08ea21093743db33f11afc0d29a85cb
+  metadata.gz: 7d6cd028bf0803b71e0abdbd53b5f1dc1f1495f3b3fce7f3d3d9550b824816dc
+  data.tar.gz: 8479aabccee8d49ba733c297e6e5ffb5751b6544eaea3f09a5cb5655090a9082
 SHA512:
-  metadata.gz: 3090d957c156550db02ba305451af8f09aae6b076038e0390003cb6eac060045a61a41f2a1d81dbcdc6278fb1fc0373a188103a0c9748f2f6974862b7361f254
-  data.tar.gz: 3d6f359bd83c0f10a7f31f6b5d20573ce4d7ed621b4742bbbd9f9ef8169e480e4a61b60d2f7cb21a2009f3d4cab79a0890331cee489363627900453b06a7ae4d
+  metadata.gz: ca686195461a9cdc3b13337aaffa8bf816ce6935692db0610e588d6ebe00f7ed2f8c48c6ee7ed1b7574242d4f4dabe789ed6ec350eb6b29114c68cc4eab51267
+  data.tar.gz: 9fa7c8a4de4b514f66c92f5dac30e8e9473452c9f4b2fd0ceeef2b022f10eb740c762b62f343ca2562374b0d13e0c29acb561fc101fcf7af1b67733d9c95f149

data/lib/rbbt/resource/path.rb

@@ -354,7 +354,7 @@ module Path
 
   def yaml
     self.open do |f|
-      YAML.unsafe_load f
+      YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(f) : YAML.load(f)
     end
   end
 

data/lib/rbbt/tsv/accessor.rb

@@ -11,7 +11,7 @@ module TSV
   attr_accessor :unnamed, :serializer_module, :entity_options, :entity_templates
 
   def info
-    {:key_field => key_field, :fields => fields, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed, :cast => cast}.delete_if{|k,v| v.nil? }
+    {:key_field => key_field, :fields => fields.dup, :namespace => namespace, :entity_options => entity_options, :type => type, :filename => filename, :identifiers => identifiers, :unnamed => unnamed, :cast => cast}.delete_if{|k,v| v.nil? }
   end
 
   def annotate(tsv)

data/lib/rbbt/tsv/parser.rb

@@ -516,7 +516,7 @@ module TSV
       data.extend TSV unless TSV === data
       data.type = @type
       data.key_field = @key_field
-      data.fields = @fields
+      data.fields = @fields.nil? ? nil : @fields.dup
       data.namespace = @namespace
       data.filename = @filename
       data.identifiers = @identifiers

data/lib/rbbt/util/migrate.rb

@@ -84,7 +84,7 @@ puts resource[path].find(search_path)
         #cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
          
         # rsync_args = "-avztAXHP --copy-unsafe-links"
-        rsync_args = "-avztAHP --copy-unsafe-links"
+        rsync_args = "-avztHP --copy-unsafe-links"
 
         rsync_args << " --link-dest '#{source_path}'" if hard_link && ! options[:source]
 
@@ -96,7 +96,7 @@ puts resource[path].find(search_path)
           puts cmd 
           exit 0
         else
-          CMD.cmd_log(cmd, :log => Log::INFO)
+          CMD.cmd_log(cmd, :log => Log::HIGH)
 
           if options[:delete] && options[:files]
             remove_files = options[:files].collect{|f| File.join(source_path, f) }

data/lib/rbbt/util/misc/development.rb

@@ -417,12 +417,42 @@ def self.add_libdir(dir=nil)
     end
   end
 
-  def self.ssh_run(server, script = nil, &block)
+  def self.ssh_run(server, script = nil)
     Log.debug "Run ssh script in #{server}:\n#{script}"
 
     CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
   end
 
+  def self.ssh_connection(server, reset = false)
+    @@ssh_connections ||= {}
+    @@ssh_connections.delete server if reset
+    @@ssh_connections[server] ||= begin
+                                    require 'pty'
+                                    master, slave = PTY.open
+                                    read, write = Misc.pipe
+                                    pid = spawn("ssh '#{server}' 'shopt -s expand_aliases; bash -l' ", :in => read, :out => slave, :err => STDERR.fileno)
+                                    read.close
+                                    slave.close
+                                    [write, master, pid]
+                                  end
+  end
+
+  def self.ssh_run(server, script = nil)
+    Log.debug "Run ssh script in #{server}:\n#{script}"
+
+    write, master, pid = ssh_connection(server)
+    write, master = ssh_connection(server, true) if PTY.check pid
+    write.puts "echo '#{script.gsub("'", '"') + "\n" + 'puts "\nCMD_OUT_END"'}' | ruby "
+    lines = []
+    while true
+      line = master.gets
+      break if line.strip == "CMD_OUT_END"
+      lines << line.strip unless line.strip.empty?
+    end
+    lines * "\n"
+
+  end
+
   def self.timeout_insist(time, msg = nil, &block)
     Misc.insist do
       begin

data/lib/rbbt/workflow/dependencies.rb

@@ -72,7 +72,12 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
 
-    dep = Step === dep ? dep.dup : Workflow.load_step(dep)
+    if Step === dep
+      dep = dep.load
+    else
+      located = Open.exists?(dep)
+      dep = Workflow.load_step(dep)
+    end
 
     dep.original_workflow ||= dep.workflow if dep.workflow
     dep.original_task_name ||= dep.task_name if dep.task_name
@@ -92,7 +97,7 @@ module Workflow
     end
 
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name && located
 
     dep.extend step_module
 

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -106,9 +106,9 @@ STDOUT.write res
       script += job_script(inputs_id, jobname)
       script +=<<-EOF
 job.init_info
-STDOUT.write job.name
+STDOUT.write job.path
       EOF
-      @name = Misc.ssh_run(server, script)
+      Misc.ssh_run(server, script)
     end
 
     def self.run_job(url, input_id, jobname = nil)
@@ -117,6 +117,7 @@ STDOUT.write job.name
       script = path_script(path)
       script += job_script(input_id, jobname)
       script +=<<-EOF
+ENV["RBBT_UPDATE"]="#{(ENV["RBBT_UPDATE"] || false).to_s}"
 job.produce
 STDOUT.write job.path
       EOF
@@ -153,31 +154,67 @@ job.clean
             new = Step.migrate(path, :user, :target => server)
             Open.write(file, new)
           end
-          CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
+          CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end
     end
 
-    def self.relay(workflow, task, jobname, inputs, server, options = {})
-      options = Misc.add_defaults options, :search_path => 'user'
-      search_path = options[:search_path]
+    #def self.relay_old(workflow, task, jobname, inputs, server, options = {})
+    #  options = Misc.add_defaults options, :search_path => 'user'
+    #  search_path = options[:search_path]
 
-      job = workflow.job(task, jobname, inputs)
+    #  job = workflow.job(task, jobname, inputs)
 
-      job.dependencies.each do |dep| 
-        dep.produce 
-      end
+    #  job.dependencies.each do |dep| 
+    #    dep.produce 
+    #  end
 
-      override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+    #  override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+
+    #  job.dependencies.each do |dep| 
+    #    Step.migrate(dep.path, search_path, :target => server)
+    #  end
+
+    #  remote = RemoteWorkflow.new("ssh://#{server}:#{workflow.to_s}", "#{workflow.to_s}")
+    #  rjob = remote.job(task, jobname, {})
+    #  rjob.override_dependencies = override_dependencies
+    #  rjob.run
+    #end
+
+    def self.relay_job(job, server, options = {})
+      migrate, produce, produce_dependencies, search_path = Misc.process_options options.dup,
+        :migrate, :produce, :produce_dependencies, :search_path
+
+      search_path ||= 'user'
+
+      produce = true if migrate
+
+      workflow_name = job.workflow.to_s
+      inputs = job.inputs.to_hash
+      job.dependencies.each do |dep|
+        dep.produce if options[:produce_dependencies]
+        next unless dep.done?
 
-      job.dependencies.each do |dep| 
         Step.migrate(dep.path, search_path, :target => server)
       end
 
-      remote = RemoteWorkflow.new("ssh://#{server}:#{workflow.to_s}", "#{workflow.to_s}")
-      rjob = remote.job(task, jobname, {})
+      remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{job.workflow.to_s}", "#{job.workflow.to_s}")
+      rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
+
+      override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
       rjob.override_dependencies = override_dependencies
-      rjob.run
+
+      if options[:migrate]
+        rjob.produce
+        Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
+      end
+
+      rjob
+    end
+
+    def self.relay(workflow, task, jobname, inputs, server, options = {})
+      job = workflow.job(task, jobname, inputs)
+      relay_job(job, server, options)
     end
 
     def workflow_description

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -22,12 +22,13 @@ class RemoteStep
 
       RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
 
-      @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
+      @remote_path ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
         Misc.insist do
           input_types = {}
           RemoteWorkflow::SSH.post_job(File.join(base_url, task.to_s), @input_id, @base_name)
         end
       end
+      @name = @remote_path.split("/").last
 
       if Open.remote?(@name)
         @url = @name
@@ -41,9 +42,10 @@ class RemoteStep
 
     def path
       @server, @server_path = RemoteWorkflow::SSH.parse_url @base_url
-      init_job unless @name
       if info[:path]
         "ssh://" + @server + ":" + info[:path]
+      elsif @remote_path
+        "ssh://" + @server + ":" + @remote_path
       else
         "ssh://" + @server + ":" + ["var/jobs", self.workflow.to_s, task_name.to_s, @name] * "/"
       end
@@ -57,6 +59,7 @@ class RemoteStep
       while ! done?
         sleep 1
       end
+      self
     end
 
     def load

data/lib/rbbt/workflow/step/run.rb

@@ -216,6 +216,12 @@ class Step
   def run(no_load = false)
     result = nil
 
+    if Workflow === workflow && workflow.relay_tasks && workflow.relay_tasks.include?(task_name)
+      server, options = workflow.relay_tasks[task_name]
+      options[:migrate] = true
+      return RemoteWorkflow::SSH.relay_job(self, server, options) 
+    end
+
     begin
       time_elapsed = total_time_elapsed = nil
       res = @mutex.synchronize do

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -111,6 +111,7 @@ module Workflow
 
       dir.glob("*#*").each do |od|
         name = File.basename(od)
+        name.sub!(/\.as_path$/,'')
         value = Open.read(od)
         Log.debug "Loading override dependency #{ name } as #{value}"
         inputs[name] = value.chomp

data/lib/rbbt/workflow/util/archive.rb

@@ -161,6 +161,13 @@ puts files * "\n"
   end
 
   def self.migrate(path, search_path, options = {})
+    if Step === path
+      if options[:source]
+        path = Rbbt.identify(path.path)
+      else
+        path = path.path
+      end
+    end
     search_path = 'user' if search_path.nil?
 
     resource = Rbbt

data/lib/rbbt/workflow.rb

@@ -45,6 +45,13 @@ module Workflow
     load_remote_tasks(Rbbt.root.etc.remote_tasks.find) if Rbbt.root.etc.remote_tasks.exists?
   end
 
+  def self.init_relay_tasks
+    return if defined? @@init_relay_tasks and @@init_relay_tasks
+    @@init_relay_tasks = true
+    load_relay_tasks(Rbbt.root.etc.relay_tasks.find) if Rbbt.root.etc.relay_tasks.exists?
+  end
+
+
 
   def self.require_remote_workflow(wf_name, url)
     require 'rbbt/workflow/remote_workflow'
@@ -142,6 +149,7 @@ module Workflow
 
   def self.require_workflow(wf_name, force_local=false)
     Workflow.init_remote_tasks
+    Workflow.init_relay_tasks
     # Already loaded
     begin
       workflow = Misc.string2const wf_name
@@ -219,6 +227,7 @@ module Workflow
   attr_accessor :step_cache
   attr_accessor :load_step_cache
   attr_accessor :remote_tasks
+  attr_accessor :relay_tasks
 
   #{{{ ATTR DEFAULTS
   
@@ -244,7 +253,7 @@ module Workflow
                        when :hash
                          clean_inputs = Annotated.purge(inputs)
                          clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (Symbol === d.overriden ? d.path : d.short_path) : d }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === d)) ? "Step: " << (Symbol === d.overriden ? d.path : d.short_path) : d }
                          key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)
@@ -543,7 +552,10 @@ module Workflow
     persist = input_values.nil? ? false : true
     persist = false
 
-    if ! (Path === step_path ? step_path.find : File.exist?(step_path)) && step_path.split("/").length == 3 && File.exist?(new_path = Rbbt.var.jobs[step_path].find)
+    if ! (Path === step_path ? step_path.find : File.exist?(step_path)) && 
+        step_path.split("/").length == 3 && 
+        File.exist?(new_path = Rbbt.var.jobs[step_path].find)
+
       step_path = new_path
     end
     
@@ -638,7 +650,7 @@ module Workflow
     end
     step = Step.new path
     relocated = false
-    step.dependencies = (step.info[:dependencies] || []).collect do |task,name,dep_path|
+    dependencies = (step.info[:dependencies] || []).collect do |task,name,dep_path|
       if Open.exists?(dep_path) || Open.exists?(dep_path + '.info') || Open.remote?(dep_path) || Open.ssh?(dep_path)
         Workflow._load_step dep_path
       else
@@ -647,6 +659,7 @@ module Workflow
         Workflow._load_step new_path
       end
     end
+    step.instance_variable_set("@dependencies", dependencies)
     step.relocated = relocated
     step.load_inputs_from_info
     step
@@ -828,4 +841,28 @@ module Workflow
     Workflow.process_remote_tasks(remote_workflow_tasks)
   end
 
+  def self.process_relay_tasks(relay_tasks)
+    require 'rbbt/workflow/remote_workflow'
+    relay_tasks.each do |workflow, servers|
+      wf = Workflow.require_workflow workflow
+      wf.relay_tasks ||= {}
+      IndiferentHash.setup wf.relay_tasks
+      servers.each do |server, tasks|
+        case tasks
+        when Array
+          tasks.each do |task|
+            wf.relay_tasks[task.to_sym] = [server, {:migrate => :true}]
+          end
+        else
+          wf.relay_tasks[tasks.to_sym] = [server, {:migrate => :true}]
+        end
+      end
+    end
+  end
+
+  def self.load_relay_tasks(filename)
+    yaml_text = Open.read(filename)
+    relay_workflow_tasks = YAML.load(yaml_text)
+    Workflow.process_relay_tasks(relay_workflow_tasks)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.20
+  version: 5.34.21
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-07 00:00:00.000000000 Z
+date: 2022-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake