rbbt-util 5.26.77 → 5.26.78

data/lib/rbbt/workflow/{archive.rb → util/archive.rb}

@@ -49,32 +49,57 @@ class Step
     end
   end
 
+  def self.job_files_for_archive(files)
+    job_files = Set.new
+
+    jobs = files.collect do |file|  
+      if Step === file
+        file
+      else
+        file = file.sub(/\.info$/,'')
+        Step.new(File.expand_path(file))
+      end
+    end.uniq
+
+    jobs.each do |step|
+      next unless File.exists?(step.path)
+      job_files << step.path
+      job_files << step.info_file if File.exists?(step.info_file)
+      job_files << step.files_dir if Dir.glob(step.files_dir + '/*').any?
+      rec_dependencies = Set.new
+      deps = [step.path]
+      seen = Set.new
+      while deps.any?
+        path = deps.shift
+        dep = Step.new path
+        seen << dep.path
+        dep.info[:dependencies].each do |task, name, path|
+          dep = Step.new path
+          next if seen.include? dep.path
+          deps << dep.path
+          rec_dependencies << dep.path
+        end if dep.info[:dependencies]
+      end
+
+      rec_dependencies.each do |path|
+        next unless File.exists?(path)
+        job_files << dep.files_dir if Dir.glob(dep.files_dir + '/*').any?
+        job_files << dep.info_file if File.exists?(dep.info_file)
+        job_files << path
+      end
+    end
+
+    job_files.to_a
+  end
+
   def self.archive(files, target = nil)
     target = self.path + '.tar.gz' if target.nil?
-    target = File.expand_path(target)
+    target = File.expand_path(target) if String === target
 
-    jobs = files.collect{|file| file = file.sub(/\.info$/,''); Step.new(File.expand_path(file))}.uniq
+    job_files = job_files_for_archive files
     TmpFile.with_file do |tmpdir|
-      jobs.each do |step|
-        Step.link_job step.path, tmpdir
-        rec_dependencies = Set.new
-        deps = [step.path]
-        seen = Set.new
-        while deps.any?
-          path = deps.shift
-          dep = Step.new path
-          seen << dep.path
-          dep.info[:dependencies].each do |task, name, path|
-            dep = Step.new path
-            next if seen.include? dep.path
-            deps << dep.path
-            rec_dependencies << dep.path
-          end if dep.info[:dependencies]
-        end
-
-        rec_dependencies.each do |path|
-          Step.link_job path, tmpdir
-        end
+      job_files.each do |file|
+        Step.link_job file, tmpdir
       end
 
       Misc.in_dir(tmpdir) do

data/share/rbbt_commands/migrate

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow/remote/ssh/get'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+RSync a directory, commonly ~/.rbbt/ 
+
+$ rbbt migrate [options] <path> <search_path> 
+
+You can specify extra rsync options after --
+
+-h--help Print this help
+-t--test Do a verbose dry run
+-p--print Print command
+-d--delete Delete source file when moved
+-s--source* Source server
+-t--target* Target server
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+#excludes = %w(.save .crap .source tmp filecache open-remote workflows apps software jobs PCAWG)
+excludes = %w(.save .crap .source tmp filecache open-remote)
+excludes += (options[:exclude] || "").split(/,\s*/)
+excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
+
+test_str = options[:test] ? '-nv' : ''
+
+path, search_path, _sep, *other = ARGV
+
+resource = Rbbt
+
+path, real_paths = if options[:source]
+          paths = SSHClient.run(options[:source], <<-EOF).split("\n")
+require 'rbbt-util'
+path = "#{path}"
+if File.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+puts path.glob_all * "\n"
+          EOF
+          [path, paths.collect{|p| [options[:source], p] * ":"}]
+        else
+          if File.exists?(path)
+            path = resource.identify(path) 
+          else
+            path = Path.setup(path)
+          end
+          [path, path.glob_all]
+        end
+
+target = if options[:target] 
+          target = SSHClient.run(options[:target], <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "#{path}"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+          EOF
+         else
+           resource[path].find(search_path)
+         end
+
+real_paths.each do |source|
+
+
+  if File.directory?(source)
+    source += "/" unless source[-1] == "/"
+    target += "/" unless target[-1] == "/"
+  end
+
+  next if source == target
+
+  if options[:target]
+    CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+  else
+    Open.mkdir File.dirname(target)
+  end
+
+  target = [options[:target], target] * ":" if options[:target]
+
+  cmd = "rsync -atAX #{test_str} #{excludes_str} #{source} #{target} #{other * " "}"
+
+  cmd << " && rm -Rf #{source}" if options[:delete]
+
+  if options[:print]
+    puts cmd 
+    exit 0
+  else
+    CMD.cmd_log(cmd)
+  end
+end

data/share/rbbt_commands/migrate_job

@@ -0,0 +1,132 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/remote/ssh/get'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+RSync a directory, commonly ~/.rbbt/ 
+
+$ rbbt migrate [options] <job_path> <search_path> 
+
+You can specify extra rsync options after --
+
+-h--help Print this help
+-t--test Do a verbose dry run
+-p--print Print command
+-d--delete Delete source file when moved
+-s--source* Source server
+-t--target* Target server
+-r--recursive Recursively move all deps
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+excludes = %w(.save .crap .source tmp filecache open-remote)
+excludes += (options[:exclude] || "").split(/,\s*/)
+excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
+
+test_str = options[:test] ? '-nv' : ''
+
+path, search_path, _sep, *other = ARGV
+
+resource = Rbbt
+
+paths = if options[:source]
+          SSHClient.run(options[:source], <<-EOF).split("\n")
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+path = "#{path}"
+if File.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+files = path.glob_all
+if #{options[:recursive].to_s == 'true'}
+  files = Step.job_files_for_archive(files)
+end
+puts files * "\n"
+          EOF
+        else
+          if File.exists?(path)
+            path = resource.identify(path)
+          else
+            path = Path.setup(path)
+          end
+          files = path.glob_all
+          if options[:recursive]
+            files = Step.job_files_for_archive(files)
+          end
+          files
+        end
+
+target = if options[:target] 
+           target = SSHClient.run(options[:target], <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "var/jobs"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+           EOF
+         else
+           resource['var/jobs'].find(search_path)
+         end
+
+subpath_files = {}
+paths.each do |source|
+  parts = source.split("/")
+  subpath = parts[0..-4] * "/"
+  source = parts[-3..-1] * "/"
+  subpath_files[subpath] ||= []
+  subpath_files[subpath] << source
+end
+
+subpath_files.each do |subpath, files|
+  if options[:target]
+    CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+  else
+    Open.mkdir File.dirname(target)
+  end
+
+  if options[:source]
+    source = [options[:source], subpath] * ":"
+  else
+    source = subpath
+  end
+  target = [options[:target], target] * ":" if options[:target]
+
+  files_and_dirs = Set.new(files )
+  files.each do |file|
+    parts = file.split("/")[0..-2]
+    while parts.any?
+      files_and_dirs << parts * "/"
+      parts.pop
+    end
+  end
+
+  TmpFile.with_file(files_and_dirs.to_a * "\n") do |tmp_include_file|
+    includes_str = "--include-from='#{tmp_include_file}'"
+    cmd = "rsync -atAXmPL --progress #{test_str} --include-from='#{tmp_include_file}' --exclude='*' #{source}/ #{target}/ #{other * " "}"
+
+    cmd << " && rm -Rf #{source}" if options[:delete]
+
+    if options[:print]
+      ppp Open.read(tmp_include_file)
+      puts cmd 
+    else
+      CMD.cmd_log(cmd)
+    end
+  end
+end

data/share/rbbt_commands/workflow/prov

@@ -6,7 +6,7 @@ require 'rbbt-util'
 require 'fileutils'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow/step'
-require 'rbbt/workflow/provenance'
+require 'rbbt/workflow/util/provenance'
 require 'rbbt/util/misc'
 
 require 'rbbt-util'

data/share/rbbt_commands/workflow/task

@@ -230,6 +230,10 @@ Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if
 
 workflow = Workflow.require_workflow workflow
 
+if clean_task
+  ENV["RBBT_UPDATE"] = 'true'
+end
+
 if options[:update_all_jobs]
   ENV["RBBT_UPDATE_ALL_JOBS"] = 'true'
 end
@@ -306,6 +310,7 @@ end
 #- get job
 
 job = workflow.job(task.name, name, job_options)
+$job = job
 
 # clean job
 if clean 
@@ -351,7 +356,7 @@ begin
 
     result_type = job.result_type
 
-    res = JSON.parse(res.read) if (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep === job) && %w(array float integer boolean).include?(result_type.to_s)
+    res = JSON.parse(res.read) if (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep === job) && %w(array float integer boolean).include?(result_type.to_s)
 
     case
     when res.respond_to?(:gets)
@@ -377,7 +382,6 @@ begin
   end
 
   if options.delete(:provenance)
-    require 'rbbt/workflow/provenance'
     if options.delete(:printpath)
       puts job.path
     else
@@ -387,7 +391,6 @@ begin
   end
 
   if tasks = options.delete(:prepare)
-    require 'rbbt/workflow/prepare'
     tasks = tasks.split(",")
     prepare_cpus = (options[:prepare_cpus] || 1)
     puts Step.prepare_dependencies(job, tasks,  prepare_cpus)
@@ -491,7 +494,7 @@ if job_file = options.delete(:job_file)
 end
 
 case res
-when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
+when (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep)
   res = job.result
   if res.respond_to? :gets
     begin

data/test/rbbt/test_resource.rb

@@ -83,4 +83,9 @@ class TestTSV < Test::Unit::TestCase
     end
   end
 
+  def test_identify
+    assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
+    assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/share/databases/DATABASE/FILE'))
+  end
+
 end

data/test/rbbt/test_workflow.rb

@@ -159,7 +159,7 @@ for this dependency
 
 end
 
-TestWF.workdir = Rbbt.tmp.test.workflow
+TestWF.workdir = Rbbt.tmp.test.jobs.TestWF
 
 class TestWorkflow < Test::Unit::TestCase
 
@@ -381,7 +381,8 @@ class TestWorkflow < Test::Unit::TestCase
     job = TestWF.job(:t3)
     job.step(:t1).clean
     Misc.with_env "RBBT_UPDATE_ALL_JOBS", "true" do
-     assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+      job = TestWF.job(:t3)
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
     end
   end
 
@@ -415,4 +416,19 @@ class TestWorkflow < Test::Unit::TestCase
 
   end
 
+  def test_archive
+    job = TmpFile.with_file("Hi") do |file|
+      job = TestWF.job(:reverse_file, nil, :file => file)
+      job.run
+      job
+    end
+    TmpFile.with_file nil, true, :extension => 'tar.gz' do |targz|
+      Step.archive([job], targz)
+      TmpFile.with_file do |dir|
+        dir = Path.setup(dir)
+        Misc.untar targz, dir
+        assert dir.glob("**/*").collect{|f| File.basename(f)}.include? job.name
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.77
+  version: 5.26.78
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-12 00:00:00.000000000 Z
+date: 2019-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -319,19 +319,27 @@ files:
 - lib/rbbt/workflow.rb
 - lib/rbbt/workflow/accessor.rb
 - lib/rbbt/workflow/annotate.rb
-- lib/rbbt/workflow/archive.rb
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
-- lib/rbbt/workflow/prepare.rb
-- lib/rbbt/workflow/provenance.rb
+- lib/rbbt/workflow/integration/nextflow.rb
+- lib/rbbt/workflow/remote/client.rb
+- lib/rbbt/workflow/remote/remote_step.rb
+- lib/rbbt/workflow/remote/rest/adaptor.rb
+- lib/rbbt/workflow/remote/rest/get.rb
+- lib/rbbt/workflow/remote/ssh/adaptor.rb
+- lib/rbbt/workflow/remote/ssh/get.rb
 - lib/rbbt/workflow/schedule.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
+- lib/rbbt/workflow/step/accessor.rb
 - lib/rbbt/workflow/step/dependencies.rb
+- lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
+- lib/rbbt/workflow/util/archive.rb
+- lib/rbbt/workflow/util/provenance.rb
 - share/Rlib/plot.R
 - share/Rlib/svg.R
 - share/Rlib/util.R
@@ -357,6 +365,8 @@ files:
 - share/rbbt_commands/file_server/list
 - share/rbbt_commands/file_server/remove
 - share/rbbt_commands/log
+- share/rbbt_commands/migrate
+- share/rbbt_commands/migrate_job
 - share/rbbt_commands/resource/exists
 - share/rbbt_commands/resource/find
 - share/rbbt_commands/resource/get