rbbt-util 5.32.1 → 5.32.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4491182540571f16f4059ec742f210152862e77bda8c31355285c81daf8bfd8e
-  data.tar.gz: 313a4e9252f8b443fef52b4267e94b177f9cacd36a89d6db63010b4ae7fb275f
+  metadata.gz: b989d145b1a34baab93f351ea9944e73dde157e9973a9443717b84a43e7b41d5
+  data.tar.gz: d6e8f5a7e8cb2d2f0e9a99f36a7594e4fc3655dd1d4f9b571f73f0f947b19fd3
 SHA512:
-  metadata.gz: 81b4b06eac8a44027a0c0f97510c75a2b0f3daf6594da4cd977c5d197318ee7e22db35e0c12f68dced5ab5aa0c1f0c152be77ed7bafe8c5588f2d274292b2035
-  data.tar.gz: 5ea83c058de9bde5bbae17b1c08308a7311c5c48d1c589f236300c3f437b667bd7012ad48a2c4890ab0c924168916809569367155b3a9300723a9c6596837d52
+  metadata.gz: da60531d5e35efd54b0c4de7e9ad6b034ef05e8d58d6a6b0a1b15422b324dd6eb5a4cb01b21ba60c091616d512106c278ef794088f87f844bd96e86d8f61d7da
+  data.tar.gz: 2327575017d9dbb301b8c0ba36ee691bc8be00175712e2121da019251c3735d0348cae6d18e8112b00c952e233eb806a19f2275563310360ed745a9b868a6021

data/lib/rbbt/hpc/lsf.rb

@@ -8,7 +8,7 @@ module HPC
     def self.batch_system_variables
       <<-EOF
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
-let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $SLURM_CPUS_PER_TASK )"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $LSB_MAX_NUM_PROCESSORS )"
 [ ! -z $LSB_MAX_MEM_RUSAGE ] && let MAX_MEMORY="$LSB_MAX_MEM_RUSAGE" || MAX_MEMORY="$MAX_MEMORY_DEFAULT"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY

data/lib/rbbt/persist/tsv/adapter.rb

@@ -132,7 +132,7 @@ module Persist
 
     def read_lock
       read if closed?
-      if read?
+      if read? || write?
         return yield
       end
 

data/lib/rbbt/resource.rb

@@ -6,6 +6,7 @@ require 'set'
 
  
 module Resource
+  class ResourceNotFound < RbbtException; end
 
   class << self
     attr_accessor :lock_dir
@@ -154,16 +155,18 @@ module Resource
       rake_dir, content = rake_for(path)
       rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
     else
-      begin
-        if path !~ /\.(gz|bgz)$/
+      if path !~ /\.(gz|bgz)$/
+        begin
+          produce(path.annotate(path + '.gz'), force)
+        rescue ResourceNotFound
           begin
-            produce(path.annotate(path + '.gz'), force)
-          rescue
             produce(path.annotate(path + '.bgz'), force)
+          rescue ResourceNotFound
+            raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
           end
         end
-      rescue
-        raise "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+      else
+        raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
       end
     end
 
@@ -174,7 +177,7 @@ module Resource
     end
 
     if type and not File.exist?(final_path) or force
-      Log.medium "Producing: #{ final_path }"
+      Log.medium "Producing: (#{self.to_s}) #{ final_path }"
       lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
 
       Misc.lock lock_filename do
@@ -310,7 +313,10 @@ url='#{url}'
   def identify(path)
     path = File.expand_path(path)
     resource ||= Rbbt
-    (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
+    locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
+    locations -= [:current, "current"]
+    locations << :current
+    locations.uniq.each do |name|
       pattern = resource.search_paths[name]
       next if pattern.nil?
       pattern = pattern.sub('{PWD}', Dir.pwd)

data/lib/rbbt/tsv/parser.rb

@@ -40,7 +40,7 @@ module TSV
       # Process fields line
 
       preamble << line if line
-      while line and (TrueClass === @header_hash or (String === @header_hash and Misc.fixutf8(line) =~ /^#{@header_hash}/ ))
+      while line && (TrueClass === @header_hash || (String === @header_hash && Misc.fixutf8(line) =~ /^#{@header_hash}/ ))
         @fields = line.split(@sep, -1)
         @key_field = @fields.shift
         @key_field = @key_field[(0 + header_hash.length)..-1] if String === @header_hash
@@ -49,7 +49,7 @@ module TSV
         line = (@header_hash != "" ?  stream.gets : nil)
         line = Misc.fixutf8 line.chomp if line
         preamble << line if line
-        @header_hash = false if TrueClass === @header_hash
+        @header_hash = false if TrueClass === @header_hash || @header_hash == ""
       end
 
       @preamble = preamble[0..-3] * "\n"

data/lib/rbbt/tsv/stream.rb

@@ -56,6 +56,7 @@ module TSV
       preambles     = []
 
       streams = streams.collect do |stream|
+
         parser = TSV::Parser.new stream, options.dup
         sfields = parser.fields
 

data/lib/rbbt/util/cmd.rb

@@ -248,16 +248,26 @@ module CMD
     pid = io.pids.first
 
     line = "" if bar
+    starting = true
     while c = io.getc
-      STDERR << c if Log.severity <= level
-      line << c if bar
-      if c == "\n"
-        bar.process(line) if bar
+      if starting
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        starting = false
+      end
+      STDERR << c if Log.severity <= level
+      line << c if bar
+      if c == "\n"
+        bar.process(line) if bar
+        starting = true
+        #if pid
+        #  Log.logn "STDOUT [#{pid}]: ", level
+        #else
+        #  Log.logn "STDOUT: ", level
+        #end
         line = "" if bar
       end
     end 

data/lib/rbbt/util/migrate.rb

@@ -0,0 +1,118 @@
+module Rbbt 
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil)
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
+require 'rbbt-util'
+path = "#{path}"
+if Open.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+puts path
+puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      if File.exists?(path)
+        path = resource.identify(path)
+      else
+        path = Path.setup(path)
+      end
+
+      [path, (path.directory? ? path.glob_all : path.find_all), path]
+    end
+  end
+
+  def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
+    if target
+      Misc.ssh_run(target, <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "#{path}"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+      EOF
+    else
+      resource[path].find(search_path)
+    end
+  end
+
+  def self.migrate_files(real_paths, target, options = {})
+    excludes = %w(.save .crap .source tmp filecache open-remote)
+    excludes += (options[:exclude] || "").split(/,\s*/)
+    excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
+
+    other = options[:other] || []
+
+    test_str = options[:test] ? '-nv' : ''
+
+    real_paths.each do |source_path|
+      if File.directory?(source_path) || source_path =~ /\/$/
+        source_path += "/" unless source_path[-1] == "/"
+        target += "/" unless target[-1] == "/"
+      end
+
+      next if source_path == target
+
+      if options[:target]
+        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+      else
+        Open.mkdir File.dirname(target)
+      end
+
+      if options[:target]
+        target_path = [options[:target], "'" + target + "'"] * ":" 
+      else
+        target_path = "'" + target + "'"
+      end
+
+      TmpFile.with_file do |tmp_files|
+        if options[:files]
+          Open.write(tmp_files, options[:files] * "\n")
+          files_from_str = "--files-from='#{tmp_files}'"
+        else
+          files_from_str = ""
+        end
+
+        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+
+
+        cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
+
+        if options[:print]
+          puts cmd 
+          exit 0
+        else
+          CMD.cmd_log(cmd, :log => Log::INFO)
+
+          if options[:delete] && options[:files]
+            remove_files = options[:files].collect{|f| File.join(source_path, f) }
+            dirs = remove_files.select{|f| File.directory? f }
+            remove_files.each do |file|
+              next if dirs.include? file
+              Open.rm file
+            end
+            dirs.each do |dir|
+              FileUtils.rmdir dir if Dir.glob(dir).empty?
+            end
+          end 
+        end
+      end
+    end
+  end
+
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
+
+    target = migrate_target_path(lpath, search_path, resource, options[:target])
+
+    migrate_files(real_paths, target, options)
+  end
+end

data/lib/rbbt/util/misc/pipes.rb

@@ -43,7 +43,7 @@ module Misc
       File.mkfifo path
       yield path
     ensure
-      FileUtils.rm path if erase
+      FileUtils.rm path if erase && File.exists?(path)
     end
   end
   

data/lib/rbbt/workflow.rb

@@ -187,17 +187,21 @@ module Workflow
       clean_name = wf_name.sub(/::.*/,'')  
       Log.info{"Looking for '#{wf_name}' in '#{clean_name}'"}
       require_workflow clean_name
-      return Misc.string2const Misc.camel_case(wf_name)
+      workflow = Misc.string2const Misc.camel_case(wf_name)
+      workflow.load_documentation
+      return workflow
     end
 
     Log.high{"Loading workflow #{wf_name}"}
     require_local_workflow(wf_name) or 
     (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
-    begin
-      Misc.string2const Misc.camel_case(wf_name)
-    rescue
-      Workflow.workflows.last || true
-    end
+    workflow = begin
+                 Misc.string2const Misc.camel_case(wf_name)
+               rescue
+                 Workflow.workflows.last || true
+               end
+    workflow.load_documentation
+    workflow
   end
 
   attr_accessor :description
@@ -596,6 +600,17 @@ module Workflow
         true
       end
     end
+
+    if ! Open.exists?(step.info_file)
+      begin
+        workflow = step.path.split("/")[-3]
+        task_name = step.path.split("/")[-2]
+        workflow = Kernel.const_get workflow
+        step.task = workflow.tasks[task_name.to_sym]
+      rescue
+        Log.exception $!
+      end
+    end
     step
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -74,6 +74,7 @@ module Workflow
   def dep_task(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension workflow.tasks[oname].extension if workflow.tasks.include?(oname) unless @extension
+    returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
       dep = dependencies.last

data/lib/rbbt/workflow/doc.rb

@@ -45,10 +45,25 @@ module Workflow
   end
 
   def load_documentation
-    @documentation = Workflow.parse_workflow_doc documentation_markdown
+    return if @documentation
+    @documentation ||= Workflow.parse_workflow_doc documentation_markdown
     @documentation[:tasks].each do |task, description|
-      raise "Documentation for #{ task }, but not a #{ self.to_s } task" unless tasks.include? task.to_sym
-      tasks[task.to_sym].description = description
+      if task.include? "#"
+        workflow, task = task.split("#")
+        workflow = begin
+                     Kernel.const_get workflow
+                   rescue
+                     next
+                   end
+      else
+        workflow = self
+      end
+
+      if workflow.tasks.include? task.to_sym
+        workflow.tasks[task.to_sym].description = description
+      else
+        Log.low "Documentation for #{ task }, but not a #{ workflow.to_s } task" 
+      end
     end
   end
 

data/lib/rbbt/workflow/usage.rb

@@ -122,7 +122,7 @@ module Workflow
       last = _prov_tasks(workflow.dep_tree(task_name))
 
       if child
-        description << "->" << task_name.to_s
+        description << "-> " << task_name.to_s
       elsif first
         description << "" << task_name.to_s
       else
@@ -198,7 +198,7 @@ module Workflow
         puts Misc.format_definition_list_item(name.to_s, description, Log.terminal_width, 20, :yellow)
 
         prov_string = prov_string(dep_tree(name))
-        puts Log.color :blue, " ->" + prov_string if prov_string && ! prov_string.empty?
+        puts Misc.format_paragraph Log.color(:blue, "-> " + prov_string) if prov_string && ! prov_string.empty?
       end
 
     else

data/lib/rbbt/workflow/util/archive.rb

@@ -1,3 +1,4 @@
+require 'rbbt/util/migrate'
 class Step
 
   MAIN_RSYNC_ARGS="-avztAXHP --copy-links"
@@ -126,63 +127,48 @@ class Step
     end
   end
 
-  def self.migrate(path, search_path, options = {})
-    resource=Rbbt
-
-    orig_path = path
-    other_rsync_args = options[:rsync]
-
-    recursive = options[:recursive]
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil, recursive = true)
     recursive = false if recursive.nil?
-
-    paths = if options[:source]
-              Misc.ssh_run(options[:source], <<-EOF).split("\n")
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
 require 'rbbt-util'
 require 'rbbt/workflow'
 
-path = "#{path}"
 recursive = #{ recursive.to_s }
+path = "#{path}"
 
-if File.exists?(path)
+if Open.exists?(path)
   path = #{resource.to_s}.identify(path)
 else
   path = Path.setup(path)
 end
 
-files = path.glob_all
-
+files = path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p }
 files = Step.job_files_for_archive(files, recursive)
 
+puts path
 puts files * "\n"
-              EOF
-
-            else
-              if File.exists?(path)
-                path = resource.identify(path)
-                raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
-              else
-                path = Path.setup(path)
-              end
-              files = path.glob_all
-              files = Step.job_files_for_archive(files, recursive)
-              files
-            end
-
-
-    target = if options[:target] 
-               target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
-require 'rbbt-util'
-path = "var/jobs"
-resource = #{resource.to_s}
-search_path = "#{search_path}"
-puts resource[path].find(search_path)
-               EOF
-             else
-               resource['var/jobs'].find(search_path)
-             end
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      path = Path.setup(path.dup)
+      files = path.glob_all
+      files = Step.job_files_for_archive(files, recursive)
+
+      [path, files, path]
+    end
+  end
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
 
     subpath_files = {}
-    paths.sort.each do |path|
+    real_paths.sort.each do |path|
       parts = path.split("/")
       subpath = parts[0..-4] * "/" + "/"
 
@@ -190,73 +176,16 @@ puts resource[path].find(search_path)
         subpath = subpath_files.keys.last
       end
 
-      source = path[subpath.length..-1]
+      source = path.chars[subpath.length..-1] * ""
 
       subpath_files[subpath] ||= []
       subpath_files[subpath] << source
     end
 
-    synced_files = []
-    subpath_files.each do |subpath, files|
-      if options[:target]
-        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
-      else
-        Open.mkdir File.dirname(target)
-      end
+    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
 
-      if options[:source]
-        source = [options[:source], subpath] * ":"
-      else
-        source = subpath
-      end
-      target = [options[:target], target] * ":" if options[:target]
-
-      next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
-
-      files_and_dirs = Set.new( files )
-      files.each do |file|
-        synced_files << File.join(subpath, file)
-
-        parts = file.split("/")[0..-2].reject{|p| p.empty?}
-        while parts.any?
-          files_and_dirs <<  parts * "/"
-          parts.pop
-        end
-      end
-
-      TmpFile.with_file(files_and_dirs.sort_by{|l| l.length}.to_a * "\n") do |tmp_include_file|
-        test_str = options[:test] ? '-nv' : ''
-
-        cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
-
-        #cmd << " && rm -Rf #{source}" if options[:delete]
-        if options[:print]
-          ppp Open.read(tmp_include_file)
-          puts cmd 
-        else
-          CMD.cmd_log(cmd, :log => Log::INFO)
-        end
-      end
-    end
-
-    if options[:delete] && synced_files.any?
-      puts Log.color :magenta, "About to erase these files:"
-      synced_files.each do |p|
-        puts Log.color :red, p
-      end 
-
-      if options[:non_interactive]
-        response = 'yes'
-      else
-        puts Log.color :magenta, "Type 'yes' if you are sure:"
-        response = STDIN.gets.chomp
-      end
-
-      if response == 'yes'
-        synced_files.each do |p|
-          Open.rm p
-        end
-      end
+    subpath_files.each do |subpath, files|
+      Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
   end
 

data/lib/rbbt/workflow/util/trace.rb

@@ -10,6 +10,9 @@ module Workflow
       started = job.info[:started]
       ddone = job.info[:done]
 
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
       code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * " · "
       code = job.name + " - " + code
 
@@ -140,7 +143,13 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
       info = tasks_info[task] ||= IndiferentHash.setup({})
       dep_info = IndiferentHash.setup(dep.info)
 
-      time = dep_info[:done] - dep_info[:started]
+      ddone = dep_info[:done]
+      started = dep_info[:started]
+
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
+      time = ddone - started
       info[:time] ||= []
       info[:time] << time
 
@@ -180,17 +189,23 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
       jobs += step.rec_dependencies + [step]
       step.info[:archived_info].each do |path,ainfo|
         archived_step = Step.new path
-        class << archived_step
-          self
-        end.define_method :info do
+
+        archived_step.define_singleton_method :info do
           ainfo
         end
+
+        #class << archived_step
+        #  self
+        #end.define_method :info do
+        #  ainfo
+        #end
+
         jobs << archived_step
       end if step.info[:archived_info]
 
     end
 
-    jobs = jobs.uniq.sort_by{|job| t = job.info[:started]; t || Open.mtime(job.path) || Time.now }
+    jobs = jobs.uniq.sort_by{|job| t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
 
     data = trace_job_times(jobs, options[:fix_gap])
 

data/share/rbbt_commands/migrate

@@ -2,7 +2,7 @@
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
-require 'rbbt/workflow/remote_workflow'
+require 'rbbt/util/migrate'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -30,83 +30,10 @@ if options[:help]
   exit 0
 end
 
-#excludes = %w(.save .crap .source tmp filecache open-remote workflows apps software jobs PCAWG)
-excludes = %w(.save .crap .source tmp filecache open-remote)
-excludes += (options[:exclude] || "").split(/,\s*/)
-excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
-
-test_str = options[:test] ? '-nv' : ''
-
 path, search_path, _sep, *other = ARGV
 
 search_path = 'user' if search_path.nil?
-resource = Rbbt
-
-path, real_paths, lpath = if options[:source]
-          lpath, *paths = Misc.ssh_run(options[:source], <<-EOF).split("\n")
-require 'rbbt-util'
-path = "#{path}"
-if Open.exists?(path)
-  path = #{resource.to_s}.identify(path)
-else
-  path = Path.setup(path)
-end
-puts path
-puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
-          EOF
-          [path, paths.collect{|p| [options[:source], p] * ":"}, lpath]
-        else
-          if File.exists?(path)
-
-            path = resource.identify(path)
-          else
-            path = Path.setup(path)
-          end
-          [path, path.glob_all, path]
-        end
-
-target = if options[:target] 
-           target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
-require 'rbbt-util'
-path = "#{path}"
-resource = #{resource.to_s}
-search_path = "#{search_path}"
-puts resource[path].find(search_path)
-           EOF
-         else
-           resource[lpath].find(search_path)
-         end
-
-real_paths.each do |source|
-
-
-  if File.directory?(source) || source =~ /\/$/
-    source += "/" unless source[-1] == "/"
-    target += "/" unless target[-1] == "/"
-  end
 
-  next if source == target
+options[:other] = other
 
-  if options[:target]
-    CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
-  else
-    Open.mkdir File.dirname(target)
-  end
-
-  if options[:target]
-   target_path = [options[:target], target] * ":" 
-  else
-   target_path = target
-  end
-
-  cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{excludes_str} #{source} #{target_path} #{other * " "}"
-
-  cmd << " && rm -Rf #{source}" if options[:delete]
-
-  if options[:print]
-    puts cmd 
-    exit 0
-  else
-    CMD.cmd_log(cmd, :log => Log::INFO)
-  end
-end
+Rbbt.migrate(path, search_path, options)

data/share/rbbt_commands/workflow/forget_deps

@@ -8,9 +8,11 @@ $0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_
 
 options = SOPT.setup <<EOF
 
-Make a job forget all its dependencies and archive their meta-data
+Make a job forget its dependencies and archive their meta-data
 
-$ #{$0} [options] <job_path>
+$ #{$0} [options] <job_path> [<task_name>|<workflow>#<task_name>] ...
+
+Specific dependencies can be specified as . Otherwise, all are forgoten.
 
 -h--help Print this help
 -p--purge Purge dependencies
@@ -26,7 +28,7 @@ if options[:help]
   exit 0
 end
 
-path = ARGV[0]
+path, *remove = ARGV
 
 raise ParameterException, "No path given" if path.nil?
 step = Workflow.load_step path
@@ -34,10 +36,20 @@ step = Workflow.load_step path
 step.archive_deps
 step.copy_files_dir
 dependencies = step.dependencies
-step.set_info :dependencies, []
 
-if options[:purge]
+if remove && remove.any?
+ remove_paths = dependencies.select do |dep|
+  remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
+ end.collect{|dep| dep.path } 
+else
+ remove_paths = dependencies.collect{|dep| dep.path }
+end
+
+step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths.include? info.last}
+
+if options[:purge] || options[:recursive_purge]
   dependencies.each do |dependency|
+    next unless remove_paths.include? dependency.path
     Step.purge(dependency.path, options[:recursive_purge])
   end
 end

data/test/rbbt/persist/tsv/test_tokyocabinet.rb

@@ -51,7 +51,7 @@ class TestPersistTSVTC < Test::Unit::TestCase
     end
   end
 
-  def test_annotation_persist_with_repeitions
+  def test_annotation_persist_with_repetitions
     TmpFile.with_file do |tmp|
       entity1 = "Entity 1"
       entity2 = "Entity 2"

data/test/rbbt/util/test_migrate.rb

@@ -0,0 +1,36 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt-util'
+require 'rbbt/util/migrate'
+
+class TestMigrate < Test::Unit::TestCase
+  def _test_source_locate
+    assert_equal 'var/jobs/', Rbbt.migrate_source_paths(Rbbt.root['var/jobs'].find(:user)).last
+    assert Rbbt.migrate_source_paths(Rbbt.root['var/jobs'].find(:user))[1].include?(File.join(ENV["HOME"], '.rbbt/var/jobs/'))
+  end
+
+  def test_migrate
+    Open.rm_rf Rbbt.tmp.test.migration_test.find(:user)
+    test_file = Rbbt.tmp.test.migration_test.migration_test_file.find(:user)
+    Open.write(test_file, "TEST")
+    TmpFile.with_file do |tmpdir|
+      Misc.in_dir tmpdir do
+        Log.with_severity 0 do
+          Rbbt.migrate('tmp/test/migration_test/migration_test_file', :current)
+        end
+      end
+      assert_equal "TEST", Open.read(File.join(tmpdir, 'tmp/test/migration_test/migration_test_file'))
+    end
+  end
+
+  def __test_migrate_turbo
+    Log.with_severity 0 do
+    TmpFile.with_file do |tmpdir|
+      Misc.in_dir tmpdir do
+        Rbbt.migrate('etc/config', :current, :source => 'mn1')
+      end
+      assert_equal "TEST", Open.read(File.join(tmpdir, 'etc/config'))
+    end
+    end
+  end
+end
+

data/test/rbbt/workflow/util/test_archive.rb

@@ -0,0 +1,31 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt-util'
+require 'rbbt/workflow'
+require 'rbbt/workflow/util/archive'
+
+module ArchiveTestWF
+  extend Workflow
+  task :test_archive => :string do
+    Open.write(file(:file1), "Test file")
+    "TEST"
+  end
+end
+
+class TestClass < Test::Unit::TestCase
+  def test_migrate
+    job = ArchiveTestWF.job(:test_archive)
+    job.run
+
+    Log.with_severity 0 do
+      TmpFile.with_file do |tmpdir|
+        Misc.in_dir tmpdir do
+          Step.migrate(job.path, :current, :delete => false, :print => false)
+        end
+        assert_equal "TEST", Open.read(File.join(tmpdir, 'var/jobs/ArchiveTestWF/test_archive/Default'))
+      end
+    end
+
+
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.1
+  version: 5.32.7
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-21 00:00:00.000000000 Z
+date: 2021-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -282,6 +282,7 @@ files:
 - lib/rbbt/util/log/progress.rb
 - lib/rbbt/util/log/progress/report.rb
 - lib/rbbt/util/log/progress/util.rb
+- lib/rbbt/util/migrate.rb
 - lib/rbbt/util/misc.rb
 - lib/rbbt/util/misc/annotated_module.rb
 - lib/rbbt/util/misc/bgzf.rb
@@ -522,6 +523,7 @@ files:
 - test/rbbt/util/test_excel2tsv.rb
 - test/rbbt/util/test_filecache.rb
 - test/rbbt/util/test_log.rb
+- test/rbbt/util/test_migrate.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_open.rb
 - test/rbbt/util/test_procpath.rb
@@ -536,13 +538,14 @@ files:
 - test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/util/test_archive.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -557,101 +560,104 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
+- test/test_helper.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_association.rb
+- test/rbbt/hpc/test_slurm.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/test_packed_index.rb
 - test/rbbt/test_entity.rb
+- test/rbbt/test_fix_width_table.rb
 - test/rbbt/workflow/test_remote_workflow.rb
-- test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/test_schedule.rb
-- test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/test_schedule.rb
+- test/rbbt/workflow/util/test_archive.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/test_task.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/util/test_colorize.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/util/test_migrate.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_procpath.rb
-- test/rbbt/util/misc/test_development.rb
-- test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/misc/test_pipes.rb
-- test/rbbt/util/misc/test_format.rb
-- test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/R/test_plot.rb
-- test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/R/test_model.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_plot.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_tmpfile.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_config.rb
 - test/rbbt/util/test_log.rb
-- test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_R.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_open.rb
 - test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/test_python.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_chain_methods.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_attach.rb
-- test/rbbt/tsv/test_filter.rb
+- test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_development.rb
+- test/rbbt/util/misc/test_format.rb
+- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_R.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/resource/test_path.rb
+- test/rbbt/test_hpc.rb
+- test/rbbt/test_persist.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_csv.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_matrix.rb
+- test/rbbt/tsv/test_manipulate.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_stream.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_matrix.rb
+- test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_excel.rb
-- test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/parallel/test_traverse.rb
-- test/rbbt/tsv/test_stream.rb
-- test/rbbt/test_association.rb
-- test/rbbt/hpc/test_batch.rb
-- test/rbbt/hpc/test_slurm.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/test_fix_width_table.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_hpc.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/test_helper.rb
+- test/rbbt/tsv/parallel/test_through.rb