rbbt-util 5.32.2 → 5.32.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 612e1472bae3839b68f143d639025a9bb19294525ecc0e95df99702883af31d3
-  data.tar.gz: bea8c5ec09525363447c5bdc140f899f8a22ccb311d225da061b01d6f68bcf10
+  metadata.gz: 3b9d236294c4bdcc32e517eadcffa4d103e5a99a220096497a50f8baa26f746d
+  data.tar.gz: 6370930ac76bd8b86666fa59556d2d8648789c3e3dda56bb71c71c2ff65b88d3
 SHA512:
-  metadata.gz: 0aef55d55b97f40011aeaa3f09450dd623ad6d682b4b3ef5dc3f2df603efeeede2e50b3be2f4f586e502adb6a4a0fd898113d3c5c892212170edb4cde7b7f6a7
-  data.tar.gz: 81261ef8e8da6ada5c1d8e672fdbd31ff6dc67a7a08edb4b6a3dcee10cfc6ed3f7abacf0c3628d604839e2743de043793c5169a147bd9b528c2afd614dca62b9
+  metadata.gz: 7dcd6fadf6424add27b6773d16ef0b794183c4b2f52616498d8c2ec21a77c5b5ad8f3fed65ca69ce5111c71e0942f84be79c107b40156c331919533f3fc5bea7
+  data.tar.gz: ea294d8aa5f8ff04a9e902fdd2d7d2287a34934f7288a11234996d06fb72dba4f333e655d591d9b78c5bc00104036356625088f9aa4c7e808ae2d42d0b253781

data/lib/rbbt/resource.rb

@@ -6,6 +6,7 @@ require 'set'
 
  
 module Resource
+  class ResourceNotFound < RbbtException; end
 
   class << self
     attr_accessor :lock_dir
@@ -154,16 +155,18 @@ module Resource
       rake_dir, content = rake_for(path)
       rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
     else
-      begin
-        if path !~ /\.(gz|bgz)$/
+      if path !~ /\.(gz|bgz)$/
+        begin
+          produce(path.annotate(path + '.gz'), force)
+        rescue ResourceNotFound
           begin
-            produce(path.annotate(path + '.gz'), force)
-          rescue
             produce(path.annotate(path + '.bgz'), force)
+          rescue ResourceNotFound
+            raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
           end
         end
-      rescue
-        raise "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
+      else
+        raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
       end
     end
 
@@ -174,7 +177,7 @@ module Resource
     end
 
     if type and not File.exist?(final_path) or force
-      Log.medium "Producing: #{ final_path }"
+      Log.medium "Producing: (#{self.to_s}) #{ final_path }"
       lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
 
       Misc.lock lock_filename do
@@ -310,7 +313,10 @@ url='#{url}'
   def identify(path)
     path = File.expand_path(path)
     resource ||= Rbbt
-    (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
+    locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
+    locations -= [:current, "current"]
+    locations << :current
+    locations.uniq.each do |name|
       pattern = resource.search_paths[name]
       next if pattern.nil?
       pattern = pattern.sub('{PWD}', Dir.pwd)

data/lib/rbbt/util/cmd.rb

@@ -248,16 +248,26 @@ module CMD
     pid = io.pids.first
 
     line = "" if bar
+    starting = true
     while c = io.getc
-      STDERR << c if Log.severity <= level
-      line << c if bar
-      if c == "\n"
-        bar.process(line) if bar
+      if starting
         if pid
           Log.logn "STDOUT [#{pid}]: ", level
         else
           Log.logn "STDOUT: ", level
         end
+        starting = false
+      end
+      STDERR << c if Log.severity <= level
+      line << c if bar
+      if c == "\n"
+        bar.process(line) if bar
+        starting = true
+        #if pid
+        #  Log.logn "STDOUT [#{pid}]: ", level
+        #else
+        #  Log.logn "STDOUT: ", level
+        #end
         line = "" if bar
       end
     end 

data/lib/rbbt/util/migrate.rb

@@ -0,0 +1,118 @@
+module Rbbt 
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil)
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
+require 'rbbt-util'
+path = "#{path}"
+if Open.exists?(path)
+  path = #{resource.to_s}.identify(path)
+else
+  path = Path.setup(path)
+end
+puts path
+puts path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p } * "\n"
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      if File.exists?(path)
+        path = resource.identify(path)
+      else
+        path = Path.setup(path)
+      end
+
+      [path, (path.directory? ? path.glob_all : path.find_all), path]
+    end
+  end
+
+  def self.migrate_target_path(path, search_path = 'user', resource = Rbbt, target = nil)
+    if target
+      Misc.ssh_run(target, <<-EOF).split("\n").first
+require 'rbbt-util'
+path = "#{path}"
+resource = #{resource.to_s}
+search_path = "#{search_path}"
+puts resource[path].find(search_path)
+      EOF
+    else
+      resource[path].find(search_path)
+    end
+  end
+
+  def self.migrate_files(real_paths, target, options = {})
+    excludes = %w(.save .crap .source tmp filecache open-remote)
+    excludes += (options[:exclude] || "").split(/,\s*/)
+    excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
+
+    other = options[:other] || []
+
+    test_str = options[:test] ? '-nv' : ''
+
+    real_paths.each do |source_path|
+      if File.directory?(source_path) || source_path =~ /\/$/
+        source_path += "/" unless source_path[-1] == "/"
+        target += "/" unless target[-1] == "/"
+      end
+
+      next if source_path == target
+
+      if options[:target]
+        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
+      else
+        Open.mkdir File.dirname(target)
+      end
+
+      if options[:target]
+        target_path = [options[:target], "'" + target + "'"] * ":" 
+      else
+        target_path = "'" + target + "'"
+      end
+
+      TmpFile.with_file do |tmp_files|
+        if options[:files]
+          Open.write(tmp_files, options[:files] * "\n")
+          files_from_str = "--files-from='#{tmp_files}'"
+        else
+          files_from_str = ""
+        end
+
+        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+
+
+        cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
+
+        if options[:print]
+          puts cmd 
+          exit 0
+        else
+          CMD.cmd_log(cmd, :log => Log::INFO)
+
+          if options[:delete] && options[:files]
+            remove_files = options[:files].collect{|f| File.join(source_path, f) }
+            dirs = remove_files.select{|f| File.directory? f }
+            remove_files.each do |file|
+              next if dirs.include? file
+              Open.rm file
+            end
+            dirs.each do |dir|
+              FileUtils.rmdir dir if Dir.glob(dir).empty?
+            end
+          end 
+        end
+      end
+    end
+  end
+
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = migrate_source_paths(path, resource, options[:source])
+
+    target = migrate_target_path(lpath, search_path, resource, options[:target])
+
+    migrate_files(real_paths, target, options)
+  end
+end

data/lib/rbbt/workflow.rb

@@ -600,6 +600,17 @@ module Workflow
         true
       end
     end
+
+    if ! Open.exists?(step.info_file)
+      begin
+        workflow = step.path.split("/")[-3]
+        task_name = step.path.split("/")[-2]
+        workflow = Kernel.const_get workflow
+        step.task = workflow.tasks[task_name.to_sym]
+      rescue
+        Log.exception $!
+      end
+    end
     step
   end
 

data/lib/rbbt/workflow/usage.rb

@@ -122,7 +122,7 @@ module Workflow
       last = _prov_tasks(workflow.dep_tree(task_name))
 
       if child
-        description << "->" << task_name.to_s
+        description << "-> " << task_name.to_s
       elsif first
         description << "" << task_name.to_s
       else
@@ -198,7 +198,7 @@ module Workflow
         puts Misc.format_definition_list_item(name.to_s, description, Log.terminal_width, 20, :yellow)
 
         prov_string = prov_string(dep_tree(name))
-        puts Log.color :blue, " ->" + prov_string if prov_string && ! prov_string.empty?
+        puts Misc.format_paragraph Log.color(:blue, "-> " + prov_string) if prov_string && ! prov_string.empty?
       end
 
     else

data/lib/rbbt/workflow/util/archive.rb

@@ -1,3 +1,4 @@
+require 'rbbt/util/migrate'
 class Step
 
   MAIN_RSYNC_ARGS="-avztAXHP --copy-links"
@@ -126,63 +127,48 @@ class Step
     end
   end
 
-  def self.migrate(path, search_path, options = {})
-    resource=Rbbt
-
-    orig_path = path
-    other_rsync_args = options[:rsync]
-
-    recursive = options[:recursive]
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil, recursive = true)
     recursive = false if recursive.nil?
-
-    paths = if options[:source]
-              Misc.ssh_run(options[:source], <<-EOF).split("\n")
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
 require 'rbbt-util'
 require 'rbbt/workflow'
 
-path = "#{path}"
 recursive = #{ recursive.to_s }
+path = "#{path}"
 
-if File.exists?(path)
+if Open.exists?(path)
   path = #{resource.to_s}.identify(path)
 else
   path = Path.setup(path)
 end
 
-files = path.glob_all
-
+files = path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p }
 files = Step.job_files_for_archive(files, recursive)
 
+puts path
 puts files * "\n"
-              EOF
-
-            else
-              if File.exists?(path)
-                path = resource.identify(path)
-                raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
-              else
-                path = Path.setup(path)
-              end
-              files = path.glob_all
-              files = Step.job_files_for_archive(files, recursive)
-              files
-            end
-
-
-    target = if options[:target] 
-               target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
-require 'rbbt-util'
-path = "var/jobs"
-resource = #{resource.to_s}
-search_path = "#{search_path}"
-puts resource[path].find(search_path)
-               EOF
-             else
-               resource['var/jobs'].find(search_path)
-             end
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      path = Path.setup(path.dup)
+      files = path.glob_all
+      files = Step.job_files_for_archive(files, recursive)
+
+      [path, files, path]
+    end
+  end
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
 
     subpath_files = {}
-    paths.sort.each do |path|
+    real_paths.sort.each do |path|
       parts = path.split("/")
       subpath = parts[0..-4] * "/" + "/"
 
@@ -190,73 +176,16 @@ puts resource[path].find(search_path)
         subpath = subpath_files.keys.last
       end
 
-      source = path[subpath.length..-1]
+      source = path.chars[subpath.length..-1] * ""
 
       subpath_files[subpath] ||= []
       subpath_files[subpath] << source
     end
 
-    synced_files = []
-    subpath_files.each do |subpath, files|
-      if options[:target]
-        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
-      else
-        Open.mkdir File.dirname(target)
-      end
+    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
 
-      if options[:source]
-        source = [options[:source], subpath] * ":"
-      else
-        source = subpath
-      end
-      target = [options[:target], target] * ":" if options[:target]
-
-      next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
-
-      files_and_dirs = Set.new( files )
-      files.each do |file|
-        synced_files << File.join(subpath, file)
-
-        parts = file.split("/")[0..-2].reject{|p| p.empty?}
-        while parts.any?
-          files_and_dirs <<  parts * "/"
-          parts.pop
-        end
-      end
-
-      TmpFile.with_file(files_and_dirs.sort_by{|l| l.length}.to_a * "\n") do |tmp_include_file|
-        test_str = options[:test] ? '-nv' : ''
-
-        cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
-
-        #cmd << " && rm -Rf #{source}" if options[:delete]
-        if options[:print]
-          ppp Open.read(tmp_include_file)
-          puts cmd 
-        else
-          CMD.cmd_log(cmd, :log => Log::INFO)
-        end
-      end
-    end
-
-    if options[:delete] && synced_files.any?
-      puts Log.color :magenta, "About to erase these files:"
-      synced_files.each do |p|
-        puts Log.color :red, p
-      end 
-
-      if options[:non_interactive]
-        response = 'yes'
-      else
-        puts Log.color :magenta, "Type 'yes' if you are sure:"
-        response = STDIN.gets.chomp
-      end
-
-      if response == 'yes'
-        synced_files.each do |p|
-          Open.rm p
-        end
-      end
+    subpath_files.each do |subpath, files|
+      Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
   end
 

data/lib/rbbt/workflow/util/trace.rb

@@ -10,6 +10,9 @@ module Workflow
       started = job.info[:started]
       ddone = job.info[:done]
 
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
       code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * " · "
       code = job.name + " - " + code
 
@@ -140,7 +143,13 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
       info = tasks_info[task] ||= IndiferentHash.setup({})
       dep_info = IndiferentHash.setup(dep.info)
 
-      time = dep_info[:done] - dep_info[:started]
+      ddone = dep_info[:done]
+      started = dep_info[:started]
+
+      started = Time.parse started if String === started
+      ddone = Time.parse ddone if String === ddone
+
+      time = ddone - started
       info[:time] ||= []
       info[:time] << time
 
@@ -196,7 +205,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
 
     end
 
-    jobs = jobs.uniq.sort_by{|job| t = job.info[:started]; t || Open.mtime(job.path) || Time.now }
+    jobs = jobs.uniq.sort_by{|job| t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
 
     data = trace_job_times(jobs, options[:fix_gap])