rbbt-util 5.32.6 → 5.32.12

data/lib/rbbt/util/simpleDSL.rb

@@ -38,13 +38,13 @@ module SimpleDSL
     hook_method(method)
 
     # Execute
+    @config ||= {}
     if actions.is_a? Proc
       begin
-        require 'parse_tree_extensions'
-        require 'parse_tree'
-        require 'ruby2ruby'
-        @config[@@method_name] = actions.to_ruby.collect[1..-2].join
+        require 'method_source'
+        @config[@@method_name] = actions.source.split("\n")[1..-2] * "\n"
       rescue Exception
+        Log.exception $!
         @config[@@method_name] = NoRuby2Ruby.new "The gem ruby2ruby is not installed. It will not work on ruby 1.9."
       end
       

data/lib/rbbt/workflow.rb

@@ -347,6 +347,11 @@ module Workflow
 
     inputs = IndiferentHash.setup(inputs)
 
+    not_overriden = inputs.delete :not_overriden 
+    if not_overriden
+      inputs[:not_overriden] = :not_overriden_dep
+    end
+
     Workflow.resolve_locals(inputs)
 
     task_info = task_info(taskname)
@@ -400,7 +405,11 @@ module Workflow
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
 
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
-    overriden = has_overriden_inputs || dependencies.select{|dep| dep.overriden }.any?
+
+    overriden_deps = dependencies.select{|d| d.overriden }
+    true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
+
+    overriden = has_overriden_inputs || overriden_deps.any?
 
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], task.extension
@@ -413,7 +422,16 @@ module Workflow
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
-    job.overriden = overriden
+
+    case not_overriden 
+    when TrueClass
+      job.overriden = has_overriden_inputs || true_overriden_deps.any?
+    when :not_overriden_dep
+      job.overriden = true if has_overriden_inputs || true_overriden_deps.any?
+    else
+      job.overriden = true if has_overriden_inputs || overriden_deps.any?
+    end
+
     job.real_inputs = real_inputs.keys
     job
   end

data/lib/rbbt/workflow/step.rb

@@ -34,13 +34,40 @@ class Step
                     end
   end
 
+
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
   def overriden
-    if @overriden.nil? 
-      return [] if dependencies.nil?
-      dependencies.select{|dep| dep.overriden }.any?
-    else
-      @overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
     end
+    ord
   end
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
@@ -134,7 +161,11 @@ class Step
 
     archived_info = {}
     dependencies.each do |dep|
-      archived_info[dep.path] = dep.info
+      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
       archived_info.merge!(dep.archived_info)
     end if dependencies
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -248,7 +248,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -551,7 +551,7 @@ class Step
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/util/archive.rb

@@ -1,3 +1,4 @@
+require 'rbbt/util/migrate'
 class Step
 
   MAIN_RSYNC_ARGS="-avztAXHP --copy-links"
@@ -126,63 +127,48 @@ class Step
     end
   end
 
-  def self.migrate(path, search_path, options = {})
-    resource=Rbbt
-
-    orig_path = path
-    other_rsync_args = options[:rsync]
-
-    recursive = options[:recursive]
+  def self.migrate_source_paths(path, resource = Rbbt, source = nil, recursive = true)
     recursive = false if recursive.nil?
-
-    paths = if options[:source]
-              Misc.ssh_run(options[:source], <<-EOF).split("\n")
+    if source
+      lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
 require 'rbbt-util'
 require 'rbbt/workflow'
 
-path = "#{path}"
 recursive = #{ recursive.to_s }
+path = "#{path}"
 
-if File.exists?(path)
+if Open.exists?(path)
   path = #{resource.to_s}.identify(path)
 else
   path = Path.setup(path)
 end
 
-files = path.glob_all
-
+files = path.glob_all.collect{|p| File.directory?(p) ? p + "/" : p }
 files = Step.job_files_for_archive(files, recursive)
 
+puts path
 puts files * "\n"
-              EOF
-
-            else
-              if File.exists?(path)
-                path = resource.identify(path)
-                raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
-              else
-                path = Path.setup(path)
-              end
-              files = path.glob_all
-              files = Step.job_files_for_archive(files, recursive)
-              files
-            end
-
-
-    target = if options[:target] 
-               target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
-require 'rbbt-util'
-path = "var/jobs"
-resource = #{resource.to_s}
-search_path = "#{search_path}"
-puts resource[path].find(search_path)
-               EOF
-             else
-               resource['var/jobs'].find(search_path)
-             end
+      EOF
+
+      [path, paths.collect{|p| [source, p] * ":"}, lpath]
+    else
+      path = Path.setup(path.dup)
+      files = path.glob_all
+      files = Step.job_files_for_archive(files, recursive)
+
+      [path, files, path]
+    end
+  end
+
+  def self.migrate(path, search_path, options = {})
+    search_path = 'user' if search_path.nil?
+
+    resource = Rbbt
+
+    path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
 
     subpath_files = {}
-    paths.sort.each do |path|
+    real_paths.sort.each do |path|
       parts = path.split("/")
       subpath = parts[0..-4] * "/" + "/"
 
@@ -190,73 +176,16 @@ puts resource[path].find(search_path)
         subpath = subpath_files.keys.last
       end
 
-      source = path[subpath.length..-1]
+      source = path.chars[subpath.length..-1] * ""
 
       subpath_files[subpath] ||= []
       subpath_files[subpath] << source
     end
 
-    synced_files = []
-    subpath_files.each do |subpath, files|
-      if options[:target]
-        CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
-      else
-        Open.mkdir File.dirname(target)
-      end
+    target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
 
-      if options[:source]
-        source = [options[:source], subpath] * ":"
-      else
-        source = subpath
-      end
-      target = [options[:target], target] * ":" if options[:target]
-
-      next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
-
-      files_and_dirs = Set.new( files )
-      files.each do |file|
-        synced_files << File.join(subpath, file)
-
-        parts = file.split("/")[0..-2].reject{|p| p.empty?}
-        while parts.any?
-          files_and_dirs <<  parts * "/"
-          parts.pop
-        end
-      end
-
-      TmpFile.with_file(files_and_dirs.sort_by{|l| l.length}.to_a * "\n") do |tmp_include_file|
-        test_str = options[:test] ? '-nv' : ''
-
-        cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
-
-        #cmd << " && rm -Rf #{source}" if options[:delete]
-        if options[:print]
-          ppp Open.read(tmp_include_file)
-          puts cmd 
-        else
-          CMD.cmd_log(cmd, :log => Log::INFO)
-        end
-      end
-    end
-
-    if options[:delete] && synced_files.any?
-      puts Log.color :magenta, "About to erase these files:"
-      synced_files.each do |p|
-        puts Log.color :red, p
-      end 
-
-      if options[:non_interactive]
-        response = 'yes'
-      else
-        puts Log.color :magenta, "Type 'yes' if you are sure:"
-        response = STDIN.gets.chomp
-      end
-
-      if response == 'yes'
-        synced_files.each do |p|
-          Open.rm p
-        end
-      end
+    subpath_files.each do |subpath, files|
+      Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
   end
 

data/lib/rbbt/workflow/util/trace.rb

@@ -188,6 +188,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
     seed_jobs.each do |step|
       jobs += step.rec_dependencies + [step]
       step.info[:archived_info].each do |path,ainfo|
+        next unless Hash === ainfo
         archived_step = Step.new path
 
         archived_step.define_singleton_method :info do
@@ -205,7 +206,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
 
     end
 
-    jobs = jobs.uniq.sort_by{|job| t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
+    jobs = jobs.uniq.sort_by{|job| [job, job.info]; t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
 
     data = trace_job_times(jobs, options[:fix_gap])
 

data/share/rbbt_commands/hpc/list

@@ -173,7 +173,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
-   select = select && cmd.match(/#{search}/) if search
+   select = select && step_path.match(/#{search}/) if search
    next unless select 
  elsif search
    select = false
@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end
@@ -292,13 +292,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      step_path = step_line.split(": ").last.strip
      step = Step.new step_path
      step.load_dependencies_from_info
+     has_bar = false
      (step.rec_dependencies + [step]).reverse.each do |j|
        next if j.done? 
-       next unless j.file(:progress).exists?
-       bar = Log::ProgressBar.new 
-       bar.load(j.file(:progress).yaml)
-       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+       if j.file(:progress).exists?
+         bar = Log::ProgressBar.new 
+         bar.load(j.file(:progress).yaml)
+         puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+         has_bar = true
+       end
      end
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/hpc/orchestrate

@@ -19,14 +19,19 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF

data/share/rbbt_commands/hpc/task

@@ -18,14 +18,19 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF

data/share/rbbt_commands/lsf/clean

@@ -0,0 +1,212 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean error or aborted jobs
+
+$ rbbt slurm clean [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-BP--batch_parameters show batch parameters
+-dr--dry_run Do not erase anything
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+Log.severity = 4
+done, error, aborted, queued, jobid, search, tail, batch_parameters, dry_run = options.values_at :done, :error, :aborted, :queued, :job, :search, :tail, :batch_parameters, :dry_run
+
+workdir = File.expand_path('~/rbbt-batch')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = HPC::BATCH_MODULE.job_status
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
+
+ if m = command_txt.match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = command_txt.match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ aborted = error = true if aborted.nil? && error.nil?
+ #if done || error || aborted || running || queued || jobid || search
+ #  select = false
+ #  select = true if done && exit_status && exit_status.to_i == 0 
+ #  select = true if error && exit_status && exit_status.to_i != 0 
+ #  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+ #  select = select && jobid.split(",").include?(id) if jobid 
+ #  select = select && cmd.match(/#{search}/) if search
+ #  next unless select 
+ #end
+
+ if done || error || aborted || queued || jobid 
+   select = false
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
+   select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+   select = true if jobid && jobid.split(",").include?(id)
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ elsif search
+   select = false
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
+ end
+
+
+ puts Log.color(:yellow, "**ERASING**")
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     puts Log.color :blue, CMD.cmd('grep "^#BSUB" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   end
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+ Open.rm_rf dir unless dry_run
+end
+
+puts 
+puts "Found #{count} jobs"
+