rbbt-util 5.32.7 → 5.32.13

data/lib/rbbt/util/simpleDSL.rb

@@ -38,13 +38,13 @@ module SimpleDSL
     hook_method(method)
 
     # Execute
+    @config ||= {}
     if actions.is_a? Proc
       begin
-        require 'parse_tree_extensions'
-        require 'parse_tree'
-        require 'ruby2ruby'
-        @config[@@method_name] = actions.to_ruby.collect[1..-2].join
+        require 'method_source'
+        @config[@@method_name] = actions.source.split("\n")[1..-2] * "\n"
       rescue Exception
+        Log.exception $!
         @config[@@method_name] = NoRuby2Ruby.new "The gem ruby2ruby is not installed. It will not work on ruby 1.9."
       end
       

data/lib/rbbt/workflow.rb

@@ -347,6 +347,11 @@ module Workflow
 
     inputs = IndiferentHash.setup(inputs)
 
+    not_overriden = inputs.delete :not_overriden 
+    if not_overriden
+      inputs[:not_overriden] = :not_overriden_dep
+    end
+
     Workflow.resolve_locals(inputs)
 
     task_info = task_info(taskname)
@@ -400,7 +405,11 @@ module Workflow
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
 
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
-    overriden = has_overriden_inputs || dependencies.select{|dep| dep.overriden }.any?
+
+    overriden_deps = dependencies.select{|d| d.overriden }
+    true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
+
+    overriden = has_overriden_inputs || overriden_deps.any?
 
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], task.extension
@@ -413,7 +422,16 @@ module Workflow
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
-    job.overriden = overriden
+
+    case not_overriden 
+    when TrueClass
+      job.overriden = has_overriden_inputs || true_overriden_deps.any?
+    when :not_overriden_dep
+      job.overriden = true if has_overriden_inputs || true_overriden_deps.any?
+    else
+      job.overriden = true if has_overriden_inputs || overriden_deps.any?
+    end
+
     job.real_inputs = real_inputs.keys
     job
   end

data/lib/rbbt/workflow/step.rb

@@ -34,13 +34,40 @@ class Step
                     end
   end
 
+
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
   def overriden
-    if @overriden.nil? 
-      return [] if dependencies.nil?
-      dependencies.select{|dep| dep.overriden }.any?
-    else
-      @overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
     end
+    ord
   end
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
@@ -134,7 +161,11 @@ class Step
 
     archived_info = {}
     dependencies.each do |dep|
-      archived_info[dep.path] = dep.info
+      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
       archived_info.merge!(dep.archived_info)
     end if dependencies
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -248,7 +248,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -551,7 +551,7 @@ class Step
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/util/data.rb

@@ -0,0 +1,31 @@
+require 'rbbt/workflow'
+require 'rbbt/workflow/examples'
+
+module Workflow
+  module Data
+    def data(directory)
+      @@data_directory = directory
+    end
+
+    def get_datadir(clean_name)
+      data_dir = File.join(@@data_directory, clean_name)
+      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      if Path === @@data_directory
+        @@data_directory.annotate data_dir
+      else
+        Path.setup(data_dir)
+      end
+    end
+
+    def data_task(name, workflow, oname, *rest, &block)
+      dep_task(name, workflow, oname, *rest) do |jobname, options|
+        data_dir = self.get_datadir(jobname)
+        task_info = workflow.task_info(oname)
+        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+        data_options = block.call data_dir, dir_options, task_info
+        {:inputs => data_options.merge(options)}
+      end
+    end
+  end
+
+end

data/lib/rbbt/workflow/util/trace.rb

@@ -188,6 +188,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
     seed_jobs.each do |step|
       jobs += step.rec_dependencies + [step]
       step.info[:archived_info].each do |path,ainfo|
+        next unless Hash === ainfo
         archived_step = Step.new path
 
         archived_step.define_singleton_method :info do
@@ -205,7 +206,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
 
     end
 
-    jobs = jobs.uniq.sort_by{|job| t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
+    jobs = jobs.uniq.sort_by{|job| [job, job.info]; t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
 
     data = trace_job_times(jobs, options[:fix_gap])
 

data/python/rbbt.py

@@ -0,0 +1,3 @@
+def rbbt():
+    print("Rbbt")
+

data/share/install/software/lib/install_helpers

@@ -499,7 +499,7 @@ install_jar(){
     local url="$2"
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
-    wget "$url" -O "$OPT_DIR/jars/$name.jar" || wget "$url" -O "$OPT_DIR/jars/$name.jar" --no-check-certificate || (rm "$OPT_DIR/jars/$name.jar"; exit -1)
+    wget "$url" -O "$OPT_DIR/$name/$name.jar" || wget "$url" -O "$OPT_DIR/$name/$name.jar" --no-check-certificate || (rm "$OPT_DIR/$name/$name.jar"; exit -1)
     link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 

data/share/rbbt_commands/hpc/list

@@ -173,7 +173,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
-   select = select && cmd.match(/#{search}/) if search
+   select = select && step_path.match(/#{search}/) if search
    next unless select 
  elsif search
    select = false
@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end
@@ -292,13 +292,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      step_path = step_line.split(": ").last.strip
      step = Step.new step_path
      step.load_dependencies_from_info
+     has_bar = false
      (step.rec_dependencies + [step]).reverse.each do |j|
        next if j.done? 
-       next unless j.file(:progress).exists?
-       bar = Log::ProgressBar.new 
-       bar.load(j.file(:progress).yaml)
-       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+       if j.file(:progress).exists?
+         bar = Log::ProgressBar.new 
+         bar.load(j.file(:progress).yaml)
+         puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+         has_bar = true
+       end
      end
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
    end
  end
 

data/share/rbbt_commands/hpc/orchestrate

@@ -19,14 +19,19 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -OR--orchestration_rules* Orchestration rules
 -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF

data/share/rbbt_commands/hpc/task

@@ -18,14 +18,19 @@ $slurm_options = SOPT.get <<EOF
 -e--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
 -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
--W--workflows* Additional workflows
 -tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF

data/share/rbbt_commands/lsf/clean

@@ -0,0 +1,212 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean error or aborted jobs
+
+$ rbbt slurm clean [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-BP--batch_parameters show batch parameters
+-dr--dry_run Do not erase anything
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+Log.severity = 4
+done, error, aborted, queued, jobid, search, tail, batch_parameters, dry_run = options.values_at :done, :error, :aborted, :queued, :job, :search, :tail, :batch_parameters, :dry_run
+
+workdir = File.expand_path('~/rbbt-batch')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = HPC::BATCH_MODULE.job_status
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
+
+ if m = command_txt.match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = command_txt.match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ aborted = error = true if aborted.nil? && error.nil?
+ #if done || error || aborted || running || queued || jobid || search
+ #  select = false
+ #  select = true if done && exit_status && exit_status.to_i == 0 
+ #  select = true if error && exit_status && exit_status.to_i != 0 
+ #  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+ #  select = select && jobid.split(",").include?(id) if jobid 
+ #  select = select && cmd.match(/#{search}/) if search
+ #  next unless select 
+ #end
+
+ if done || error || aborted || queued || jobid 
+   select = false
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
+   select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+   select = true if jobid && jobid.split(",").include?(id)
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ elsif search
+   select = false
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
+ end
+
+
+ puts Log.color(:yellow, "**ERASING**")
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     puts Log.color :blue, CMD.cmd('grep "^#BSUB" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   end
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+ Open.rm_rf dir unless dry_run
+end
+
+puts 
+puts "Found #{count} jobs"
+