rbbt-util 5.30.12 → 5.31.3

data/lib/rbbt/resource/path.rb

@@ -3,7 +3,7 @@ require 'rbbt/util/misc/indiferent_hash'
 require 'yaml'
 
 module Path
-  attr_accessor :resource, :pkgdir, :original, :search_paths, :search_order, :libdir
+  attr_accessor :resource, :pkgdir, :original, :search_paths, :search_order, :libdir, :where
 
   def self.setup(string, pkgdir = nil, resource = nil, search_paths = nil, search_order = nil, libdir = nil)
     return string if string.nil?
@@ -99,6 +99,7 @@ module Path
 
       paths = paths.each do |p|
         p.original = File.join(found.original, p.sub(/^#{found}/, ''))
+        p.where = where
       end if found.original and pattern
 
       location_paths[where] = paths
@@ -256,6 +257,7 @@ module Path
                      end
 
                      res.original = self.original || self
+                     res.where = where
 
                      res
                    end

data/lib/rbbt/tsv/accessor.rb

@@ -713,8 +713,11 @@ module TSV
       break
     end
 
-    filename = Path === filename ? filename.find : (filename || "No filename")
-    filename + " [" + persistence_path + "]" if respond_to?(:persistence_path) and persistence_path
+    filename = @filename
+    filename = "No filename" if filename.nil? || filename.empty?
+    filename.find if Path === filename 
+    filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
+
     with_unnamed do
       <<-EOF
 Filename = #{filename}

data/lib/rbbt/tsv/dumper.rb

@@ -78,6 +78,7 @@ module TSV
     end
 
     def close_in
+      @in_stream.join if @in_stream.respond_to?(:join) && ! @in_stream.joined?
       @in_stream.close unless @in_stream.closed?
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -610,7 +610,7 @@ module TSV
     thread = Thread.new do 
       begin
         traverse_run(obj, threads, cpus, options, &block)
-        into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?) 
+        into.close if into.respond_to?(:close) and not (into.respond_to?(:closed?) and into.closed?) 
       rescue Exception
         abort_stream obj
         abort_stream into

data/lib/rbbt/tsv/stream.rb

@@ -294,20 +294,19 @@ module TSV
   end
 
 
-  def self.reorder_stream_tsv(stream, key_field, fields=nil, zipped = true)
+  def self.reorder_stream_tsv(stream, key_field, fields=nil, zipped = true, bar = nil)
     parser = TSV::Parser.new TSV.get_stream(stream), :key_field => key_field, :fields => fields
     dumper_options = parser.options
     dumper = TSV::Dumper.new dumper_options
     dumper.init 
     case parser.type
     when :single
-      TSV.traverse parser, :into => dumper do |keys,values|
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         key = keys.first
         [key, [values]]
       end
     when :double
-      TSV.traverse parser, :into => dumper do |keys,values|
-        raise [keys, values].inspect if keys.include? 'gain'
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         res = []
         keys.each_with_index do |key,i|
           vs = zipped ?  values.collect{|l| l.length == 1 ? l : [l[i]] } : values
@@ -317,12 +316,12 @@ module TSV
         res
       end
     when :list
-      TSV.traverse parser, :into => dumper do |keys,values|
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         key = keys === Array ? keys.first : keys
         [key, values]
       end
     when :flat
-      TSV.traverse parser, :into => dumper do |keys,values|
+      TSV.traverse parser, :into => dumper, :bar => bar do |keys,values|
         key = keys === Array ? keys.first : keys
         [key, values]
       end

data/lib/rbbt/util/log.rb

@@ -341,10 +341,31 @@ module Log
     end unless stack.nil?
   end
 
-  def self.tsv(tsv)
+  def self.tsv(tsv, example = false)
     STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
     STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true) 
     STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
+    if example && ! tsv.empty?
+      key = case example
+            when TrueClass, :first, "first"
+              tsv.keys.first
+            when :random, "random"
+              tsv.keys.shuffle.first
+            else
+              example
+            end
+
+      values = tsv[key]
+      values = [values] if tsv.type == :flat || tsv.type == :single
+      if values.nil?
+        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
+      else
+        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
+        tsv.fields.zip(values).each do |field,value|
+          STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
+        end
+      end
+    end
   end
 
   def self.stack(stack)

data/lib/rbbt/util/misc/development.rb

@@ -428,9 +428,9 @@ def self.add_libdir(dir=nil)
     end
   end
 
-
-  def self.ssh_run(server, script)
+  def self.ssh_run(server, script = nil, &block)
     Log.debug "Run ssh script in #{server}:\n#{script}"
+
     CMD.cmd("ssh '#{server}' 'shopt -s expand_aliases; bash -l -c \"ruby\"' ", :in => script, :log => true).read
   end
 

data/lib/rbbt/util/misc/inspect.rb

@@ -271,6 +271,22 @@ module Misc
   end
 
 
+  def self.step_file?(path)
+    return true if defined?(Step) && Step === path.resource
+    return false unless path.include?('.files/')
+    parts = path.split("/")
+    job = parts.select{|p| p =~ /\.files$/}.first
+    if job
+      i = parts.index job
+      begin
+        workflow, task = parts.values_at i - 2, i - 1
+        return Kernel.const_get(workflow).tasks.include? task.to_sym
+      rescue
+      end
+    end
+    false
+  end
+
   def self.obj2str(obj)
     _obj = obj
     obj = Annotated.purge(obj) if Annotated === obj
@@ -289,7 +305,7 @@ module Misc
           when (defined?(Path) and Path)
             if defined?(Step) && Open.exists?(Step.info_file(obj))
               obj2str(Workflow.load_step(obj))
-            elsif defined?(Step) && Step === obj.resource
+            elsif step_file?(obj)
               "Step file: " + obj
             else
               if obj.exists?

data/lib/rbbt/util/misc/options.rb

@@ -135,9 +135,14 @@ module Misc
     end
 
     new_options
+
+    options.replace new_options
   end
 
   def self.process_options(hash, *keys)
+    defaults = keys.pop if Hash === keys.last
+    hahs = Misc.add_defaults hash, defaults if defaults
+
     if keys.length == 1
       hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s) 
     else

data/lib/rbbt/workflow/integration/ansible.rb

@@ -2,7 +2,7 @@ require_relative 'ansible/workflow'
 require 'rbbt/workflow/usage'
 
 module Ansible
-  def self.play(playbook, inventory = nil)
+  def self.play(playbook, inventory = nil, verbose = false)
     inventory = Rbbt.etc.ansible_inventory.find
     Log.with_severity 0 do
       TmpFile.with_file do |tmp|
@@ -10,7 +10,11 @@ module Ansible
           Open.write(tmp, [playbook].to_yaml)
           playbook = tmp
         end
-        CMD.cmd_log("ansible-playbook -i #{inventory} #{playbook}")
+        if verbose
+          CMD.cmd_log("ansible-playbook -i #{inventory} -v #{playbook}")
+        else
+          CMD.cmd_log("ansible-playbook -i #{inventory} #{playbook}")
+        end
       end
     end
   end
@@ -46,7 +50,7 @@ module Ansible
   def self.playbook(file, task = nil, options = {})
     task = 'default' if task.nil?
 
-    workflow = Workflow.require_workflow file
+    workflow = Workflow === file ? file : Workflow.require_workflow(file)
     task = workflow.tasks.keys.last if workflow.tasks[task].nil?
     workflow2playbook workflow, task, options
   end

data/lib/rbbt/workflow/step/accessor.rb

@@ -99,6 +99,7 @@ class Step
         if String === value && File.exists?(value)
           Open.ln_s(value, path)
         else
+          value = "#{value}" if Path === value
           Open.write(path + '.yaml', value.to_yaml)
         end
       when Array === value
@@ -122,15 +123,18 @@ class Step
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
     if workflow
-      task_info = workflow.task_info(task_name)
-      input_types = task_info[:input_types]
-      task_inputs = task_info[:inputs]
-      input_defaults = task_info[:input_defaults]
+      task_info = IndiferentHash.setup(workflow.task_info(task_name))
+      input_types = IndiferentHash.setup(task_info[:input_types])
+      task_inputs = IndiferentHash.setup(task_info[:inputs])
+      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
     else
-      task_info = input_types = task_inputs = input_defaults = {}
+      task_info = IndiferentHash.setup({})
+      input_types = IndiferentHash.setup({})
+      task_inputs = IndiferentHash.setup({})
+      input_defaults = IndiferentHash.setup({})
     end
 
-    inputs = {}
+    inputs = IndiferentHash.setup({})
     real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
@@ -545,7 +549,7 @@ class Step
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/usage.rb

@@ -6,19 +6,6 @@ module Task
     puts "\n" << Misc.format_paragraph(description.strip)  << "\n" if description and not description.empty?
     puts
 
-    case
-    when (input_types.values & [:array]).any?
-      puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
-      also works in addition to the others. You may use the '--array_separator' option
-      the change this default. Whenever a file is specified it may also accept STDIN using
-      the '-' character."))
-      puts
-
-    when (input_types.values & [:text, :tsv]).any?
-      puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
-      puts
-    end
-
     selects = []
     if inputs.any?
       inputs.zip(input_types.values_at(*inputs)).select{|i,t| t.to_sym == :select && input_options[i] && input_options[i][:select_options] }.each{|i,t| selects << [i, input_options[i][:select_options]]  }
@@ -50,6 +37,19 @@ module Task
       puts
     end
 
+    case
+    when (input_types.values & [:array]).any?
+      puts Log.color(:green, Misc.format_paragraph("Lists are specified as arguments using ',' or '|'. When specified as files the '\\n'
+      also works in addition to the others. You may use the '--array_separator' option
+      the change this default. Whenever a file is specified it may also accept STDIN using
+      the '-' character."))
+      puts
+
+    when (input_types.values & [:text, :tsv]).any?
+      puts Log.color(:green, Misc.format_paragraph("Whenever a file is specified it may also accept STDIN using the '-' character."))
+      puts
+    end
+
     puts Log.color(:magenta, "Returns: ") << Log.color(:blue, result_type.to_s) << "\n"
     puts
 

data/share/config.ru

@@ -43,6 +43,9 @@ app_eval app, Rbbt.etc['app.d/remote_workflow_tasks.rb'].find_all
 #{{{ BASE
 app_eval app, Rbbt.etc['app.d/base.rb'].find
 
+#{{{ SINATRA
+load_file Rbbt.lib['sinatra.rb'].find_all
+
 #{{{ RESOURCES
 load_file Rbbt.etc['app.d/resources.rb'].find
 
@@ -70,9 +73,6 @@ load_file Rbbt.etc['app.d/preload.rb'].find_all
 #{{{ PRELOAD
 load_file Rbbt.etc['app.d/semaphores.rb'].find_all
 
-#{{{ SINATRA
-load_file Rbbt.lib['sinatra.rb'].find_all
-
 Entity.entity_list_cache     = Rbbt.var.sinatra.app[app_name].find.entity_lists
 Entity.entity_map_cache      = Rbbt.var.sinatra.app[app_name].find.entity_maps
 Entity.entity_property_cache = Rbbt.var.sinatra.app[app_name].find.entity_properties

data/share/rbbt_commands/{slurm → hpc}/clean

@@ -2,6 +2,7 @@
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
 
 #$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -9,16 +10,17 @@ options = SOPT.setup <<EOF
 
 Clean error or aborted jobs
 
-$ rbbt mnl [options] 
+$ rbbt slurm clean [options] 
 
 -h--help Print this help
 -d--done Done jobs only
 -e--error Error jobs only
 -a--aborted SLURM aboted jobs
+-q--queued Queued jobs only
 -j--job* Job ids
 -s--search* Regular expression
 -t--tail* Show the last lines of the STDERR
--SBP--sbatch_parameters show sbatch parameters
+-BP--batch_parameters show batch parameters
 -dr--dry_run Do not erase anything
 EOF
 
@@ -31,14 +33,47 @@ if options[:help]
  exit 0
 end
 
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
 Log.severity = 4
-done, error, aborted, jobid, search, tail, sbatch_parameters, dry_run = options.values_at :done, :error, :aborted, :job, :search, :tail, :sbatch_parameters, :dry_run
+done, error, aborted, queued, jobid, search, tail, batch_parameters, dry_run = options.values_at :done, :error, :aborted, :queued, :job, :search, :tail, :batch_parameters, :dry_run
 
-workdir = File.expand_path('~/rbbt-slurm')
+workdir = File.expand_path('~/rbbt-batch')
 Path.setup(workdir)
 
 running_jobs = begin
-                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt = HPC::BATCH_MODULE.job_status
                 squeue_txt.split("\n").collect{|l| l.to_i.to_s}
                rescue
                 Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
@@ -58,27 +93,35 @@ else
 end
 
 count = 0
-workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  dir = File.dirname(fcmd)
+ command_txt = Open.read(fcmd)
 
- if m = Open.read(fcmd).match(/#CMD: (.*)/)
+ if m = command_txt.match(/#CMD: (.*)/)
   cmd = m[1]
  else
   cmd = nil
  end
 
- if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+ if m = command_txt.match(/# Run command\n(.*?)\n/im)
   exe = m[1]
  else
   exe = nil
  end
 
- if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+ if m = command_txt.match(/^CONTAINER_DIR=(.*)/)
   container_home = m[1]
  else
   container_home = nil
  end
 
+ if m = command_txt.match(/^BATCH_SYSTEM=(.*)/)
+  job_batch_system = m[1].downcase
+ else
+  job_batch_system = nil
+ end
+
+ different_system = job_batch_system != batch_system
 
  if File.exists?(fid = File.join(dir, 'job.id'))
   id = Open.read(fid).chomp
@@ -93,7 +136,16 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  if File.exists?(fstatus = File.join(dir, 'job.status'))
-  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+   fstatus_txt = Open.read(fstatus) 
+   begin
+     if job_batch_system == "lsf"
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
+     else
+       nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+     end
+   rescue
+     nodes = []
+   end
  elsif job_nodes[id]
   nodes = job_nodes[id]
  else
@@ -113,20 +165,36 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
 
  aborted = error = true if aborted.nil? && error.nil?
- if done || error || aborted || running || queued || jobid || search
+ #if done || error || aborted || running || queued || jobid || search
+ #  select = false
+ #  select = true if done && exit_status && exit_status.to_i == 0 
+ #  select = true if error && exit_status && exit_status.to_i != 0 
+ #  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+ #  select = select && jobid.split(",").include?(id) if jobid 
+ #  select = select && cmd.match(/#{search}/) if search
+ #  next unless select 
+ #end
+
+ if done || error || aborted || queued || jobid 
    select = false
-   select = true if done && exit_status && exit_status.to_i == 0 
-   select = true if error && exit_status && exit_status.to_i != 0 
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-   select = select && jobid.split(",").include?(id) if jobid 
+   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
+   select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+   select = true if jobid && jobid.split(",").include?(id)
    select = select && cmd.match(/#{search}/) if search
    next unless select 
+ elsif search
+   select = false
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
  end
 
 
  puts Log.color(:yellow, "**ERASING**")
  puts Log.color :blue, dir
- puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
  puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
@@ -137,9 +205,14 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
  puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
- if options[:sbatch_parameters]
-   puts Log.color(:magenta, "SBATCH parameters: ")
-   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ if options[:batch_parameters]
+   puts Log.color(:magenta, "BATCH parameters: ")
+   case job_batch_system 
+   when 'slurm'
+     puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   when 'lsf'
+     puts Log.color :blue, CMD.cmd('grep "^#BSUB" |tail -n +6', :in => Open.read(fcmd)).read.strip
+   end
  end
 
  if tail && File.exists?(File.join(dir, 'std.err'))