rbbt-util 5.30.0 → 5.30.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0e0e6d9165c16d73d1122a8d85e8038befd335215646bc3779400f5747b4b07
-  data.tar.gz: 7c0e92d49399111157fae43149c7ec8e598aa64510591fc2f74314bbefc89b2d
+  metadata.gz: aa5b1545b1a92199d163bf398ca973c6aa3d78ab37cc724b988e007c87824b3b
+  data.tar.gz: 6cd3ce80ea589fb97a211ebe1d2973dd7c0d3f93b1ccefe10b9ecefbc77ff515
 SHA512:
-  metadata.gz: 6f1ecdb8ea4481b12138f6b63d2508f888ad60d42b56cc187ac74e3d705bb40db76d52f7e615bbd3b7fba74e49298325dcf7e5477f5527e55aa4a42e6744586d
-  data.tar.gz: 5b3761d1300940911383fc0659a4335b422db217a9118a0fd41ebd2374cec87884775564f003d5a0a14555eff68fcac7b70c68c3cea72c6d802908a48b8b514c
+  metadata.gz: d8246d2a2686cc38086800fb6c30949bba93c8e78b7feac220b710a4e80f66858f979c5ea41de0beb67aeefd29de045e2315c3b50128f448bfef521ce3d52e15
+  data.tar.gz: 26559322cae3bfb736565e8da0c7e63c3b006c793ab5cda8ec37a04fabfa460001bae6d7da8164ccd310730b81553d1d3b5a1d14cee9e6929793329ce508ec59

data/lib/rbbt/hpc/orchestrate.rb

@@ -5,7 +5,10 @@ module HPC
     def self.job_rules(rules, job)
       workflow = job.workflow.to_s
       task_name = job.task_name.to_s
+      task_name = job.overriden.to_s if Symbol === job.overriden
+
       defaults = rules["defaults"] || {}
+      defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
 
       job_rules = IndiferentHash.setup(defaults.dup)
 
@@ -83,7 +86,7 @@ module HPC
 
         deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
         if job.canfail_paths.include? dep.path
-          [deps].flatten.collect{|id| ['canfail', id] * ":"}
+          [deps].flatten.compact.collect{|id| ['canfail', id] * ":"}
         else
           deps
         end

data/lib/rbbt/hpc/slurm.rb

@@ -21,10 +21,10 @@ module HPC
       exclusive        = options.delete :exclusive
       highmem          = options.delete :highmem
 
-      queue            = options.delete(:queue) || 'bsc_ls'
+      queue            = options.delete(:queue) || Rbbt::Config.get('queue', :slurm_queue, :slurm, :SLURM, :default => 'bsc_ls')
       task_cpus        = options.delete(:task_cpus) || 1
       nodes            = options.delete(:nodes) || 1
-      time             = options.delete(:time) || "0:00:10"
+      time             = options.delete(:time) || "0:02:00"
 
       inputs_dir       = options.delete :inputs_dir
       config_keys      = options.delete :config_keys
@@ -301,11 +301,15 @@ EOF
         coda +=<<-EOF
 
 # Sync data to target location
-mkdir -p "$(dirname '#{target}')"
-rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{target}/" &>> #{fsync} 
-sync_es="$?" 
-echo $sync_es > #{fsyncexit}
-find '#{target}' -type l -ls | awk '$13 ~ /^#{target.gsub('/','\/')}/ { sub("#{source}", "#{target}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
+if [ $exit_status == '0' ]; then 
+  mkdir -p "$(dirname '#{target}')"
+  rsync -avztAXHP --copy-unsafe-links "#{source}/" "#{target}/" &>> #{fsync} 
+  sync_es="$?" 
+  echo $sync_es > #{fsyncexit}
+  find '#{target}' -type l -ls | awk '$13 ~ /^#{target.gsub('/','\/')}/ { sub("#{source}", "#{target}", $13); print $11, $13 }' | while read A B; do rm $A; ln -s $B $A; done
+else
+  sync_es="$exit_status" 
+fi
 EOF
 
         if  contain && (wipe_container == "post" || wipe_container == "both")
@@ -331,11 +335,11 @@ EOF
           else
             coda +=<<-EOF
 ##{exec_cmd} system clean
-if [ $exit_status == '0' -a $sync_es == '0' ]; then 
+#if [ $exit_status == '0' -a $sync_es == '0' ]; then 
     rm -Rfv #{contain} &>> #{fsync}
-else
-    echo "ERROR: Process failed or results could not sync correctly. Contain directory not purged" &>> #{fsync}
-fi
+#else
+#    echo "ERROR: Process failed or results could not sync correctly. Contain directory not purged" &>> #{fsync}
+#fi
 EOF
 
           end
@@ -525,7 +529,10 @@ EOF
       tail             = options.delete :tail
       dependencies     = options.delete :slurm_dependencies
       procpath         = options.delete :SLURM_procpath
+
       options[:jobname] = job.clean_name
+      log_level         = options.delete :log
+      log_level       ||= Log.severity
 
       workflow = job.workflow
 
@@ -550,16 +557,13 @@ EOF
         inputs_dir = File.join(tmp_directory, 'inputs_dir')
         saved = Step.save_job_inputs(job, inputs_dir)
 
-        if saved && saved.any?
-          options[:inputs_dir] = inputs_dir
-          cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
-        else
-          cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', (options[:log] || Log.severity).to_s]
-        end
+        cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', log_level.to_s]
+
+        cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
 
         cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
 
-        cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
+        cmd << "--load_inputs='#{inputs_dir}'" if saved && saved.any?
 
         template = self.template(cmd, options)
         jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))

data/lib/rbbt/persist/tsv/adapter.rb

@@ -133,11 +133,7 @@ module Persist
     def read_lock
       read if closed?
       if read?
-        begin
-          return yield
-        ensure
-          close
-        end
+        return yield
       end
 
       lock do

data/lib/rbbt/tsv.rb

@@ -113,11 +113,12 @@ module TSV
 
     data.entity_options = entity_options
 
-    if Path === source and data.identifiers
-      data.identifiers = Path.setup(data.identifiers, source.pkgdir, source.resource)
+    if Path === source && data.identifiers
+      Path.setup(data.identifiers, source.pkgdir, source.resource)
     end
 
     if data.respond_to? :persistence_path
+      data.read
       data
     else
       h = data.dup

data/lib/rbbt/util/open.rb

@@ -748,7 +748,9 @@ module Open
     if (dir_sub_path = find_repo_dir(path))
       writable_repo?(*dir_sub_path)
     else
-      if File.exist?(path)
+      if File.symlink?(path)
+        File.writable?(File.dirname(path))
+      elsif File.exist?(path)
         File.writable?(path)
       else
         File.writable?(File.dirname(File.expand_path(path)))

data/lib/rbbt/workflow.rb

@@ -190,7 +190,7 @@ module Workflow
       return Misc.string2const Misc.camel_case(wf_name)
     end
 
-    Log.info{"Loading workflow #{wf_name}"}
+    Log.high{"Loading workflow #{wf_name}"}
     require_local_workflow(wf_name) or 
     (Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
     begin

data/lib/rbbt/workflow/examples.rb

@@ -53,7 +53,11 @@ module Workflow
           if file =~ /\.yaml/
             inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
-            inputs[input.to_sym]  = Open.realpath(file)
+            if File.symlink?(file)
+              inputs[input.to_sym]  = File.readlink(file)
+            else
+              inputs[input.to_sym]  = Open.realpath(file)
+            end
           end
         when :text
           Log.debug "Reading #{ input } from #{file}"

data/lib/rbbt/workflow/util/archive.rb

@@ -232,7 +232,7 @@ puts resource[path].find(search_path)
           ppp Open.read(tmp_include_file)
           puts cmd 
         else
-          CMD.cmd_log(cmd)
+          CMD.cmd_log(cmd, :log => Log::INFO)
         end
       end
     end

data/lib/rbbt/workflow/util/provenance.rb

@@ -78,6 +78,7 @@ class Step
     name = info[:name] || File.basename(path)
     status = :unsync if status == :done and not Open.exist?(path)
     status = :notfound if status == :noinfo and not Open.exist?(path)
+
     str = " " * offset
     str << prov_report_msg(status, name, path, info)
     step.dependencies.reverse.each do |dep|
@@ -90,7 +91,7 @@ class Step
         if expand_repeats
           str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
         else
-          str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
+          str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(dep.status, dep.info[:name], dep.path, dep.info)))
         end
       end
     end if step.dependencies

data/share/rbbt_commands/migrate

@@ -107,6 +107,6 @@ real_paths.each do |source|
     puts cmd 
     exit 0
   else
-    CMD.cmd_log(cmd)
+    CMD.cmd_log(cmd, :log => Log::INFO)
   end
 end

data/share/rbbt_commands/slurm/list

@@ -71,7 +71,7 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
-  exe = m[1]
+  exe = m[1].sub('step_path=$(','')
  else
   exe = nil
  end
@@ -98,15 +98,24 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  if File.exists?(fstatus = File.join(dir, 'job.status'))
   nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
  elsif job_nodes[id]
-  nodes = job_nodes[id]
+  nodes = job_nodes[id].reject{|n| n.include? "("}
  else
   nodes = []
  end
 
+ if File.exists?(File.join(dir, 'exit.status'))
+   now = File.ctime(File.join(dir, 'exit.status'))
+ else
+   now = Time.now
+ end
+
  if File.exists?(File.join(dir, 'std.out'))
+  cerrt = File.ctime File.join(dir, 'std.err')
+  coutt = File.ctime File.join(dir, 'std.out')
   outt = File.mtime File.join(dir, 'std.out')
   errt = File.mtime File.join(dir, 'std.err')
-  time_diff = Time.now - [outt, errt].max
+  time_diff = now - [outt, errt].max
+  time_elapsed = now - [cerrt, coutt].min
  end
 
  fdep = File.join(dir, 'dependencies.list')
@@ -115,14 +124,19 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  fcadep = File.join(dir, 'canfail_dependencies.list')
  cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
 
- if done || error || aborted || running || queued || jobid || search
+ if done || error || aborted || running || queued || jobid 
    select = false
    select = true if done && exit_status == 0
    select = true if error && exit_status && exit_status != 0 
    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
-   select = true if queued && deps && (running_jobs & deps).any?
-   select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+   is_running =  exit_status.nil? && running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)
+   select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
+   select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
+   select = select && cmd.match(/#{search}/) if search
+   next unless select 
+ elsif search
+   select = false
    select = true if search && cmd.match(/#{search}/)
    next unless select 
  end
@@ -130,6 +144,7 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
  puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
@@ -138,7 +153,8 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
  puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
  puts Log.color(:magenta, "Nodes: ") << nodes * ", "
- puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+ puts Log.color(:magenta, "Time elapsed: ") << Misc.format_seconds(time_elapsed) if time_elapsed
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? || File.exists?(File.join(dir, 'exit.status')) ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
 
  if options[:sbatch_parameters]
    puts Log.color(:magenta, "SBATCH parameters: ")
@@ -180,8 +196,11 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
    start = rss_average.keys.sort.first
    eend = rss_average.keys.sort.last
    time_elapsed = eend - start
+   ticks = 1 if ticks == 0
+   time_elapsed = 1 if time_elapsed == 0
    puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
    puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s 
+   puts Log.color(:yellow, "Time: ") + Misc.format_seconds((eend - start))
 
  end
 

data/share/rbbt_commands/workflow/task

@@ -410,10 +410,20 @@ begin
 
   if options[:procpath_performance]
     require 'rbbt/util/procpath'
+    current_pid = job.info[:pid]
     job.fork
     job.soft_grace
-    pid = job.info[:pid]
-    ProcPath.monitor(pid, options[:procpath_performance])
+    sleep 2 if job.info[:pid] == current_pid
+    if job.info[:pid] != current_pid
+      pid = job.info[:pid]
+      begin
+        ProcPath.monitor(pid, options[:procpath_performance])
+      rescue Errno::ECHILD
+        Log.warn "Procpath didn't find process #{pid} to monitor. Maybe it finished already"
+      rescue
+        Log.warn "Procpath failed: #{$!.message}"
+      end
+    end
   end
 
   if do_fork

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.0
+  version: 5.30.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake