rbbt-util 5.28.14 → 5.29.0

data/lib/rbbt/workflow.rb

@@ -385,7 +385,7 @@ module Workflow
       next if default == v 
       next if (String === default and Symbol === v and v.to_s == default)
       next if (Symbol === default and String === v and v == default.to_s)
-      real_inputs[k] = v 
+      real_inputs[k.to_sym] = v 
     end
 
     jobname_input_value = inputs[jobname_input] || all_defaults[jobname_input]
@@ -410,6 +410,7 @@ module Workflow
     job.workflow = self
     job.clean_name = jobname
     job.overriden = overriden
+    job.real_inputs = real_inputs.keys
     job
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -302,6 +302,7 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     dep = Step === dep ? dep : Workflow.load_step(dep)
     dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
@@ -309,7 +310,7 @@ module Workflow
       Log.exception $!
     end
     dep.task_name = task_name
-    dep.overriden = true
+    dep.overriden = dep.original_task_name.to_sym
     dep
   end
 

data/lib/rbbt/workflow/examples.rb

@@ -50,8 +50,8 @@ module Workflow
         case input_types[input]
         when :file
           Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.read$/
-            inputs[input.to_sym]  = Open.read(file)
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             inputs[input.to_sym]  = Open.realpath(file)
           end

data/lib/rbbt/workflow/step.rb

@@ -12,6 +12,7 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name
 
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
@@ -145,11 +146,13 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      dep_info[:inputs].each do |k,v|
-        all_inputs[k] = v unless all_inputs.include?(k)
-      end if dep_info[:inputs]
-      deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-      deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
       seen << path
     end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -93,18 +93,18 @@ class Step
 
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
-      when Array === value
-        Open.write(path, value * "\n")
-      when IO === value
-        Open.write(path, value)
-      when type == "file"
+      when Step === value
+        Open.link(value.path, path)
+      when type.to_s == "file"
         if String === value && File.exists?(value)
           Open.link(value, path)
         else
-          Open.write(path + '.read', value.to_s)
+          Open.write(path + '.yaml', value.to_yaml)
         end
-      when Step === value
-        value = value.produce.load 
+      when Array === value
+        Open.write(path, value * "\n")
+      when IO === value
+        Open.write(path, value)
       else
         Open.write(path, value.to_s)
       end
@@ -114,7 +114,7 @@ class Step
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = job.task_name
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
     task_info = workflow.task_info(task_name)
@@ -123,9 +123,11 @@ class Step
     input_defaults = task_info[:input_defaults]
 
     inputs = {}
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
-      next if options and ! options.include?(name)
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
       next if value.nil?
       next if input_defaults[name] == value
       inputs[name] = value
@@ -222,7 +224,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -26,15 +26,25 @@ module Workflow
       workload
     end
 
+    def self.workload(jobs)
+      jobs.inject({}) do |acc,job| 
+          Orchestrator.job_workload(job).each do |j,d|
+            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
+          end
+          acc
+        end
+    end
+
     def self.job_rules(rules, job)
       workflow = job.workflow.to_s
       task_name = job.task_name.to_s
+      defaults = rules["defaults"] || {}
 
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow]
-      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
+      return IndiferentHash.setup(defaults) unless rules[workflow]
+      return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
 
       job_rules = IndiferentHash.setup(rules[workflow][task_name])
-      rules["defaults"].each{|k,v| job_rules[k] = v if job_rules[k].nil? } if rules["defaults"]
+      defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
       job_rules
     end
 
@@ -169,12 +179,7 @@ module Workflow
     def process(rules, jobs)
       begin
 
-        workload = jobs.inject({}) do |acc,job| 
-          Orchestrator.job_workload(job).each do |j,d|
-            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
-          end
-          acc
-        end
+        workload = Orchestrator.workload(jobs)
         all_jobs = workload.keys
 
         top_level_jobs = jobs.collect{|job| job.path }

data/share/rbbt_commands/slurm/list

@@ -0,0 +1,141 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+EOF
+
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+Log.severity = 4
+done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                   CMD.cmd('squeue').read.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                   Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                   $norunningjobs = true
+                   []
+               end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+    dir = File.dirname(fcmd)
+
+    if m = Open.read(fcmd).match(/#CMD: (.*)/)
+        cmd = m[1]
+    else
+        cmd = nil
+    end
+
+    if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+        exe = m[1]
+    else
+        exe = nil
+    end
+
+    if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+        container_home = m[1]
+    else
+        container_home = nil
+    end
+
+
+    if File.exists?(fid = File.join(dir, 'job.id'))
+        id = Open.read(fid).chomp
+    else
+        id = nil
+    end
+
+    if File.exists?(fstatus = File.join(dir, 'exit.status'))
+        exit_status = Open.read(fstatus).to_i
+    else
+        exit_status = nil
+    end
+    
+    if File.exists?(fstatus = File.join(dir, 'job.status'))
+        nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+    else
+        nodes = []
+    end
+
+    if File.exists?(File.join(dir, 'std.out'))
+        outt = File.mtime File.join(dir, 'std.out')
+        errt = File.mtime File.join(dir, 'std.err')
+        time_diff = Time.now - [outt, errt].max
+    end
+
+    fdep = File.join(dir, 'dependencies.list')
+    deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+    if done || error || aborted || running || queued || jobid || search
+      select = false
+      select = true if done && exit_status == 0
+      select = true if error && exit_status && exit_status != 0 
+      select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+      select = true if queued && deps && (running_jobs & deps).any?
+      select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+      select = true if jobid && jobid.split(",").include?(id)
+      select = true if search && cmd.match(/#{search}/)
+      next unless select 
+    end
+
+
+    puts Log.color :blue, dir
+    puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+    puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+    puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+    puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+    puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+    puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+    puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+    puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+    puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+    if tail && File.exists?(File.join(dir, 'std.err'))
+        if exit_status && exit_status != 0
+            puts Log.color(:magenta, "First error or exception found: ")
+            puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+        elsif exit_status
+            puts Log.color(:magenta, "Completed jobs: ")
+            puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+        else
+            puts Log.color(:magenta, "Log tail: ")
+            puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+        end
+    end
+
+    count += 1
+
+end
+
+puts 
+puts "Found #{count} jobs"
+

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-R--rules* Orchestration rules
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/{workflow/slurm → slurm/task}

@@ -9,8 +9,9 @@ require 'time'
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
---drbbt Use development version of rbbt 
+--drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
 -e--exclusive Make exclusive use of the node
@@ -21,8 +22,9 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -q--queue* Queue
 -t--task_cpus* Tasks
+-W--workflows* Additional workflows
 -tm--time* Time
--S--server* SLURM login node
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
 class Step
@@ -30,9 +32,14 @@ class Step
         if done?
             self.load
         else
-            Marenostrum::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
         end
     end
 end
 
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.14
+  version: 5.29.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-18 00:00:00.000000000 Z
+date: 2021-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -211,6 +211,8 @@ files:
 - lib/rbbt/entity/identifiers.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/hpc.rb
+- lib/rbbt/hpc/orchestrate.rb
+- lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
 - lib/rbbt/knowledge_base/enrichment.rb
 - lib/rbbt/knowledge_base/entity.rb
@@ -376,6 +378,9 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/list
+- share/rbbt_commands/slurm/orchestrate
+- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists
@@ -430,7 +435,6 @@ files:
 - share/rbbt_commands/workflow/remote/list
 - share/rbbt_commands/workflow/remote/remove
 - share/rbbt_commands/workflow/server
-- share/rbbt_commands/workflow/slurm
 - share/rbbt_commands/workflow/task
 - share/rbbt_commands/workflow/trace
 - share/unicorn.rb