rbbt-util 5.28.12 → 5.29.4

data/share/rbbt_commands/slurm/list

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt mnl [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-SBP--sbatch_parameters show sbatch parameters
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+Log.severity = 4
+done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
+
+workdir = File.expand_path('~/rbbt-slurm')
+Path.setup(workdir)
+
+running_jobs = begin
+                squeue_txt = CMD.cmd('squeue').read
+                squeue_txt.split("\n").collect{|l| l.to_i.to_s}
+               rescue
+                Log.warn "Cannot determine if jobs are running, they will seem to be all alive (Job ID in green)"
+                squeue_txt = nil
+                $norunningjobs = true
+                []
+               end
+
+if squeue_txt
+ job_nodes = {}
+ squeue_txt.split("\n").each do |line|
+  parts = line.strip.split(/\s+/)
+  job_nodes[parts.first] = parts.last.split(",")
+ end
+else
+ job_nodes = nil
+end
+
+count = 0
+workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
+ dir = File.dirname(fcmd)
+
+ if m = Open.read(fcmd).match(/#CMD: (.*)/)
+  cmd = m[1]
+ else
+  cmd = nil
+ end
+
+ if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
+  exe = m[1]
+ else
+  exe = nil
+ end
+
+ if m = Open.read(fcmd).match(/^CONTAINER_DIR=(.*)/)
+  container_home = m[1]
+ else
+  container_home = nil
+ end
+
+
+ if File.exists?(fid = File.join(dir, 'job.id'))
+  id = Open.read(fid).chomp
+ else
+  id = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'exit.status'))
+  exit_status = Open.read(fstatus).to_i
+ else
+  exit_status = nil
+ end
+
+ if File.exists?(fstatus = File.join(dir, 'job.status'))
+  nodes = Open.read(fstatus).split("\n").last.split(/\s+/).last.split(",")
+ elsif job_nodes[id]
+  nodes = job_nodes[id]
+ else
+  nodes = []
+ end
+
+ if File.exists?(File.join(dir, 'std.out'))
+  outt = File.mtime File.join(dir, 'std.out')
+  errt = File.mtime File.join(dir, 'std.err')
+  time_diff = Time.now - [outt, errt].max
+ end
+
+ fdep = File.join(dir, 'dependencies.list')
+ deps = Open.read(fdep).split("\n") if File.exists?(fdep)
+
+ fcadep = File.join(dir, 'canfail_dependencies.list')
+ cadeps = Open.read(fcadep).split("\n") if File.exists?(fcadep)
+
+ if done || error || aborted || running || queued || jobid || search
+   select = false
+   select = true if done && exit_status == 0
+   select = true if error && exit_status && exit_status != 0 
+   select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
+   select = true if queued && deps && (running_jobs & deps).any?
+   select = true if running && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
+   select = true if jobid && jobid.split(",").include?(id)
+   select = true if search && cmd.match(/#{search}/)
+   next unless select 
+ end
+
+
+ puts Log.color :blue, dir
+ puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
+ puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
+ puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
+ puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")
+ puts Log.color(:magenta, "HOME: ") << Log.color(:yellow, container_home) if container_home
+ puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
+ puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
+ puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
+ puts Log.color(:magenta, "Nodes: ") << nodes * ", "
+ puts Log.color(:magenta, "Output: ") << File.exists?(File.join(dir, 'std.out')).to_s << (id.nil? ? "" : " (last update " + Misc.format_seconds(time_diff) + " ago)")
+
+ if options[:sbatch_parameters]
+   puts Log.color(:magenta, "SBATCH parameters: ")
+   puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
+ end
+
+ if tail && File.exists?(File.join(dir, 'std.err'))
+   if exit_status && exit_status != 0
+     puts Log.color(:magenta, "First error or exception found: ")
+     puts CMD.cmd("grep -i -w 'error\\|[a-z]*exception' #{File.join(dir, 'std.err')} -A #{tail.to_i} |head -n #{tail.to_i}", :no_fail => true).read
+   elsif exit_status
+     puts Log.color(:magenta, "Completed jobs: ")
+     puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
+   else
+     puts Log.color(:magenta, "Log tail: ")
+     puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
+   end
+ end
+
+ count += 1
+
+end
+
+puts 
+puts "Found #{count} jobs"
+

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-OR--orchestration_rules* Orchestration rules
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/{workflow/slurm → slurm/task}

@@ -9,8 +9,9 @@ require 'time'
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
---drbbt Use development version of rbbt 
+--drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
 -e--exclusive Make exclusive use of the node
@@ -21,8 +22,9 @@ $slurm_options = SOPT.get <<EOF
 -t--tail Tail the logs
 -q--queue* Queue
 -t--task_cpus* Tasks
+-W--workflows* Additional workflows
 -tm--time* Time
--S--server* SLURM login node
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
 class Step
@@ -30,9 +32,14 @@ class Step
         if done?
             self.load
         else
-            Marenostrum::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
         end
     end
 end
 
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/workflow/task

@@ -20,7 +20,7 @@ def usage(workflow = nil, task = nil, exception=nil, abridge = false)
   puts
   if workflow.nil?
     puts "No workflow specified. Use `rbbt workflow list` to list available workflows."
-    exit -1 
+    exit! -1 
   end
 
   if task.nil?
@@ -206,7 +206,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 EOF
 
 workflow = ARGV.shift
-usage and exit -1 if workflow.nil?
+usage and exit! -1 if workflow.nil?
 
 task     = ARGV.shift
 
@@ -232,7 +232,8 @@ else
   remote_workflows = {}
 end
 
-Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+#Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
 
 workflow = Workflow.require_workflow workflow
 
@@ -486,7 +487,7 @@ rescue ParameterException
   puts
   report_options saved_job_options
   puts
-  exit -1
+  exit! -1
 end
 
 if options.delete(:list_job_files)
@@ -538,7 +539,7 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     rescue Exception
       Log.exception $!
@@ -547,9 +548,11 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     end
+  elsif detach
+    exit! 0
   else
     res.join
     out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

data/test/rbbt/test_workflow.rb

@@ -386,9 +386,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
@@ -427,7 +427,7 @@ class TestWorkflow < Test::Unit::TestCase
     TmpFile.with_file do |dir|
       Path.setup(dir)
       Step.save_job_inputs(job, dir)
-      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.read']
+      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.yaml']
       inputs  = Workflow.load_inputs(dir, [:file], :file => :file)
       assert_equal inputs, {:file => 'code'}
     end
@@ -455,9 +455,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?

data/test/rbbt/tsv/test_attach.rb

@@ -566,7 +566,7 @@ row2,CC
       tsv3.keys.each{|k| tsv3[k] = nil if tsv3[k] == ""}
     end
 
-    assert_equal tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"], [nil, "B", nil]
+    assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
   end
 
   def test_attach_index_both_non_key
@@ -597,10 +597,10 @@ A    Id3
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
 
-    tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce.find #.to_s
+    tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce(true).produce.find #.to_s
 
     tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
     Log.tsv tsv1
@@ -627,12 +627,92 @@ E    B
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
 
     tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
     Log.tsv tsv1
     
   end
+
+  def test_attach_complete
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueB
+row1    b
+row3    C
+    EOF
+    Rbbt.claim Rbbt.tmp.test.test1.data, :string, content1
+    Rbbt.claim Rbbt.tmp.test.test2.data, :string, content2
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.attach tsv2, :complete => true
+    assert_equal [[], ["C"]], tsv1["row3"]
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+
+    ppp tsv1.attach tsv2, :complete => ["AA"]
+    tsv1.attach tsv2, :complete => ["AA"]
+    assert_equal [["AA"], ["C"]], tsv1["row3"]
+  end
+
+  def test_attach_complete_identifiers
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id2    ValueB
+ROW_1    b
+ROW_2    C
+    EOF
+
+    identifiers =<<-EOF
+#: :sep=/\\s+/
+#Id    Id2
+row1    ROW_1
+row2    ROW_2
+row3    ROW_3
+    EOF
+    Rbbt.claim Rbbt.tmp.test.test1.data, :string, content1
+    Rbbt.claim Rbbt.tmp.test.test2.data, :string, content2
+    Rbbt.claim Rbbt.tmp.test.identifiers.data, :string, identifiers
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    ids = Rbbt.tmp.test.identifiers.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1.attach tsv2
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    ids = Rbbt.tmp.test.identifiers.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1.attach tsv2, :complete => true
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+  end
 end