rbbt-util 5.44.1 → 6.0.3

data/share/rbbt_commands/slurm/orchestrate

@@ -18,9 +18,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -57,7 +57,7 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 class Step
   def join
-    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir)
+    HPC::BATCH_MODULE.wait_for_job(@batch_job_dir) unless done?
   end
 
   def run(no_load = true)

data/share/rbbt_commands/slurm/tail

@@ -33,6 +33,8 @@ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint
 
 directory = ARGV.shift
 
+directory = HPC::BATCH_MODULE.jobs.first if directory.nil?
+
 raise ParameterException if directory.nil?
 
 if directory =~ /^[0-9]*$/

data/share/rbbt_commands/slurm/task

@@ -9,7 +9,7 @@ require 'time'
 
 rbbt_options = SOPT::GOT_OPTIONS
 
-$slurm_options = SOPT.get <<EOF
+$batch_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
 --drbbt* Use development version of rbbt 
@@ -17,9 +17,9 @@ $slurm_options = SOPT.get <<EOF
 -si--singularity_img* Singularity image to use
 -sm--singularity_mounts* Singularity image to use
 -ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
+--contain* Contain in directory (using Singularity)
+--sync* Contain in directory and sync jobs
+--exclusive Make exclusive use of the node
 -hm--highmem Make use of highmem cores
 -wc--wipe_container* Wipe the jobs from the contain directory
 -pd--purge_deps Purge job dependencies
@@ -45,11 +45,14 @@ $slurm_options = SOPT.get <<EOF
 -OR--orchestration_rules* Orchestration rules
 EOF
 
-batch_system = $slurm_options.delete :batch_system
+batch_system = $batch_options.delete :batch_system
 batch_system ||= 'auto'
+$batch_options[:log] = rbbt_options[:log]
+$batch_options[:config_keys] = rbbt_options[:config_keys]
 
 HPC::BATCH_MODULE = HPC.batch_system batch_system
 
+
 raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
 
 class Step
@@ -60,7 +63,7 @@ class Step
     else
       begin
         Log.debug "Issuing BATCH job for #{self.path}"
-        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
+        HPC::BATCH_MODULE.run_job(self, $batch_options) 
       rescue HPC::BATCH_DRY_RUN
       end
     end
@@ -70,7 +73,7 @@ end
 module RemoteStep::SSH
 
   def _run
-    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $batch_options)
   end
 
 end

data/share/rbbt_commands/system/clean

@@ -22,7 +22,7 @@ Specify workflow '.' and no task to examine the jobs of the current directory (u
 -d--dirty Clean dirty jobs
 -h--help Print this help
 EOF
-rbbt_usage and exit 0 if options[:help]
+scout_usage and exit 0 if options[:help]
 
 workflow, task = ARGV
 
@@ -98,7 +98,7 @@ TSV.traverse jobs do |file,i|
   else
     info = begin
              Open.open(i[:info_file]) do |f|
-               Step::INFO_SERIALIZER.load(f)
+               Step.load_info(f)
              end
            rescue
              {:status => :noinfo}
@@ -136,9 +136,9 @@ TSV.traverse jobs do |file,i|
   end
 
   if (force && status !~ /done/) or 
-    status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync)$/ or
-    (status == "noinfo" and not done) or
-    status == "" 
+      status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync|waiting)$/ or
+      (status == "noinfo" and not done) or
+      status == "" 
 
     puts "  Removing #{ file } - #{status}" 
     Step.clean(file) 

data/share/rbbt_commands/system/status

@@ -74,7 +74,7 @@ end
 def input_msg(file, inputs)
 
   str = ""
-  job_inputs = Workflow.load_step(file).recursive_inputs.to_hash
+  job_inputs = Step.new(file).recursive_inputs.to_hash
   IndiferentHash.setup(job_inputs)
 
   inputs.each do |input|
@@ -187,9 +187,10 @@ workflows.sort.each do |workflow,tasks|
         if info_fields and info_fields.any?
           info = begin
                    Open.open(i[:info_file]) do |f|
-                     Step::INFO_SERIALIZER.load(f)
+                     Step.load_info(f)
                    end
                  rescue
+                   Log.exception $!
                    {:status => :noinfo}
                  end
           IndiferentHash.setup(info)
@@ -198,10 +199,9 @@ workflows.sort.each do |workflow,tasks|
       else
         info = begin
                  Open.open(i[:info_file]) do |f|
-                   Step::INFO_SERIALIZER.load(f)
+                   Step.load_info(f)
                  end
                rescue
-                 #Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)
@@ -219,7 +219,7 @@ workflows.sort.each do |workflow,tasks|
           end
         end
         str << " #{ Step.prov_status_msg status }"
-        str << " (dirty)" if status == 'done' && Workflow.load_step(file).dirty?
+        str << " (dirty)" if status == 'done' && Step.new(file).dirty?
 
         if inputs and inputs.any? 
           str << input_msg(file, inputs)

data/share/rbbt_commands/tsv/get

@@ -78,11 +78,10 @@ if TSV === tsv
   exit
 end
 
-parser = TSV::Parser.new tsv, :key_field => key_field,  :fields => fields, :type => options[:type], :header_hash => options[:header_hash], :sep => options[:sep]
+parser = TSV::Parser.new tsv, **options
 fields ||= parser.fields
-
 i = 0
-TSV.traverse(parser) do |k,v|
+TSV.traverse(parser, :key_field => key_field,  :fields => fields, :type => options[:type]) do |k,v|
   if key== "#{i}"
     key = k.first
   elsif key.nil?

data/share/rbbt_commands/tsv/info

@@ -14,10 +14,7 @@ Display summary information. Works with Tokyocabinet HDB and BDB as well.
 
 -tch--tokyocabinet File is a TC HDB
 -tcb--tokyocabinet_bd File is a TC BDB
--t--type* Type of tsv (single, list, double, flat)
 -hh--header_hash* Change the character used to mark the header line (defaults to #)
--k--key_field* Change the key field
--f--fields* Change the fields to load
 -fp--fingerprint Fingerprint values
 -s--sep* Change the fields separator (default TAB)
 -h--help Help
@@ -32,8 +29,8 @@ file = STDIN if file == '-'
 raise ParameterException, "Please specify the tsv file as argument" if file.nil?
 
 options[:fields] = options[:fields].split(/[,\|]/) if options[:fields]
-options[:header_hash] = options["header_hash"]
-options[:sep] = options["sep"]
+
+options.keys_to_sym!
 
 case
 when options[:tokyocabinet]
@@ -43,16 +40,16 @@ when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
   puts tsv.summary
 else
-  header = TSV.parse_header(Open.open(file, :nocache => true), options)
+  options, key_field, fields, first_line = TSV.parse_header(Open.open(file, :nocache => true), **options)
 
   puts "File: #{ file }"
-  puts "Type: #{header.type}"
-  puts "Namespace: #{header.namespace}"
-  puts "Key: #{Log.color :yellow, header.key_field}"
+  puts "Type: #{options[:type]}"
+  puts "Namespace: #{options[:namespace]}"
+  puts "Key: #{Log.color :yellow, key_field}"
   puts "Fields: "
-  if header.fields.nil?
+  if fields.nil?
   else
-    header.fields.each_with_index do |f,i|
+    fields.each_with_index do |f,i|
       puts "  - #{Log.color :cyan, i + 1}: " << Log.color(:yellow, f)
     end
   end
@@ -71,9 +68,9 @@ else
   
   puts "Rows: #{Log.color :blue, rows}" 
   parts = []
-  header.first_line.split(header.sep, -1).each_with_index do |p,i| 
+  first_line.split(options[:sep] || "\t", -1).each_with_index do |p,i| 
    p.strip!
-   p = Misc.fingerprint p.split("|") if %w(double flat).include?(header.options[:type].to_s) && options[:fingerprint]
+   p = Misc.fingerprint p.split("|") if %w(double flat).include?(options[:type].to_s) && options[:fingerprint]
    parts << (Log.color(:cyan, "(#{i}) ") << p.strip) 
   end
   puts parts * "\t"

data/share/rbbt_commands/tsv/keys

@@ -37,22 +37,26 @@ when options[:tokyocabinet_bd]
   exit
 end
 
-parser = TSV::Parser.new TSV.get_stream(file), options.merge(:fields => [])
+parser = TSV::Parser.new TSV.get_stream(file)
 
 options[:merge] = false if options[:merge] == "false"
 
-line = parser.first_line
-bar = Log::ProgressBar.new
-while line 
-  bar.tick
-
-  line = Misc.fixutf8(line)
-  line = parser.process line
-  raise SKIP_LINE if line.empty?
-  parts = parser.chop_line line
-  key, values = parser.get_values parts
-  values = parser.cast_values values if parser.cast?
-
+parser.traverse fields: [] do |key,v|
   puts key
-  line = parser.stream.gets
 end
+
+#line = parser.first_line
+#bar = Log::ProgressBar.new
+#while line 
+#  bar.tick
+#
+#  line = Misc.fixutf8(line)
+#  line = parser.traverse line
+#  raise SKIP_LINE if line.empty?
+#  parts = parser.chop_line line
+#  key, values = parser.get_values parts
+#  values = parser.cast_values values if parser.cast?
+#
+#  puts key
+#  line = parser.stream.gets
+#end

data/share/rbbt_commands/tsv/slice

@@ -35,7 +35,7 @@ file = case file
 fields = options[:fields]
 raise ParameterException, "Please specify the fields to slice" if fields.nil?
 
-options[:header_hash] ||= options["header_hash"]
+options.keys_to_sym!
 
 case
 when options[:tokyocabinet]
@@ -45,7 +45,7 @@ when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
   puts tsv.summary
 else
-  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
+  stream = TSV.traverse file, **options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
     [k,fields].flatten * "\t"
   end
   puts stream.read

data/share/rbbt_commands/tsv/transpose

@@ -32,9 +32,13 @@ file = case file
          raise ParameterException, "Please specify the tsv file as argument" if file.nil?
        end
 
-field = options[:field]
 
-options[:header_hash] = options["header_hash"]
+options[:header_hash] ||= options.delete "header_hash"
+options.delete :header_hash if options[:header_hash].nil?
+
+field = options.delete :field
+
+options.keys_to_sym!
 
 case
 when options[:tokyocabinet]

data/share/rbbt_commands/workflow/info

@@ -77,8 +77,8 @@ if options[:original]
   exit 0
 end
 
+inputs = step.inputs
 dependencies = info[:dependencies]
-inputs = info[:inputs]
 status = info[:status]
 time = info[:time_elapsed]
 total_time = info[:total_time_elapsed]
@@ -89,6 +89,7 @@ exception = info[:exception]
 rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
 
 
+status = :noinfo if status.nil?
 puts Log.color(:magenta, "File") << ": " << step.path
 puts Log.color(:magenta, "Status") << ": " << status_msg(status) << ((step.aborted? || step.error?) && step.recoverable_error? ? " (recoverable)" : "" ) << (step.dirty? ? " (dirty)" : "")
 puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
@@ -96,17 +97,19 @@ puts Log.color(:magenta, "Time") << ": " << time.to_i.to_s << " sec." if time
 puts Log.color(:magenta, "Total time") << ": " << total_time.to_i.to_s << " sec." if time
 if inputs and inputs.any?
   puts Log.color(:magenta, "Inputs")
-  inputs.each do |input,value|
+  inputs.to_hash.each do |input,value|
     case value
     when nil
-      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', width, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', 20, width, color: :blue)
     when Array
-      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), width, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 20, width, color: :blue)
     when TrueClass, FalseClass
-      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, width, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, 20, width, color: :blue)
+    when Step
+      puts Misc.format_definition_list_item("  " << input.to_s, value.short_path, 20, width, color: :blue)
     else
       text = value.to_s.split("\n")[0..5].compact * "\n\n"
-      puts Misc.format_definition_list_item("  " + input.to_s, text, width, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, text, 20, width, color: :blue)
     end
   end
 end
@@ -150,32 +153,25 @@ if all
 end
 
 if recursive
-  inputs = step.info[:inputs] || {}
-  deps = step.dependencies
-
-  while deps.any? do
-    dep = deps.shift
-    inputs = {} if inputs.nil?
-    inputs = inputs.merge(dep.info[:inputs] || {})
-    deps = deps.concat dep.dependencies if dep.dependencies
-  end
+  inputs = step.recursive_inputs
+  inputs = NamedArray.setup(inputs.values, inputs.keys)
 
-  inputs = inputs.merge step.archived_inputs
+  inputs = inputs.concat step.archived_inputs
   if inputs.any?
     puts Log.color(:magenta, "Recursive inputs") << ": "
-    inputs.each do |input,value|
+    inputs.to_hash.each do |input,value|
       case value
       when nil
-        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', width, 20, :blue)
+        puts Misc.format_definition_list_item("  " + input.to_s, 'nil', 20, width, color: :blue)
       when Array
-        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), width, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 20, width, color: :blue)
       when TrueClass, FalseClass
-        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, width, 20, :blue)
+        puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, 20, width, color: :blue)
+      when Step
+        puts Misc.format_definition_list_item("  " << input.to_s, value.short_path, 20, width, color: :blue)
       else
-        lines = value.to_s.split("\n").collect{|l| l.length >= width - 5 ? l[0..width - 5] + " ..." : l }
-        text = lines[0..5].compact * "\n\n"
-        text << "\n\n...\n\n" if lines.length > 6
-        puts Misc.format_definition_list_item("  " << input.to_s, text, width, 20, :blue).gsub("\n\n","\n")
+        text = value.to_s.split("\n")[0..5].compact * "\n\n"
+        puts Misc.format_definition_list_item("  " + input.to_s, text, 20, width, color: :blue)
       end
     end
   end

data/share/rbbt_commands/workflow/list

@@ -6,7 +6,7 @@ require 'rbbt/util/simpleopt'
 
 options = SOPT.get('-wd--workflow_dir*:-d--describe')
 
-workflow_dir = options[:workflow_dir] || Workflow.workflow_dir
+workflow_dir = options[:workflow_dir] || Rbbt.workflows
 
 Path.setup(workflow_dir) unless Path === workflow_dir
 

data/share/rbbt_commands/workflow/prov

@@ -1,23 +1,24 @@
 #!/usr/bin/env ruby
 
-require 'rbbt/workflow'
-
-require 'rbbt-util'
-require 'fileutils'
-require 'rbbt/util/simpleopt'
-require 'rbbt/workflow/step'
-require 'rbbt/workflow/util/provenance'
-require 'rbbt/util/misc'
-
-require 'rbbt-util'
-require 'rbbt/util/simpleopt'
+require 'scout/workflow'
+
+#require 'rbbt/workflow'
+#require 'rbbt-util'
+#require 'fileutils'
+#require 'rbbt/util/simpleopt'
+#require 'rbbt/workflow/step'
+#require 'rbbt/workflow/util/provenance'
+#require 'rbbt/util/misc'
+#
+#require 'rbbt-util'
+#require 'rbbt/util/simpleopt'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
 options = SOPT.setup <<EOF
 Examine the provenance of a job result
 
-$ rbbt workflow prov <job-result>
+$ rbbt workflow prov <job_result> [<dependency_task>]
 
 -h--help Help
 -p--plot* draw the dependency plot into <file.png>
@@ -34,12 +35,15 @@ $info_fields = (options[:info_fields] || "").split(",")
 
 file = ARGV.shift
 
+raise MissingParameterException, :job_result if file.nil?
+dependency_task = ARGV.shift
+
 $seen = []
 def get_step(file)
   file = File.expand_path(file) if File.exist?(file)
   file = file.sub(/\.(info|files)$/,'')
   $seen << file
-  Workflow.load_step file
+  Step.load file
 end
 
 def touch(step)
@@ -130,6 +134,9 @@ if options[:plot]
     end
   end
 
+elsif dependency_task
+  deps = step.rec_dependencies.select{|dep| dep.task_name.to_s == dependency_task}
+  puts deps.collect{|dep| dep.path } * "\n"
 else
   puts Step.prov_report(step, 0, nil, [], options[:expand_repeats])
 end

data/share/rbbt_commands/workflow/server

@@ -127,7 +127,11 @@ TmpFile.with_file do |app_dir|
     when 'puma_production'
       #`puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
       CMD.cmd_log("env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload")
+    when 'puma_single'
+      #`puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
+      CMD.cmd_log("env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 1 -t 1:1 ")
     else
+
       options[:config] = config_ru_file
 
       fixed_options = {}
@@ -138,7 +142,13 @@ TmpFile.with_file do |app_dir|
         Rack::Server.start(options)
       rescue LoadError
         require 'rackup'
-        Rackup::Server.start(options)
+
+        begin
+          Rackup::Server.start(options)
+        rescue Exception
+          Log.exception $!
+          raise $!
+        end
       end
     end
   end