rbbt-util 5.32.25 → 5.32.30

data/lib/rbbt/workflow/step/dependencies.rb

@@ -254,7 +254,12 @@ class Step
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
 
-      cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil? || cpus.to_i == 0
+      if cpus.nil? 
+        keys = ['bootstrap'] + list.collect{|d| [d.task_name, d.task_signature] }.flatten.uniq
+        cpus = config('dep_cpus', *keys, :default => [5, list.length / 2].min)
+      elsif Symbol === cpus
+        cpus = config('dep_cpus', cpus, :default => [5, list.length / 2].min)
+      end
 
       respawn = rest && rest.include?(:respawn)
       respawn = false if rest && rest.include?(:norespawn)
@@ -369,7 +374,8 @@ class Step
       next unless step.dependencies and step.dependencies.any?
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next unless step.dependencies.include?(step_dep)
-        next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
+        next if step_dep.done? or step_dep.running? or 
+          (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end

data/lib/rbbt/workflow/step/run.rb

@@ -419,7 +419,7 @@ class Step
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           config_keys = Rbbt::Config::GOT_KEYS[config_keys_pre.length..-1]
-          set_info :config_keys, config_keys
+          set_info :config_keys, config_keys.uniq
 
           if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
             Open.mv self.tmp_path, self.path

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -0,0 +1,175 @@
+module Workflow
+  def self.load_inputs(dir, input_names, input_types)
+    inputs = {}
+    if File.exists?(dir) && ! File.directory?(dir)
+      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
+      tarfile = dir
+      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
+      tmpdir = Rbbt.tmp.input_bundle[digest].find
+      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+      files = tmpdir.glob("*")
+      if files.length == 1 && File.directory?(files.first)
+        tmpdir = files.first
+      end
+      load_inputs(tmpdir, input_names, input_types)
+    else
+      dir = Path.setup(dir.dup)
+      input_names.each do |input|
+        file = dir[input].find
+        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
+        Log.debug "Trying #{ input }: #{file}"
+        next unless file and (File.symlink?(file) || file.exists?)
+
+        type = input_types[input]
+
+        type = :io if file.split(".").last == 'as_io'
+
+        type = :path if file.split(".").last == 'as_path'
+
+        case type
+        when :path
+          inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
+        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :file, :binary
+          Log.debug "Pointing #{ input } to #{file}"
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
+          else
+            if File.symlink?(file)
+              link_target = File.expand_path(File.readlink(file), File.dirname(file))
+              inputs[input.to_sym]  = link_target
+            else
+              inputs[input.to_sym]  = Open.realpath(file)
+            end
+          end
+        when :text
+          Log.debug "Reading #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file)
+        when :array
+          Log.debug "Reading array #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file).split("\n")
+        when :tsv
+          Log.debug "Opening tsv #{ input } from #{file}"
+          inputs[input.to_sym]  = TSV.open(file)
+        when :boolean
+          inputs[input.to_sym]  = (file.read.strip == 'true')
+        else
+          Log.debug "Loading #{ input } from #{file}"
+          inputs[input.to_sym]  = file.read.strip
+        end
+
+      end
+      inputs = IndiferentHash.setup(inputs)
+
+      dir.glob("*#*").each do |od|
+        name = File.basename(od)
+        value = Open.read(od)
+        Log.debug "Loading override dependency #{ name } as #{value}"
+        inputs[name] = value.chomp
+      end
+
+      inputs
+    end
+  end
+
+  def task_inputs_from_directory(task_name, directory)
+    task_info = self.task_info(task_name)
+    Workflow.load_inputs(directory, task_info[:inputs], task_info[:input_types])
+  end
+
+  def job_for_directory_inputs(task_name, directory, jobname = nil)
+    inputs = task_inputs_from_directory(task_name, directory)
+    job(task_name, jobname, inputs)
+  end
+
+end
+
+class Step
+  def self.save_inputs(inputs, input_types, input_options, dir)
+    inputs.each do |name,value|
+      type = input_types[name]
+      type = type.to_s if type
+      path = File.join(dir, name.to_s)
+
+      path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
+      Log.debug "Saving job input #{name} (#{type}) into #{path}"
+
+      case
+      when IO === value
+        Open.write(path, value.to_s)
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "binary"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        elsif String === value && Misc.is_filename?(value, false)
+          Open.write(path + '.as_path' , value)
+        else
+          Open.write(path, value, :mode => 'wb')
+        end
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        else
+          value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
+          value = "#{value}" if Path === value
+          Open.write(path + '.yaml', value.to_yaml)
+        end
+      when Array === value
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
+      when IO === value
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
+        else
+          Open.write(path, value)
+        end
+      else
+        Open.write(path, value.to_s)
+      end
+    end.any?
+  end
+
+  def self.save_job_inputs(job, dir, options = nil)
+    options = IndiferentHash.setup options.dup if options
+
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
+    workflow = job.workflow
+    workflow = Kernel.const_get workflow if String === workflow
+    if workflow
+      task_info = IndiferentHash.setup(workflow.task_info(task_name))
+      input_types = IndiferentHash.setup(task_info[:input_types])
+      input_options = IndiferentHash.setup(task_info[:input_options])
+      task_inputs = IndiferentHash.setup(task_info[:inputs])
+      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
+    else
+      task_info = IndiferentHash.setup({})
+      input_types = IndiferentHash.setup({})
+      task_inputs = IndiferentHash.setup({})
+      task_options = IndiferentHash.setup({})
+      input_defaults = IndiferentHash.setup({})
+    end
+
+    inputs = IndiferentHash.setup({})
+    real_inputs = job.real_inputs || job.info[:real_inputs]
+    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
+      next unless task_inputs.include? name.to_sym
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
+      next if value.nil?
+      next if input_defaults[name] == value
+      inputs[name] = value
+    end
+
+    if options && options.include?('override_dependencies')
+      inputs.merge!(:override_dependencies => open[:override_dependencies])
+      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
+    end
+
+    save_inputs(inputs, input_types, input_options, dir)
+
+    inputs.keys
+  end
+end

data/lib/rbbt/workflow/step.rb

@@ -6,6 +6,7 @@ require 'rbbt/workflow/step/accessor'
 require 'rbbt/workflow/step/prepare'
 require 'rbbt/workflow/step/status'
 require 'rbbt/workflow/step/info'
+require 'rbbt/workflow/step/save_load_inputs'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
@@ -321,7 +322,7 @@ class Step
   def load
     res = begin
             @result = nil if IO === @result && @result.closed?
-            if @result && @path != @result
+            if @result && @path != @result &&  ! StreamArray === @result
               res = @result
             else
               join if not done?

data/lib/rbbt/workflow/task.rb

@@ -90,11 +90,11 @@ module Task
 
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task.name]
+      next if seen.include? [wf, task.name, maps]
 
       task.workflow = wf if wf
 
-      seen << [wf, task.name]
+      seen << [wf, task.name, maps]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
       if task_inputs[task].nil?

data/lib/rbbt/workflow.rb

@@ -244,7 +244,7 @@ module Workflow
                        when :hash
                          clean_inputs = Annotated.purge(inputs)
                          clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (Symbol === d.overriden ? d.path : d.short_path) : d }
                          key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)
@@ -465,7 +465,14 @@ module Workflow
       extension = nil
       if dependencies.any?
         dep_basename = File.basename(dependencies.last.path)
-        extension = dep_basename.split(".").last if dep_basename.include?('.')
+        if dep_basename.include? "."
+          parts = dep_basename.split(".")
+          extension = [parts.pop]
+          while parts.last.length <= 4
+            extension << parts.pop
+          end
+          extension = extension.reverse * "."
+        end
       end
     end
 

data/share/rbbt_commands/hpc/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/lsf/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/slurm/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/tsv/keys

@@ -41,19 +41,18 @@ parser = TSV::Parser.new TSV.get_stream(file), options.merge(:fields => [])
 
 options[:merge] = false if options[:merge] == "false"
 
-Thread.new do
-  line = parser.first_line
-  bar = Log::ProgressBar.new
-  while line 
-    bar.tick
-
-    line = Misc.fixutf8(line)
-    line = parser.process line
-    raise SKIP_LINE if line.empty?
-    parts = parser.chop_line line
-    key, values = parser.get_values parts
-    values = parser.cast_values values if parser.cast?
-    
-    puts key
-  end
+line = parser.first_line
+bar = Log::ProgressBar.new
+while line 
+  bar.tick
+
+  line = Misc.fixutf8(line)
+  line = parser.process line
+  raise SKIP_LINE if line.empty?
+  parts = parser.chop_line line
+  key, values = parser.get_values parts
+  values = parser.cast_values values if parser.cast?
+
+  puts key
+  line = parser.stream.gets
 end

data/share/rbbt_commands/tsv/read_excel

@@ -22,7 +22,7 @@ Use - to read from STDIN
 -h--help Print this help
 -s--sheet* Sheet to extract
 -skip--skip_rows* Initial rows to skip
-
+-o--original Dump the rows without parsing them into TSV
 EOF
 if options[:help]
   if defined? rbbt_usage
@@ -39,5 +39,5 @@ raise ParameterException, "No excel file given" if excelfile.nil?
 
 options[:zipped] ||= true if options[:merge]
 require 'rbbt/tsv/excel'
-puts TSV.excel(excelfile, options).to_s
+puts TSV.excel(excelfile, options.merge(:text => options[:original]))
 

data/share/rbbt_commands/workflow/task

@@ -104,7 +104,7 @@ def fix_options(workflow, task, job_options)
               elsif input_options[name] and input_options[name][:stream] and value == "-"
                 STDIN
               else
-                if Array === value
+                if Array === value || IO === value
                   value
                 else
                   array_separator = $array_separator
@@ -137,6 +137,8 @@ def fix_options(workflow, task, job_options)
                   TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 when (Misc.is_filename?(value) and String)
                   TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
+                when IO
+                  TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 else
                   TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 end
@@ -571,10 +573,14 @@ when Step
   elsif detach
     exit! 0
   else
-    res.join
-    Open.open(res.path, :mode => 'rb') do |io|
-      Misc.consume_stream(io, false, out)
-    end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    if %w(float integer string boolean).include?(res.result_type.to_s)
+      out.puts res.load
+    else
+      res.join
+      Open.open(res.path, :mode => 'rb') do |io|
+        Misc.consume_stream(io, false, out)
+      end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    end
   end
 else
   if Array === res

data/test/rbbt/annotations/test_util.rb

@@ -1,6 +1,11 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
 require 'rbbt/annotations'
 
+module TestEntityString
+  extend Entity
+  self.annotation :code
+end
+
 class TestAnnotation < Test::Unit::TestCase
   def test_marshal
     a = "STRING"
@@ -9,5 +14,11 @@ class TestAnnotation < Test::Unit::TestCase
     assert !(Annotated === Marshal.load(Marshal.dump(a)))
     assert_equal a, Marshal.load(Marshal.dump(a))
   end
+
+  def test_hash
+    e = TestEntityString.setup("TEST", :code => 10)
+    assert_equal "TEST", Annotated.to_hash(e)[:literal]
+    assert_equal 10, Annotated.to_hash(e)[:info][:code]
+  end
 end
 

data/test/rbbt/hpc/orchestrate/test_batches.rb

@@ -0,0 +1,113 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow'
+require 'rbbt/hpc/orchestrate/batches'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+class TestOrchestrate < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+    time: 10min
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+    time: 10s
+  EOF
+
+
+  def test_job_batches_d
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 3, batches.length
+  end
+
+  def test_job_batches_c3
+    job = TestWFC.job(:c3, nil)
+    job.recursive_clean
+
+    batches = HPC::Orchestration.job_batches(RULES, job)
+  end
+
+  def test_job_batches_c4
+    job = TestWFC.job(:c4, nil)
+    job.recursive_clean
+
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 3, batches.length
+  end
+
+end