rbbt-util 5.32.26 → 5.33.0

data/lib/rbbt/workflow/step/accessor.rb

@@ -44,7 +44,7 @@ class Step
   end
 
   def self.files_dir(path)
-    path.nil? ? nil : path + '.files'
+    path.nil? ? nil : Path.setup(path + '.files')
   end
 
   def self.info_file(path)
@@ -86,76 +86,6 @@ class Step
     end
   end
 
-  def self.save_inputs(inputs, input_types, dir)
-    inputs.each do |name,value|
-      type = input_types[name]
-      type = type.to_s if type
-      path = File.join(dir, name.to_s)
-
-      Log.debug "Saving job input #{name} (#{type}) into #{path}"
-      case
-      when Step === value
-        Open.ln_s(value.path, path)
-      when type.to_s == "file"
-        if String === value && File.exists?(value)
-          value = File.expand_path(value)
-          Open.ln_s(value, path)
-        else
-          value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
-          value = "#{value}" if Path === value
-          Open.write(path + '.yaml', value.to_yaml)
-        end
-      when Array === value
-        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
-      when IO === value
-        if value.filename && String === value.filename && File.exists?(value.filename)
-          Open.ln_s(value.filename, path)
-        else
-          Open.write(path, value)
-        end
-      else
-        Open.write(path, value.to_s)
-      end
-    end.any?
-  end
-
-  def self.save_job_inputs(job, dir, options = nil)
-    options = IndiferentHash.setup options.dup if options
-
-    task_name = Symbol === job.overriden ? job.overriden : job.task_name
-    workflow = job.workflow
-    workflow = Kernel.const_get workflow if String === workflow
-    if workflow
-      task_info = IndiferentHash.setup(workflow.task_info(task_name))
-      input_types = IndiferentHash.setup(task_info[:input_types])
-      task_inputs = IndiferentHash.setup(task_info[:inputs])
-      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
-    else
-      task_info = IndiferentHash.setup({})
-      input_types = IndiferentHash.setup({})
-      task_inputs = IndiferentHash.setup({})
-      input_defaults = IndiferentHash.setup({})
-    end
-
-    inputs = IndiferentHash.setup({})
-    real_inputs = job.real_inputs || job.info[:real_inputs]
-    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
-      next unless task_inputs.include? name.to_sym
-      next unless real_inputs.include? name.to_sym
-      next if options && ! options.include?(name)
-      next if value.nil?
-      next if input_defaults[name] == value
-      inputs[name] = value
-    end
-
-    if options && options.include?('override_dependencies')
-      inputs.merge!(:override_dependencies => open[:override_dependencies])
-      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
-    end
-    save_inputs(inputs, input_types, dir)
-
-    inputs.keys
-  end
 
   def name
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
@@ -224,8 +154,12 @@ class Step
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
+      value = Annotated.purge(value)
+
       i[key] = value 
+
       dump = Step.serialize_info(i)
+
       @info_cache = IndiferentHash.setup(i)
       Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now

data/lib/rbbt/workflow/step/dependencies.rb

@@ -254,7 +254,12 @@ class Step
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
 
-      cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil? || cpus.to_i == 0
+      if cpus.nil? 
+        keys = ['bootstrap'] + list.collect{|d| [d.task_name, d.task_signature] }.flatten.uniq
+        cpus = config('dep_cpus', *keys, :default => [5, list.length / 2].min)
+      elsif Symbol === cpus
+        cpus = config('dep_cpus', cpus, :default => [5, list.length / 2].min)
+      end
 
       respawn = rest && rest.include?(:respawn)
       respawn = false if rest && rest.include?(:norespawn)
@@ -369,7 +374,8 @@ class Step
       next unless step.dependencies and step.dependencies.any?
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next unless step.dependencies.include?(step_dep)
-        next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
+        next if step_dep.done? or step_dep.running? or 
+          (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end

data/lib/rbbt/workflow/step/info.rb

@@ -101,7 +101,7 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      if dep_info
+      if Hash === dep_info
         dep_info[:inputs].each do |k,v|
           all_inputs[k] = v unless all_inputs.include?(k)
         end if dep_info[:inputs]

data/lib/rbbt/workflow/step/run.rb

@@ -419,7 +419,7 @@ class Step
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           config_keys = Rbbt::Config::GOT_KEYS[config_keys_pre.length..-1]
-          set_info :config_keys, config_keys
+          set_info :config_keys, config_keys.uniq
 
           if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
             Open.mv self.tmp_path, self.path

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -0,0 +1,179 @@
+module Workflow
+  def self.load_inputs(dir, input_names, input_types)
+    inputs = {}
+    if File.exists?(dir) && ! File.directory?(dir)
+      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
+      tarfile = dir
+      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
+      tmpdir = Rbbt.tmp.input_bundle[digest].find
+      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
+      files = tmpdir.glob("*")
+      if files.length == 1 && File.directory?(files.first)
+        tmpdir = files.first
+      end
+      load_inputs(tmpdir, input_names, input_types)
+    else
+      dir = Path.setup(dir.dup)
+      input_names.each do |input|
+        file = dir[input].find
+        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
+        Log.debug "Trying #{ input }: #{file}"
+        next unless file and (File.symlink?(file) || file.exists?)
+
+        type = input_types[input]
+
+        type = :io if file.split(".").last == 'as_io'
+
+        type = :path if file.split(".").last == 'as_path'
+
+        type = :nofile if file.split(".").last == 'nofile'
+
+        case type
+        when :nofile
+          inputs[input.to_sym]  = Open.realpath(file)
+        when :path
+          inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
+        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :file, :binary
+          Log.debug "Pointing #{ input } to #{file}"
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
+          else
+            if File.symlink?(file)
+              link_target = File.expand_path(File.readlink(file), File.dirname(file))
+              inputs[input.to_sym]  = link_target
+            else
+              inputs[input.to_sym]  = Open.realpath(file)
+            end
+          end
+        when :text
+          Log.debug "Reading #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file)
+        when :array
+          Log.debug "Reading array #{ input } from #{file}"
+          inputs[input.to_sym]  = Open.read(file).split("\n")
+        when :tsv
+          Log.debug "Opening tsv #{ input } from #{file}"
+          inputs[input.to_sym]  = TSV.open(file)
+        when :boolean
+          inputs[input.to_sym]  = (file.read.strip == 'true')
+        else
+          Log.debug "Loading #{ input } from #{file}"
+          inputs[input.to_sym]  = file.read.strip
+        end
+
+      end
+      inputs = IndiferentHash.setup(inputs)
+
+      dir.glob("*#*").each do |od|
+        name = File.basename(od)
+        value = Open.read(od)
+        Log.debug "Loading override dependency #{ name } as #{value}"
+        inputs[name] = value.chomp
+      end
+
+      inputs
+    end
+  end
+
+  def task_inputs_from_directory(task_name, directory)
+    task_info = self.task_info(task_name)
+    Workflow.load_inputs(directory, task_info[:inputs], task_info[:input_types])
+  end
+
+  def job_for_directory_inputs(task_name, directory, jobname = nil)
+    inputs = task_inputs_from_directory(task_name, directory)
+    job(task_name, jobname, inputs)
+  end
+
+end
+
+class Step
+  def self.save_inputs(inputs, input_types, input_options, dir)
+    inputs.each do |name,value|
+      type = input_types[name]
+      type = type.to_s if type
+      path = File.join(dir, name.to_s)
+
+      path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
+      Log.debug "Saving job input #{name} (#{type}) into #{path}"
+
+      case
+      when IO === value
+        Open.write(path, value.to_s)
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "binary"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        elsif String === value && Misc.is_filename?(value, false)
+          Open.write(path + '.as_path' , value)
+        else
+          Open.write(path, value, :mode => 'wb')
+        end
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          value = File.expand_path(value)
+          Open.ln_s(value, path)
+        else
+          value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
+          value = "#{value}" if Path === value
+          Open.write(path + '.yaml', value.to_yaml)
+        end
+      when Array === value
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
+      when IO === value
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
+        else
+          Open.write(path, value)
+        end
+      else
+        Open.write(path, value.to_s)
+      end
+    end.any?
+  end
+
+  def self.save_job_inputs(job, dir, options = nil)
+    options = IndiferentHash.setup options.dup if options
+
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
+    workflow = job.workflow
+    workflow = Kernel.const_get workflow if String === workflow
+    if workflow
+      task_info = IndiferentHash.setup(workflow.task_info(task_name))
+      input_types = IndiferentHash.setup(task_info[:input_types])
+      input_options = IndiferentHash.setup(task_info[:input_options])
+      task_inputs = IndiferentHash.setup(task_info[:inputs])
+      input_defaults = IndiferentHash.setup(task_info[:input_defaults])
+    else
+      task_info = IndiferentHash.setup({})
+      input_types = IndiferentHash.setup({})
+      task_inputs = IndiferentHash.setup({})
+      task_options = IndiferentHash.setup({})
+      input_defaults = IndiferentHash.setup({})
+    end
+
+    inputs = IndiferentHash.setup({})
+    real_inputs = job.real_inputs || job.info[:real_inputs]
+    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
+      next unless task_inputs.include? name.to_sym
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
+      next if value.nil?
+      next if input_defaults[name] == value
+      inputs[name] = value
+    end
+
+    if options && options.include?('override_dependencies')
+      inputs.merge!(:override_dependencies => open[:override_dependencies])
+      input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
+    end
+
+    save_inputs(inputs, input_types, input_options, dir)
+
+    inputs.keys
+  end
+end

data/lib/rbbt/workflow/step.rb

@@ -6,6 +6,7 @@ require 'rbbt/workflow/step/accessor'
 require 'rbbt/workflow/step/prepare'
 require 'rbbt/workflow/step/status'
 require 'rbbt/workflow/step/info'
+require 'rbbt/workflow/step/save_load_inputs'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
@@ -321,7 +322,7 @@ class Step
   def load
     res = begin
             @result = nil if IO === @result && @result.closed?
-            if @result && @path != @result
+            if @result && @path != @result &&  ! StreamArray === @result
               res = @result
             else
               join if not done?

data/lib/rbbt/workflow/task.rb

@@ -90,11 +90,11 @@ module Task
 
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
-      next if seen.include? [wf, task.name]
+      next if seen.include? [wf, task.name, maps]
 
       task.workflow = wf if wf
 
-      seen << [wf, task.name]
+      seen << [wf, task.name, maps]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?
       if task_inputs[task].nil?

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -65,7 +65,10 @@ module Workflow
 
       IndiferentHash.setup(resources)
 
-      default_resources = rules["default_resources"] || rules["defaults"]["resources"]
+      default_resources = rules["default_resources"] 
+      default_resources ||= rules["defaults"]["resources"] if rules["defaults"]
+      default_resources ||= {}
+
       default_resources.each{|k,v| resources[k] ||= v } if default_resources
 
       resources
@@ -98,6 +101,10 @@ module Workflow
       candidates
     end
 
+    def self.process(*args)
+      self.new.process(*args)
+    end
+
     attr_accessor :available_resources, :resources_requested, :resources_used, :timer
 
     def initialize(timer = 5, available_resources = {})
@@ -176,7 +183,9 @@ module Workflow
       end
     end
 
-    def process(rules, jobs)
+    def process(rules, jobs = nil)
+      jobs, rules = rules, {} if jobs.nil?
+      jobs = [jobs] if Step === jobs
       begin
 
         workload = Orchestrator.workload(jobs)

data/lib/rbbt/workflow.rb

@@ -244,7 +244,7 @@ module Workflow
                        when :hash
                          clean_inputs = Annotated.purge(inputs)
                          clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (Symbol === d.overriden ? d.path : d.short_path) : d }
                          key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)
@@ -465,7 +465,14 @@ module Workflow
       extension = nil
       if dependencies.any?
         dep_basename = File.basename(dependencies.last.path)
-        extension = dep_basename.split(".").last if dep_basename.include?('.')
+        if dep_basename.include? "."
+          parts = dep_basename.split(".")
+          extension = [parts.pop]
+          while parts.last.length <= 4
+            extension << parts.pop
+          end
+          extension = extension.reverse * "."
+        end
       end
     end
 

data/share/rbbt_commands/hpc/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/lsf/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/slurm/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/tsv/keys

@@ -41,19 +41,18 @@ parser = TSV::Parser.new TSV.get_stream(file), options.merge(:fields => [])
 
 options[:merge] = false if options[:merge] == "false"
 
-Thread.new do
-  line = parser.first_line
-  bar = Log::ProgressBar.new
-  while line 
-    bar.tick
-
-    line = Misc.fixutf8(line)
-    line = parser.process line
-    raise SKIP_LINE if line.empty?
-    parts = parser.chop_line line
-    key, values = parser.get_values parts
-    values = parser.cast_values values if parser.cast?
-    
-    puts key
-  end
+line = parser.first_line
+bar = Log::ProgressBar.new
+while line 
+  bar.tick
+
+  line = Misc.fixutf8(line)
+  line = parser.process line
+  raise SKIP_LINE if line.empty?
+  parts = parser.chop_line line
+  key, values = parser.get_values parts
+  values = parser.cast_values values if parser.cast?
+
+  puts key
+  line = parser.stream.gets
 end

data/share/rbbt_commands/tsv/read_excel

@@ -22,7 +22,7 @@ Use - to read from STDIN
 -h--help Print this help
 -s--sheet* Sheet to extract
 -skip--skip_rows* Initial rows to skip
-
+-o--original Dump the rows without parsing them into TSV
 EOF
 if options[:help]
   if defined? rbbt_usage
@@ -39,5 +39,5 @@ raise ParameterException, "No excel file given" if excelfile.nil?
 
 options[:zipped] ||= true if options[:merge]
 require 'rbbt/tsv/excel'
-puts TSV.excel(excelfile, options).to_s
+puts TSV.excel(excelfile, options.merge(:text => options[:original]))
 

data/share/rbbt_commands/workflow/task

@@ -104,7 +104,7 @@ def fix_options(workflow, task, job_options)
               elsif input_options[name] and input_options[name][:stream] and value == "-"
                 STDIN
               else
-                if Array === value
+                if Array === value || IO === value
                   value
                 else
                   array_separator = $array_separator
@@ -137,6 +137,8 @@ def fix_options(workflow, task, job_options)
                   TSV.open(STDIN, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 when (Misc.is_filename?(value) and String)
                   TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
+                when IO
+                  TSV.open(value, :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 else
                   TSV.open(StringIO.new(value), :unnamed => true, :sep => $field_separator, :sep2 => ($array_separator || "|"))
                 end
@@ -189,6 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
 -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
+--orchestrate Run the job through the orchestrator
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
 -O--output* Save job result into file
@@ -436,6 +439,9 @@ begin
     end
 
     job.fork
+  elsif options[:orchestrate]
+    require 'rbbt/workflow/util/orchestrator'
+    Workflow::Orchestrator.process job
   else
     job.run(:stream)
     res = job
@@ -571,10 +577,14 @@ when Step
   elsif detach
     exit! 0
   else
-    res.join
-    Open.open(res.path, :mode => 'rb') do |io|
-      Misc.consume_stream(io, false, out)
-    end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    if %w(float integer string boolean).include?(res.result_type.to_s)
+      out.puts res.load
+    else
+      res.join
+      Open.open(res.path, :mode => 'rb') do |io|
+        Misc.consume_stream(io, false, out)
+      end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    end
   end
 else
   if Array === res

data/test/rbbt/annotations/test_util.rb

@@ -1,6 +1,11 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
 require 'rbbt/annotations'
 
+module TestEntityString
+  extend Entity
+  self.annotation :code
+end
+
 class TestAnnotation < Test::Unit::TestCase
   def test_marshal
     a = "STRING"
@@ -9,5 +14,11 @@ class TestAnnotation < Test::Unit::TestCase
     assert !(Annotated === Marshal.load(Marshal.dump(a)))
     assert_equal a, Marshal.load(Marshal.dump(a))
   end
+
+  def test_hash
+    e = TestEntityString.setup("TEST", :code => 10)
+    assert_equal "TEST", Annotated.to_hash(e)[:literal]
+    assert_equal 10, Annotated.to_hash(e)[:info][:code]
+  end
 end