rbbt-util 5.32.27 → 5.32.28

data/lib/rbbt/hpc/orchestrate.rb

@@ -1,203 +1,10 @@
-require 'rbbt/workflow/util/orchestrator'
+#require 'rbbt/workflow/util/orchestrator'
+require 'rbbt/hpc/orchestrate/rules'
+require 'rbbt/hpc/orchestrate/chains'
+require 'rbbt/hpc/orchestrate/batches'
 module HPC
   module Orchestration
 
-    def job_rules(rules, job)
-      workflow = job.workflow.to_s
-      task_name = job.task_name.to_s
-      task_name = job.overriden.to_s if Symbol === job.overriden
-
-      defaults = rules["defaults"] || {}
-      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
-
-      job_rules = IndiferentHash.setup(defaults.dup)
-
-      rules["chains"].each do |name,info|
-        IndiferentHash.setup(info)
-        chain_tasks = info[:tasks].split(/,\s*/)
-
-        chain_tasks.each do |task|
-          task_workflow, chain_task = task.split("#")
-          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
-          job_rules["chain_tasks"] ||= {}
-          job_rules["chain_tasks"][task_workflow] ||= []
-          job_rules["chain_tasks"][task_workflow]  << chain_task
-          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
-          config_keys = job_rules.delete :config_keys
-          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
-          if config_keys
-            config_keys.gsub!(/,\s+/,',') 
-            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
-          end
-        end
-
-        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
-          break
-        else
-          job_rules.delete "chain_tasks" 
-        end
-      end if rules["chains"]
-
-      config_keys = job_rules.delete :config_keys
-      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
-
-      if config_keys
-        config_keys.gsub!(/,\s+/,',') 
-        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
-      end
-
-      if rules["skip"] && rules["skip"][workflow]
-        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
-      end
-
-      job_rules
-    end
-
-    def get_job_dependencies(job, job_rules = nil)
-      deps = job.dependencies || []
-      deps += job.input_dependencies || []
-      deps
-    end
-
-    def get_recursive_job_dependencies(job)
-      deps = get_job_dependencies(job) 
-      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
-    end
-
-    def piggyback(job, job_rules, job_deps)
-      return false unless job_rules["skip"]
-      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
-      final_deps = final_deps.reject{|dep| dep.done? }
-      return final_deps.first if final_deps.length == 1
-      return false
-    end
-
-    def get_chains(job, rules, chains = {})
-      job_rules = self.job_rules(rules, job)
-      job_deps = get_job_dependencies(job)
-
-      input_deps = []
-      job.rec_dependencies.each do |dep|
-        input_deps.concat dep.input_dependencies
-      end
-
-      job_deps.each do |dep|
-        input_deps.concat dep.input_dependencies
-        get_chains(dep, rules, chains)
-      end
-
-      job_deps.select do |dep|
-        chained = job_rules["chain_tasks"] &&
-          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
-          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
-
-        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
-        chained || dep_skip
-      end.each do |dep|
-        chains[job] ||= [] 
-        chains[job] << dep 
-        chains[job].concat chains[dep] if chains[dep]
-        chains[job].uniq!
-      end
-
-      chains
-    end
-
-    def workload(job, rules, chains, options, seen = nil)
-      return [] if job.done?
-      if seen.nil?
-        seen = {}
-        target_job = true
-      end
-
-      job_rules = self.job_rules(rules, job)
-      job_deps = get_job_dependencies(job)
-
-      chain = chains[job]
-      chain = chain.reject{|j| seen.include? j.path} if chain
-      chain = chain.reject{|dep| dep.done? } if chain
-      piggyback = piggyback(job, job_rules, job_deps)
-      dep_ids = job_deps.collect do |dep|
-        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
-        next_options = IndiferentHash.setup(options.dup)
-        if piggyback and piggyback == dep
-          next_options[:piggyback] ||= []
-          next_options[:piggyback].push job
-          ids = workload(dep, rules, chains, next_options, seen)
-        else
-          next_options.delete :piggyback
-          ids = workload(dep, rules, chains, next_options, seen)
-        end
-
-        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
-
-        seen[dep.path] = ids
-        ids
-      end.compact.flatten.uniq
-
-      return seen[job.path] || dep_ids if seen.include?(job.path)
-
-      if piggyback and seen[piggyback.path]
-        return seen[job.path] = seen[piggyback.path] 
-      end
-
-      job_rules.delete :chain_tasks
-      job_rules.delete :tasks
-      job_rules.delete :workflow
-      
-
-      option_config_keys = options[:config_keys]
-
-      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:batch_dependencies => dep_ids))
-      job_options.delete :orchestration_rules
-
-      config_keys = job_rules.delete(:config_keys)
-      if config_keys
-        config_keys.gsub!(/,\s+/,',') 
-        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
-      end
-
-      if option_config_keys
-        option_config_keys = option_config_keys.gsub(/,\s+/,',') 
-        job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + option_config_keys : option_config_keys
-      end
-
-      if options[:piggyback]
-        manifest = options[:piggyback].uniq
-        manifest += [job]
-        manifest.concat chain if chain
-
-        job = options[:piggyback].first
-
-        job_rules = self.job_rules(rules, job)
-        new_config_keys = self.job_rules(rules, job)[:config_keys]
-        if new_config_keys
-          new_config_keys = new_config_keys.gsub(/,\s+/,',') 
-          job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + new_config_keys : new_config_keys
-        end
-
-        job_options.delete :piggyback
-      else
-        manifest = [job]
-        manifest.concat chain if chain
-      end
-
-      manifest.uniq!
-
-      job_options[:manifest] = manifest.collect{|j| j.task_signature }
-
-      job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
-
-      if options[:dry_run]
-        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
-        job_options[:manifest].first
-      else
-        run_job(job, job_options)
-      end
-    end
-
-
     def orchestrate_job(job, options)
       options.delete "recursive_clean"
       options.delete "clean_task"
@@ -211,9 +18,26 @@ module HPC
       rules ||= {}
       IndiferentHash.setup(rules)
 
-      chains = get_chains(job, rules)
-
-      workload(job, rules, chains, options)
+      batches = HPC::Orchestration.job_batches(rules, job)
+
+      batch_ids = {}
+      while batches.any?
+        top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
+        raise "No batch without unmet dependencies" if top.nil?
+        batches.delete top
+        job_options = options.merge(top[:rules])
+        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+        job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
+
+        if options[:dry_run]
+          puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          batch_ids[top] = top[:top_level].task_signature
+        else
+          id = run_job(top[:top_level], job_options)
+          batch_ids[top] = id
+        end
+      end
     end
 
   end

data/lib/rbbt/persist/tsv.rb

@@ -78,7 +78,7 @@ module Persist
     end
   end
 
-  def self.persist_tsv(source, filename, options = {}, persist_options = {}, &block)
+  def self.persist_tsv(source, filename = nil, options = {}, persist_options = {}, &block)
     persist_options[:prefix] ||= "TSV"
 
     if data = persist_options[:data]

data/lib/rbbt/util/log.rb

@@ -108,8 +108,9 @@ module Log
 
 
   def self._ignore_stderr
-    backup_stderr = STDERR.dup
-    File.open('/dev/null', 'w') do |f|
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
         STDERR.reopen(f)
         begin
           yield
@@ -117,6 +118,9 @@ module Log
           STDERR.reopen backup_stderr
           backup_stderr.close
         end
+      end
+    rescue Errno::ENOENT
+      yield
     end
   end
 

data/lib/rbbt/util/migrate.rb

@@ -49,6 +49,7 @@ puts resource[path].find(search_path)
     test_str = options[:test] ? '-nv' : ''
 
     real_paths.each do |source_path|
+      Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}"
       if File.directory?(source_path) || source_path =~ /\/$/
         source_path += "/" unless source_path[-1] == "/"
         target += "/" unless target[-1] == "/"
@@ -76,8 +77,12 @@ puts resource[path].find(search_path)
           files_from_str = ""
         end
 
-        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+        #cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+         
+        # rsync_args = "-avztAXHP --copy-unsafe-links"
+        rsync_args = "-avztAHP --copy-unsafe-links"
 
+        cmd = "rsync #{rsync_args} #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
 
         cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
 

data/lib/rbbt/util/misc/inspect.rb

@@ -280,8 +280,11 @@ module Misc
       i = parts.index job
       begin
         workflow, task = parts.values_at i - 2, i - 1
-        return Kernel.const_get(workflow).tasks.include? task.to_sym
+        Workflow.require_workflow workflow
+        #return Kernel.const_get(workflow).tasks.include? task.to_sym
+        return true
       rescue
+        Log.exception $!
       end
     end
     false

data/lib/rbbt/util/misc.rb

@@ -70,6 +70,11 @@ module Misc
   end
 
   def self.timespan(str, default = "s")
+    if str.include?(":")
+      seconds, minutes, hours = str.split(":").reverse
+      return seconds.to_i + minutes.to_i * 60 + hours.to_i * 60 * 60
+    end
+
     tokens = {
       "s" => (1),
       "sec" => (1),

data/lib/rbbt/util/python.rb

@@ -52,7 +52,7 @@ module RbbtPython
       if Array === imports
         pyfrom mod, :import => imports
       elsif Hash === imports
-        pyimport mod, imports
+        pyimport mod, **imports
       else
         pyimport mod 
       end

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -1,37 +1,97 @@
 module Workflow
-  def nextflow_file(file, name = nil)
+  def self.nextflow_file_params(file)
+    Open.read(file).scan(/params\.\w+/).collect{|p| p.split(".").last}.uniq
+  end
+
+  def self.nextflow_includes(file)
+    Open.read(file).scan(/^include\s*{\s*(.*?)(?:\s*as.*?)?}\s*from\s+"(.*?)"(?:\s*params.*)?/).collect{|p| p}.uniq
+  end
+
+  def self.nextflow_recursive_params(file)
+    params = nextflow_file_params(file)
+    dir = File.dirname(file)
+    nextflow_includes(file).inject(params) do |params,info|
+      name_str, included_file = info
+      included_file = File.join(dir, included_file)
+      included_file += '.nf' unless File.exists?(included_file) || ! File.exists?(included_file + '.nf')
+      name_str.split(";").each do |name|
+        name = name.strip
+        include_params = nextflow_recursive_params(included_file).collect{|p| [p,name] * "-"}
+        params += include_params
+      end
+      params
+    end
+  end
+
+  def nextflow_file(file, name = nil, output = nil)
+    name, output = nil, name if Hash === name
+
+    if Hash === output
+      result, output = output.collect.first
+    else
+      result = :text
+    end
+
     file = file + '.nf' unless File.exists?(file) || ! File.exists?(file + '.nf')
     file = File.expand_path(file)
-    name ||= File.basename(file).sub(/\.nf$/,'')
-    params = Open.read(file).scan(/params\.\w+/).collect{|p| p.split(".").last}.uniq
+    name ||= File.basename(file).sub(/\.nf$/,'').gsub(/\s/,'_')
+    params = Workflow.nextflow_recursive_params(file)
 
     params.each do |param|
-      input param, :string
+      p,_sep, section = param.partition("-")
+      if section.nil? || section.empty?
+        input param, :string, "Nextflow param #{p}", nil, :nofile => true
+      else
+        input param, :string, "Nextflow param #{p} from import #{section}", nil, :nofile => true
+      end
     end
-    task name => :text do 
+    task name => result do 
       work = file('work')
-      output = file('output')
       profile = config :profile, :nextflow
-      Misc.in_dir output do
+
+      new_inputs = inputs.zip(inputs.fields).collect do |v,f|
+        if String === v && m = v.match(/^JOB_FILE:(.*)/)
+          file(m[1]) 
+        elsif v.nil?
+          Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+        else
+          v
+        end
+      end
+      
+      inputs.replace new_inputs
+
+      Misc.in_dir file('stage') do
         if profile
-          cmd("nextflow run -work-dir #{work} -name #{clean_name} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
+          cmd("nextflow run -work-dir #{work} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
         else
-          cmd("nextflow run -work-dir #{work} -name #{clean_name} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
+          cmd("nextflow run -work-dir #{work} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
         end
       end
+
+      output_file = file(output).glob.first if output
+      output_file = work[File.join('*', '*', output)].glob.first if output && output_file.nil?
+
+      if output_file.nil?
+        work[File.join("*", "*", "*")].glob * "\n"
+      else
+        Open.link output_file, self.tmp_path
+        #Open.rm_rf file('work')
+        nil
+      end
     end
   end
 
-  def nextflow_dir(path)
+  def nextflow_dir(path, output = nil)
     main = File.join(path, 'main.nf')
-    nextflow_file main, File.basename(path)
+    nextflow_file main, File.basename(path), output
   end
 
-  def nextflow(path)
+  def nextflow(path, *args)
     if File.directory?(path)
-      nextflow_dir path
+      nextflow_dir path, *args
     else
-      nextflow_file path
+      nextflow_file path, *args
     end
   end
 end

data/lib/rbbt/workflow/step/run.rb

@@ -419,7 +419,7 @@ class Step
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           config_keys = Rbbt::Config::GOT_KEYS[config_keys_pre.length..-1]
-          set_info :config_keys, config_keys
+          set_info :config_keys, config_keys.uniq
 
           if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
             Open.mv self.tmp_path, self.path

data/lib/rbbt/workflow.rb

@@ -244,7 +244,7 @@ module Workflow
                        when :hash
                          clean_inputs = Annotated.purge(inputs)
                          clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
-                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (d.overriden? ? d.path : d.short_path) : d }
+                         deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << (Symbol === d.overriden ? d.path : d.short_path) : d }
                          key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)
@@ -465,7 +465,14 @@ module Workflow
       extension = nil
       if dependencies.any?
         dep_basename = File.basename(dependencies.last.path)
-        extension = dep_basename.split(".").last if dep_basename.include?('.')
+        if dep_basename.include? "."
+          parts = dep_basename.split(".")
+          extension = [parts.pop]
+          while parts.last.length <= 4
+            extension << parts.pop
+          end
+          extension = extension.reverse * "."
+        end
       end
     end
 

data/share/rbbt_commands/hpc/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/lsf/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/slurm/tail

@@ -13,19 +13,6 @@ Queue a job in Marenostrum
 $ rbbt slurm tail <directory> [options] 
 
 -h--help Print this help
--d--done Done jobs only
--e--error Error jobs only
--a--aborted SLURM aboted jobs
--r--running Running jobs only
--q--queued Queued jobs only
--j--job* Job ids
--s--search* Regular expression
--t--tail* Show the last lines of the STDERR
--p--progress Report progress of job and the dependencies
--SBP--sbatch_parameters show sbatch parameters
--PERF--procpath_performance show Procpath performance summary
--sacct--sacct_peformance show sacct performance summary
--bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 EOF
 
 if options[:help]

data/share/rbbt_commands/tsv/keys

@@ -41,19 +41,18 @@ parser = TSV::Parser.new TSV.get_stream(file), options.merge(:fields => [])
 
 options[:merge] = false if options[:merge] == "false"
 
-Thread.new do
-  line = parser.first_line
-  bar = Log::ProgressBar.new
-  while line 
-    bar.tick
-
-    line = Misc.fixutf8(line)
-    line = parser.process line
-    raise SKIP_LINE if line.empty?
-    parts = parser.chop_line line
-    key, values = parser.get_values parts
-    values = parser.cast_values values if parser.cast?
-    
-    puts key
-  end
+line = parser.first_line
+bar = Log::ProgressBar.new
+while line 
+  bar.tick
+
+  line = Misc.fixutf8(line)
+  line = parser.process line
+  raise SKIP_LINE if line.empty?
+  parts = parser.chop_line line
+  key, values = parser.get_values parts
+  values = parser.cast_values values if parser.cast?
+
+  puts key
+  line = parser.stream.gets
 end

data/share/rbbt_commands/workflow/task

@@ -573,10 +573,14 @@ when Step
   elsif detach
     exit! 0
   else
-    res.join
-    Open.open(res.path, :mode => 'rb') do |io|
-      Misc.consume_stream(io, false, out)
-    end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    if %w(float integer string boolean).include?(res.result_type.to_s)
+      out.puts res.load
+    else
+      res.join
+      Open.open(res.path, :mode => 'rb') do |io|
+        Misc.consume_stream(io, false, out)
+      end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
+    end
   end
 else
   if Array === res

data/test/rbbt/annotations/test_util.rb

@@ -1,6 +1,11 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
 require 'rbbt/annotations'
 
+module TestEntityString
+  extend Entity
+  self.annotation :code
+end
+
 class TestAnnotation < Test::Unit::TestCase
   def test_marshal
     a = "STRING"
@@ -9,5 +14,11 @@ class TestAnnotation < Test::Unit::TestCase
     assert !(Annotated === Marshal.load(Marshal.dump(a)))
     assert_equal a, Marshal.load(Marshal.dump(a))
   end
+
+  def test_hash
+    e = TestEntityString.setup("TEST", :code => 10)
+    assert_equal "TEST", Annotated.to_hash(e)[:literal]
+    assert_equal 10, Annotated.to_hash(e)[:info][:code]
+  end
 end