rbbt-util 5.32.26 → 5.33.0

data/lib/rbbt/hpc/orchestrate.rb

@@ -1,203 +1,10 @@
-require 'rbbt/workflow/util/orchestrator'
+#require 'rbbt/workflow/util/orchestrator'
+require 'rbbt/hpc/orchestrate/rules'
+require 'rbbt/hpc/orchestrate/chains'
+require 'rbbt/hpc/orchestrate/batches'
 module HPC
   module Orchestration
 
-    def job_rules(rules, job)
-      workflow = job.workflow.to_s
-      task_name = job.task_name.to_s
-      task_name = job.overriden.to_s if Symbol === job.overriden
-
-      defaults = rules["defaults"] || {}
-      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
-
-      job_rules = IndiferentHash.setup(defaults.dup)
-
-      rules["chains"].each do |name,info|
-        IndiferentHash.setup(info)
-        chain_tasks = info[:tasks].split(/,\s*/)
-
-        chain_tasks.each do |task|
-          task_workflow, chain_task = task.split("#")
-          chain_task, task_workflow = task_workflow, info[:workflow] if chain_task.nil? or chain_tasks.empty?
-          job_rules["chain_tasks"] ||= {}
-          job_rules["chain_tasks"][task_workflow] ||= []
-          job_rules["chain_tasks"][task_workflow]  << chain_task
-          next unless task_name == chain_task.to_s && workflow == task_workflow.to_s
-          config_keys = job_rules.delete :config_keys
-          job_rules = IndiferentHash.setup(job_rules.merge(info)) 
-          if config_keys
-            config_keys.gsub!(/,\s+/,',') 
-            job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
-          end
-        end
-
-        if job_rules["chain_tasks"][workflow] && job_rules["chain_tasks"][workflow].include?(task_name)
-          break
-        else
-          job_rules.delete "chain_tasks" 
-        end
-      end if rules["chains"]
-
-      config_keys = job_rules.delete :config_keys
-      job_rules = IndiferentHash.setup(job_rules.merge(rules[workflow][task_name])) if rules[workflow] && rules[workflow][task_name]
-
-      if config_keys
-        config_keys.gsub!(/,\s+/,',') 
-        job_rules[:config_keys] = job_rules[:config_keys] ? config_keys + "," + job_rules[:config_keys] : config_keys
-      end
-
-      if rules["skip"] && rules["skip"][workflow]
-        job_rules["skip"] = true if rules["skip"][workflow].split(/,\s*/).include? task_name
-      end
-
-      job_rules
-    end
-
-    def get_job_dependencies(job, job_rules = nil)
-      deps = job.dependencies || []
-      deps += job.input_dependencies || []
-      deps
-    end
-
-    def get_recursive_job_dependencies(job)
-      deps = get_job_dependencies(job) 
-      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
-    end
-
-    def piggyback(job, job_rules, job_deps)
-      return false unless job_rules["skip"]
-      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
-      final_deps = final_deps.reject{|dep| dep.done? }
-      return final_deps.first if final_deps.length == 1
-      return false
-    end
-
-    def get_chains(job, rules, chains = {})
-      job_rules = self.job_rules(rules, job)
-      job_deps = get_job_dependencies(job)
-
-      input_deps = []
-      job.rec_dependencies.each do |dep|
-        input_deps.concat dep.input_dependencies
-      end
-
-      job_deps.each do |dep|
-        input_deps.concat dep.input_dependencies
-        get_chains(dep, rules, chains)
-      end
-
-      job_deps.select do |dep|
-        chained = job_rules["chain_tasks"] &&
-          job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
-          job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
-
-        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
-        chained || dep_skip
-      end.each do |dep|
-        chains[job] ||= [] 
-        chains[job] << dep 
-        chains[job].concat chains[dep] if chains[dep]
-        chains[job].uniq!
-      end
-
-      chains
-    end
-
-    def workload(job, rules, chains, options, seen = nil)
-      return [] if job.done?
-      if seen.nil?
-        seen = {}
-        target_job = true
-      end
-
-      job_rules = self.job_rules(rules, job)
-      job_deps = get_job_dependencies(job)
-
-      chain = chains[job]
-      chain = chain.reject{|j| seen.include? j.path} if chain
-      chain = chain.reject{|dep| dep.done? } if chain
-      piggyback = piggyback(job, job_rules, job_deps)
-      dep_ids = job_deps.collect do |dep|
-        seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
-        next_options = IndiferentHash.setup(options.dup)
-        if piggyback and piggyback == dep
-          next_options[:piggyback] ||= []
-          next_options[:piggyback].push job
-          ids = workload(dep, rules, chains, next_options, seen)
-        else
-          next_options.delete :piggyback
-          ids = workload(dep, rules, chains, next_options, seen)
-        end
-
-        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
-
-        seen[dep.path] = ids
-        ids
-      end.compact.flatten.uniq
-
-      return seen[job.path] || dep_ids if seen.include?(job.path)
-
-      if piggyback and seen[piggyback.path]
-        return seen[job.path] = seen[piggyback.path] 
-      end
-
-      job_rules.delete :chain_tasks
-      job_rules.delete :tasks
-      job_rules.delete :workflow
-      
-
-      option_config_keys = options[:config_keys]
-
-      job_options = IndiferentHash.setup(options.merge(job_rules).merge(:batch_dependencies => dep_ids))
-      job_options.delete :orchestration_rules
-
-      config_keys = job_rules.delete(:config_keys)
-      if config_keys
-        config_keys.gsub!(/,\s+/,',') 
-        job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
-      end
-
-      if option_config_keys
-        option_config_keys = option_config_keys.gsub(/,\s+/,',') 
-        job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + option_config_keys : option_config_keys
-      end
-
-      if options[:piggyback]
-        manifest = options[:piggyback].uniq
-        manifest += [job]
-        manifest.concat chain if chain
-
-        job = options[:piggyback].first
-
-        job_rules = self.job_rules(rules, job)
-        new_config_keys = self.job_rules(rules, job)[:config_keys]
-        if new_config_keys
-          new_config_keys = new_config_keys.gsub(/,\s+/,',') 
-          job_options[:config_keys] = job_options[:config_keys] ? job_options[:config_keys] + "," + new_config_keys : new_config_keys
-        end
-
-        job_options.delete :piggyback
-      else
-        manifest = [job]
-        manifest.concat chain if chain
-      end
-
-      manifest.uniq!
-
-      job_options[:manifest] = manifest.collect{|j| j.task_signature }
-
-      job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
-
-      if options[:dry_run]
-        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
-        job_options[:manifest].first
-      else
-        run_job(job, job_options)
-      end
-    end
-
-
     def orchestrate_job(job, options)
       options.delete "recursive_clean"
       options.delete "clean_task"
@@ -211,9 +18,26 @@ module HPC
       rules ||= {}
       IndiferentHash.setup(rules)
 
-      chains = get_chains(job, rules)
-
-      workload(job, rules, chains, options)
+      batches = HPC::Orchestration.job_batches(rules, job)
+
+      batch_ids = {}
+      while batches.any?
+        top = batches.select{|b| b[:deps].nil? || (b[:deps] - batch_ids.keys).empty? }.first
+        raise "No batch without unmet dependencies" if top.nil?
+        batches.delete top
+        job_options = options.merge(top[:rules])
+        job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
+        job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
+
+        if options[:dry_run]
+          puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          batch_ids[top] = top[:top_level].task_signature
+        else
+          id = run_job(top[:top_level], job_options)
+          batch_ids[top] = id
+        end
+      end
     end
 
   end

data/lib/rbbt/hpc/slurm.rb

@@ -51,6 +51,7 @@ export BATCH_SYSTEM=SLURM
                        "nodes" => nodes,
                        "time" => time,
                        "exclusive" => exclusive,
+                       "highmem" => highmem,
                        "licenses" => licenses,
                        "gres" => gres,
                        "mem" => mem,

data/lib/rbbt/persist/tsv/adapter.rb

@@ -111,6 +111,31 @@ module Persist
       end
     end
 
+    def with_read(&block)
+      if read? || write?
+        return yield
+      else
+        read_and_close &block
+      end
+    end
+
+    def with_write(&block)
+      if write?
+        return yield
+      else
+        if self.read?
+          self.write_and_read do
+            return yield
+          end
+        else
+          self.write_and_close do
+            return yield
+          end
+        end
+      end
+    end
+
+
     def read_and_close
       if read? || write?
         begin

data/lib/rbbt/persist/tsv.rb

@@ -78,7 +78,7 @@ module Persist
     end
   end
 
-  def self.persist_tsv(source, filename, options = {}, persist_options = {}, &block)
+  def self.persist_tsv(source, filename = nil, options = {}, persist_options = {}, &block)
     persist_options[:prefix] ||= "TSV"
 
     if data = persist_options[:data]

data/lib/rbbt/persist.rb

@@ -423,6 +423,8 @@ module Persist
           end
         end
 
+        repo.read
+
         case
         when (keys.length == 1 and keys.first == subkey + 'NIL')
           nil
@@ -430,19 +432,19 @@ module Persist
           []
         when (keys.length == 1 and keys.first =~ /:SINGLE$/)
           key = keys.first
-          values = repo.read_and_close do
+          values = repo.with_read do
             repo[key]
           end
           Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
         when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
             keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
               Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
             }
           end
         when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
-          repo.read_and_close do
+          repo.with_read do
 
             res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
               v = repo[key]
@@ -457,7 +459,7 @@ module Persist
         else
           entities = yield
 
-          repo.write_and_close do 
+          repo.write_and_read do 
             case
             when entities.nil?
               repo[subkey + "NIL"] = nil

data/lib/rbbt/resource/path.rb

@@ -462,4 +462,8 @@ module Path
       Path.setup relative_file , @pkgdir, @resource
     end
   end
+
+  def clean_annotations
+    "" << self.to_s
+  end
 end

data/lib/rbbt/tsv/util.rb

@@ -297,6 +297,8 @@ module TSV
     when :double
       if field.nil?
         through do |k,v| new[k] = v.first end
+      elsif field == :all
+        through do |k,v| new[k] = v.flatten.compact end
       else
         pos = identify_field field
         through do |k,v| new[k] = v[pos] end
@@ -313,7 +315,16 @@ module TSV
       end
     end
     self.annotate(new)
-    new.fields = new.fields[0..0] if new.fields
+    if new.fields
+      case field
+      when nil
+        new.fields = new.fields[0..0]
+      when :all
+        new.fields = [new.fields * "+"]
+      else
+        new.fields = [field]
+      end
+    end
     new.type = :flat
     new
   end

data/lib/rbbt/util/concurrency/processes.rb

@@ -205,7 +205,8 @@ class RbbtProcessQueue
     end
 
     @queue.close_read
-    Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
+    Log.info "Cpu process #{@master_pid} with #{num_processes} workers."
+    Log.low "Signal #{@master_pid} USR1/USR2 (#10/#12) to increase/decrease workers."
   end
 
   def init(&block)

data/lib/rbbt/util/log.rb

@@ -108,8 +108,9 @@ module Log
 
 
   def self._ignore_stderr
-    backup_stderr = STDERR.dup
-    File.open('/dev/null', 'w') do |f|
+    begin
+      File.open('/dev/null', 'w') do |f|
+        backup_stderr = STDERR.dup
         STDERR.reopen(f)
         begin
           yield
@@ -117,6 +118,9 @@ module Log
           STDERR.reopen backup_stderr
           backup_stderr.close
         end
+      end
+    rescue Errno::ENOENT
+      yield
     end
   end
 

data/lib/rbbt/util/migrate.rb

@@ -49,6 +49,7 @@ puts resource[path].find(search_path)
     test_str = options[:test] ? '-nv' : ''
 
     real_paths.each do |source_path|
+      Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}" if options[:files]
       if File.directory?(source_path) || source_path =~ /\/$/
         source_path += "/" unless source_path[-1] == "/"
         target += "/" unless target[-1] == "/"
@@ -76,8 +77,12 @@ puts resource[path].find(search_path)
           files_from_str = ""
         end
 
-        cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+        #cmd = "rsync -avztAXHP --copy-unsafe-links #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
+         
+        # rsync_args = "-avztAXHP --copy-unsafe-links"
+        rsync_args = "-avztAHP --copy-unsafe-links"
 
+        cmd = "rsync #{rsync_args} #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
 
         cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
 

data/lib/rbbt/util/misc/inspect.rb

@@ -280,8 +280,11 @@ module Misc
       i = parts.index job
       begin
         workflow, task = parts.values_at i - 2, i - 1
-        return Kernel.const_get(workflow).tasks.include? task.to_sym
+        Workflow.require_workflow workflow
+        #return Kernel.const_get(workflow).tasks.include? task.to_sym
+        return true
       rescue
+        Log.exception $!
       end
     end
     false

data/lib/rbbt/util/misc.rb

@@ -70,6 +70,11 @@ module Misc
   end
 
   def self.timespan(str, default = "s")
+    if str.include?(":")
+      seconds, minutes, hours = str.split(":").reverse
+      return seconds.to_i + minutes.to_i * 60 + hours.to_i * 60 * 60
+    end
+
     tokens = {
       "s" => (1),
       "sec" => (1),

data/lib/rbbt/util/python.rb

@@ -52,7 +52,7 @@ module RbbtPython
       if Array === imports
         pyfrom mod, :import => imports
       elsif Hash === imports
-        pyimport mod, imports
+        pyimport mod, **imports
       else
         pyimport mod 
       end

data/lib/rbbt/workflow/definition.rb

@@ -76,7 +76,7 @@ module Workflow
     extension :dep_task unless @extension
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
-      raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
+      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join

data/lib/rbbt/workflow/examples.rb

@@ -25,71 +25,6 @@ module Workflow
     end.compact
   end
 
-  def self.load_inputs(dir, input_names, input_types)
-    inputs = {}
-    if File.exists?(dir) && ! File.directory?(dir)
-      Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
-      tarfile = dir
-      digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
-      tmpdir = Rbbt.tmp.input_bundle[digest].find
-      Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
-      files = tmpdir.glob("*")
-      if files.length == 1 && File.directory?(files.first)
-        tmpdir = files.first
-      end
-      load_inputs(tmpdir, input_names, input_types)
-    else
-      dir = Path.setup(dir.dup)
-      input_names.each do |input|
-        file = dir[input].find
-        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not file.exists?
-        Log.debug "Trying #{ input }: #{file}"
-        next unless file and file.exists?
-
-
-        case input_types[input]
-        when :file, :binary
-          Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.yaml/
-            inputs[input.to_sym]  = YAML.load(Open.read(file))
-          else
-            if File.symlink?(file)
-              link_target = File.expand_path(File.readlink(file), File.dirname(file))
-              inputs[input.to_sym]  = link_target
-            else
-              inputs[input.to_sym]  = Open.realpath(file)
-            end
-          end
-        when :text
-          Log.debug "Reading #{ input } from #{file}"
-          inputs[input.to_sym]  = Open.read(file)
-        when :array
-          Log.debug "Reading array #{ input } from #{file}"
-          inputs[input.to_sym]  = Open.read(file).split("\n")
-        when :tsv
-          Log.debug "Opening tsv #{ input } from #{file}"
-          inputs[input.to_sym]  = TSV.open(file)
-        when :boolean
-          inputs[input.to_sym]  = (file.read.strip == 'true')
-        else
-          Log.debug "Loading #{ input } from #{file}"
-          inputs[input.to_sym]  = file.read.strip
-        end
-
-      end
-      inputs = IndiferentHash.setup(inputs)
-
-      dir.glob("*#*").each do |od|
-        name = File.basename(od)
-        value = Open.read(od)
-        Log.debug "Loading override dependency #{ name } as #{value}"
-        inputs[name] = value.chomp
-      end
-
-      inputs
-    end
-  end
-
   def example_inputs(task_name, example)
     inputs = {}
     IndiferentHash.setup(inputs)

data/lib/rbbt/workflow/integration/nextflow.rb

@@ -1,37 +1,97 @@
 module Workflow
-  def nextflow_file(file, name = nil)
+  def self.nextflow_file_params(file)
+    Open.read(file).scan(/params\.\w+/).collect{|p| p.split(".").last}.uniq
+  end
+
+  def self.nextflow_includes(file)
+    Open.read(file).scan(/^include\s*{\s*(.*?)(?:\s*as.*?)?}\s*from\s+"(.*?)"(?:\s*params.*)?/).collect{|p| p}.uniq
+  end
+
+  def self.nextflow_recursive_params(file)
+    params = nextflow_file_params(file)
+    dir = File.dirname(file)
+    nextflow_includes(file).inject(params) do |params,info|
+      name_str, included_file = info
+      included_file = File.join(dir, included_file)
+      included_file += '.nf' unless File.exists?(included_file) || ! File.exists?(included_file + '.nf')
+      name_str.split(";").each do |name|
+        name = name.strip
+        include_params = nextflow_recursive_params(included_file).collect{|p| [p,name] * "-"}
+        params += include_params
+      end
+      params
+    end
+  end
+
+  def nextflow_file(file, name = nil, output = nil)
+    name, output = nil, name if Hash === name
+
+    if Hash === output
+      result, output = output.collect.first
+    else
+      result = :text
+    end
+
     file = file + '.nf' unless File.exists?(file) || ! File.exists?(file + '.nf')
     file = File.expand_path(file)
-    name ||= File.basename(file).sub(/\.nf$/,'')
-    params = Open.read(file).scan(/params\.\w+/).collect{|p| p.split(".").last}.uniq
+    name ||= File.basename(file).sub(/\.nf$/,'').gsub(/\s/,'_')
+    params = Workflow.nextflow_recursive_params(file)
 
     params.each do |param|
-      input param, :string
+      p,_sep, section = param.partition("-")
+      if section.nil? || section.empty?
+        input param, :string, "Nextflow param #{p}", nil, :nofile => true
+      else
+        input param, :string, "Nextflow param #{p} from import #{section}", nil, :nofile => true
+      end
     end
-    task name => :text do 
+    task name => result do 
       work = file('work')
-      output = file('output')
       profile = config :profile, :nextflow
-      Misc.in_dir output do
+
+      new_inputs = inputs.zip(inputs.fields).collect do |v,f|
+        if String === v && m = v.match(/^JOB_FILE:(.*)/)
+          file(m[1]) 
+        elsif v.nil?
+          Rbbt::Config.get(['nextflow', f] * "_", 'default', f)
+        else
+          v
+        end
+      end
+      
+      inputs.replace new_inputs
+
+      Misc.in_dir file('stage') do
         if profile
-          cmd("nextflow run -work-dir #{work} -name #{clean_name} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
+          cmd("nextflow run -work-dir #{work} -ansi-log false  -profile #{profile} #{file}", inputs.to_hash.merge('add_option_dashes' => true))
         else
-          cmd("nextflow run -work-dir #{work} -name #{clean_name} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
+          cmd("nextflow run -work-dir #{work} -ansi-log false #{file}", inputs.to_hash.merge('add_option_dashes' => true))
         end
       end
+
+      output_file = file(output).glob.first if output
+      output_file = work[File.join('*', '*', output)].glob.first if output && output_file.nil?
+
+      if output_file.nil?
+        work[File.join("*", "*", "*")].glob * "\n"
+      else
+        Open.link output_file, self.tmp_path
+        #Open.rm_rf file('work')
+        nil
+      end
     end
   end
 
-  def nextflow_dir(path)
+  def nextflow_dir(path, output = nil)
     main = File.join(path, 'main.nf')
-    nextflow_file main, File.basename(path)
+    nextflow_file main, File.basename(path), output
   end
 
-  def nextflow(path)
+  def nextflow(path, *args)
     if File.directory?(path)
-      nextflow_dir path
+      nextflow_dir path, *args
     else
-      nextflow_file path
+      nextflow_file path, *args
     end
   end
 end