rbbt-util 5.30.6 → 5.30.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a9ed1ed74562cdb81e14bb2b2213398646f76e532bdee48e67f9ff2f5330aad
-  data.tar.gz: e54ecf83a0b4323ffe98f8968f4a65acc2aae2ee5437330f44f14dc17c450ec5
+  metadata.gz: 3275cafa7bc74ca88f5d487287ff736113dad5ee6e607bb13f283bb4aa5e0562
+  data.tar.gz: 1a7642c60ce3f3828790599c2522ad1e083bd877c53de10401b93735e952934d
 SHA512:
-  metadata.gz: 4d2eb887a184e24d78e8c565cefc25f09246d14e0f22d9a9037447806489cb4148fb2d9d3aad4cfe9bca1f256c0ec43afcc343047f20f974e6facea3214f40c0
-  data.tar.gz: 7f593e22911a501e353b6eb49830291a29b038474fd7bfc182e402f255219fc1eaf07311ad334118b43c275abc9ab2a5198d68ccbcdce7f3b4636c38dc698afd
+  metadata.gz: bc9717f6611646c099305587bdcf224243102a500ba68047e382df49e583a3c764489e09b7c9aa4993eab3953d72dc9ce206d5450213e1ae832b1622c96fd309
+  data.tar.gz: 791cdba47b74e1f1fd47c439bb3f2b025e540002ec2b1e18d846bab08c6fd858cc25a94f9c84056383c5125cca22c19f00d43ae0c53a8c77ef4de423c19f0d8c

data/lib/rbbt/hpc/orchestrate.rb

@@ -8,7 +8,7 @@ module HPC
       task_name = job.overriden.to_s if Symbol === job.overriden
 
       defaults = rules["defaults"] || {}
-      defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
+      defaults = defaults.merge(rules[workflow]["defaults"] || {}) if rules[workflow]
 
       job_rules = IndiferentHash.setup(defaults.dup)
 
@@ -53,50 +53,88 @@ module HPC
       job_rules
     end
 
-    def self.get_job_dependencies(job, job_rules)
+    def self.get_job_dependencies(job, job_rules = nil)
       deps = job.dependencies || []
       deps += job.input_dependencies || []
       deps
     end
 
-    def self.orchestrate_job(job, options, skip = false, seen = {}, chains = {})
-      return if job.done?
-      return unless job.path.split("/")[-4] == "jobs"
-      seen[:orchestration_target_job] ||= job
-
-      options.delete "recursive_clean"
-      options.delete "clean_task"
-      options.delete "clean"
-      options.delete "tail"
-      options.delete "printfile"
-      options.delete "detach"
+    def self.get_recursive_job_dependencies(job)
+      deps = get_job_dependencies(job) 
+      (deps + deps.collect{|dep| get_recursive_job_dependencies(dep) }).flatten
+    end
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
-      rules ||= {}
-      IndiferentHash.setup(rules)
+    def self.piggyback(job, job_rules, job_deps)
+      return false unless job_rules["skip"]
+      final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
+      return final_deps.first if final_deps.length == 1
+      return false
+    end
 
+    def self.get_chains(job, rules, chains = {})
       job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
 
-      deps = get_job_dependencies(job, job_rules)
+      input_deps = []
+      job.rec_dependencies.each do |dep|
+        input_deps.concat dep.input_dependencies
+      end
 
-      chains[job.path] ||= []
-      dep_ids = deps.collect do |dep|
-        skip_dep = job_rules["chain_tasks"] &&
+      job_deps.each do |dep|
+        input_deps.concat dep.input_dependencies
+        get_chains(dep, rules, chains)
+      end
+
+      job_deps.select do |dep|
+        chained = job_rules["chain_tasks"] &&
           job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
           job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
 
-          chains[job.path] << dep if skip_dep
+        dep_skip = dep.done? && ! input_deps.include?(dep) && self.job_rules(rules, dep)["skip"] 
+        chained || dep_skip
+      end.each do |dep|
+        chains[job] ||= [] 
+        chains[job] << dep 
+        chains[job].concat chains[dep] if chains[dep]
+      end
+
+      chains
+    end
 
-        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen, chains)
-        if job.canfail_paths.include? dep.path
-          [deps].flatten.compact.collect{|id| ['canfail', id] * ":"}
+    def self.workload(job, rules, chains, options, seen = nil)
+      return [] if job.done?
+      if seen.nil?
+        seen = {} 
+        target_job = true
+      end
+
+      job_rules = self.job_rules(rules, job)
+      job_deps = get_job_dependencies(job)
+
+
+      chain = chains[job]
+      chain -= seen.keys if chain
+      piggyback = piggyback(job, job_rules, job_deps)
+      dep_ids = job_deps.collect do |dep|
+        seen[dep] = nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep) 
+        next_options = IndiferentHash.setup(options.dup)
+        if piggyback and piggyback == dep
+          next_options[:piggyback] ||= []
+          next_options[:piggyback].push job
+          ids = workload(dep, rules, chains, next_options, seen)
         else
-          deps
+          next_options.delete :piggyback
+          ids = workload(dep, rules, chains, next_options, seen)
         end
-      end.flatten.compact.uniq
 
-      skip = true if job_rules[:skip]
-      return dep_ids if skip and seen[:orchestration_target_job] != job
+        ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
+
+        seen[dep] = ids
+        ids
+      end.compact.flatten.uniq
+
+      return seen[job] || dep_ids if seen.include?(job) 
+      return seen[piggyback] if piggyback
 
       job_rules.delete :chain_tasks
       job_rules.delete :tasks
@@ -111,21 +149,48 @@ module HPC
         job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
       end
 
-      manifest = []
-      stack = [job]
-      while dep = stack.pop
-        manifest << dep
-        stack += chains[dep.path] if chains[dep.path]
+
+      if options[:piggyback]
+        manifest = options[:piggyback].uniq
+        manifest += [job]
+        manifest.concat chain if chain
+        job = options[:piggyback].first
+        job_options.delete :piggyback
+      else
+        manifest = [job]
+        manifest.concat chain if chain
       end
 
-      job_options[:manifest] = manifest.uniq.collect{|dep| dep.workflow_short_path}
+      manifest.uniq!
+
+      job_options[:manifest] = manifest.collect{|j| j.workflow_short_path }
 
       if options[:dry_run]
-        puts Log.color(:yellow, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-        []
+        puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+        puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:slurm_dependencies]*", ")
+        job_options[:manifest].first
       else
         run_job(job, job_options)
       end
     end
+
+
+    def self.orchestrate_job(job, options)
+      options.delete "recursive_clean"
+      options.delete "clean_task"
+      options.delete "clean"
+      options.delete "tail"
+      options.delete "printfile"
+      options.delete "detach"
+
+      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
+      rules ||= {}
+      IndiferentHash.setup(rules)
+
+      chains = get_chains(job, rules)
+
+      workload(job, rules, chains, options)
+    end
+
   end
 end

data/lib/rbbt/tsv/manipulate.rb

@@ -717,7 +717,7 @@ module TSV
   def transpose(key_field = "Unkown ID")
     case type
     when :single, :flat
-      transpose_list self.to_list, key_field
+      self.to_list.transpose_list key_field
     when :list
       transpose_list key_field
     when :double

data/lib/rbbt/workflow.rb

@@ -379,7 +379,7 @@ module Workflow
     has_overriden_inputs = false
 
     inputs.each do |k,v|
-      has_overriden_inputs = true if String === k and k.include? "#"
+      #has_overriden_inputs = true if String === k and k.include? "#"
       next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
       default = all_defaults[k]
       next if default == v 

data/lib/rbbt/workflow/step.rb

@@ -114,9 +114,13 @@ class Step
 
   def copy_files_dir
     if File.symlink?(self.files_dir)
-      realpath = Open.realpath(self.files_dir)
-      Open.rm self.files_dir
-      Open.cp realpath, self.files_dir
+      begin
+        realpath = Open.realpath(self.files_dir)
+        Open.rm self.files_dir
+        Open.cp realpath, self.files_dir
+      rescue
+        Log.warn "Copy files_dir for #{self.workflow_short_path}: " + $!.message
+      end
     end
   end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -168,6 +168,10 @@ class Step
     @task_name ||= task.name
   end
 
+  def task_signature
+    [workflow.to_s, task_name] * "#"
+  end
+
   # {{{ INFO
 
   def info_file

data/share/Rlib/util.R

@@ -484,7 +484,7 @@ rbbt.model.inpute <- function(data, formula, ...){
     data
 }
 
-rbbt.tsv.melt <- function(tsv, variable = NULL, value = NULL, key.field = NULL){
+rbbt.tsv.melt <- function(tsv, variable = NULL, value = NULL, key.field = NULL, ...){
     rbbt.require('reshape2')
     if (is.null(key.field)){ key.field = attributes(data)$key.field;}
     if (is.null(key.field)){ key.field = "ID" }
@@ -494,7 +494,7 @@ rbbt.tsv.melt <- function(tsv, variable = NULL, value = NULL, key.field = NULL){
 
     tsv[key.field] = rownames(tsv)
 
-    m <- melt(tsv)
+    m <- melt(tsv, id.vars=c(key.field), ...)
 
     names(m) <- c(key.field, variable, value)
 
@@ -784,6 +784,12 @@ rbbt.plot.venn <- function(data, a=NULL, category=NULL, fill=NULL, ...) {
     return(out)
 }
 
+rbbt.plot.upset <- function(data, variable = NULL, ...){
+  rbbt.require('UpSetR')
+  data[data == TRUE] = 1
+  return(upset(data, ...))
+}
+
 rbbt.plot.pca <- function(data, center = TRUE, scale. = TRUE, ...) {
   rbbt.require('vqv/ggbiplot')
   data <- rbbt.impute(data)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.6
+  version: 5.30.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-09 00:00:00.000000000 Z
+date: 2021-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake