rbbt-util 5.28.2 → 5.28.8

data/lib/rbbt/workflow/definition.rb

@@ -81,9 +81,10 @@ module Workflow
       if forget
         remove = config :remove_dep_tasks, :remove_dep_tasks, :default => REMOVE_DEP_TASKS
         self.archive_deps
+        self.copy_files_dir
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
-        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist? dep.files_dir
+        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
         Open.ln_h dep.path, self.tmp_path
         case remove.to_s
         when 'true'
@@ -92,8 +93,10 @@ module Workflow
           dep.recursive_clean
         end
       else
-        Open.rm_rf self.files_dir 
-        Open.link dep.files_dir, self.files_dir
+        if Open.exists?(dep.files_dir)
+          Open.rm_rf self.files_dir 
+          Open.link dep.files_dir, self.files_dir
+        end
         Open.link dep.path, self.path
       end
       nil

data/lib/rbbt/workflow/integration/cromwell.rb

@@ -1,13 +1,20 @@
-module Workflow
+module Cromwell
 
   Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
   Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
 
-  def run_cromwell(file, work_dir, options = {})
+  def self.run_cromwell(file, work_dir, options = {})
+    cromwell_inputs_file = Misc.process_options options, :cromwell_inputs_file
     jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
-    CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
+    if cromwell_inputs_file
+      CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}' -i #{cromwell_inputs_file}", options.merge("add_option_dashes" => true))
+    else
+      CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
+    end
   end
+end
 
+module Workflow
   def load_cromwell(file)
     jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
     inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -85,7 +85,11 @@ class RemoteWorkflow
 
       RemoteWorkflow::REST.__prepare_inputs_for_restclient(params)
       name = RemoteWorkflow.capture_exception do
-        RestClient.post(self.encode(url), params)
+        begin
+          RestClient.post(self.encode(url), params)
+        rescue RestClient::MovedPermanently, RestClient::Found, RestClient::TemporaryRedirect
+          raise RbbtException, "REST end-point moved to: #{$!.response.headers[:location]}"
+        end
       end
 
       Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }

data/lib/rbbt/workflow/step.rb

@@ -84,6 +84,7 @@ class Step
   end
 
   def load_dependencies_from_info
+    relocated = nil
     @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
       if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
         Workflow._load_step dep_path
@@ -108,6 +109,14 @@ class Step
     @inputs || []
   end
 
+  def copy_files_dir
+    if File.symlink?(self.files_dir)
+      realpath = Open.realpath(self.files_dir)
+      Open.rm self.files_dir
+      Open.cp realpath, self.files_dir
+    end
+  end
+
   def archive_deps
     self.set_info :archived_info, archived_info
     self.set_info :archived_dependencies, info[:dependencies]
@@ -411,7 +420,7 @@ class Step
       return 
     end
 
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
 
       @result = nil
       @pid = nil
@@ -419,8 +428,8 @@ class Step
       Misc.insist do
         Open.rm info_file if Open.exists?(info_file)
         Open.rm md5_file if Open.exists?(md5_file)
-        Open.rm path if (Open.exists?(path) or Open.broken_link?(path))
-        Open.rm_rf files_dir if Open.exists?(files_dir)
+        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
+        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
         Open.rm pid_file if Open.exists?(pid_file)
         Open.rm tmp_path if Open.exists?(tmp_path)
       end
@@ -454,12 +463,15 @@ class Step
     return [] if dependencies.nil? or dependencies.empty?
 
     new_dependencies = []
+    archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+
     dependencies.each{|step| 
       #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
-      next if seen.include? step
+      next if archived_deps.include? step.path
+      next if seen.include? step.path
       next if self.done? && need_run && ! updatable?
 
-      r = step.rec_dependencies(need_run, new_dependencies)
+      r = step.rec_dependencies(need_run, new_dependencies.collect{|d| d.path})
       new_dependencies.concat r
       new_dependencies << step
     }

data/lib/rbbt/workflow/step/accessor.rb

@@ -93,8 +93,8 @@ class Step
         else
           Open.write(path + '.read', value.to_s)
         end
-      when Step === v
-        v = v.produce.load 
+      when Step === value
+        value = value.produce.load 
       else
         Open.write(path, value.to_s)
       end
@@ -110,12 +110,14 @@ class Step
     task_info = workflow.task_info(task_name)
     input_types = task_info[:input_types]
     task_inputs = task_info[:inputs]
+    input_defaults = task_info[:input_defaults]
 
     inputs = {}
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
       next if options and ! options.include?(name)
       next if value.nil?
+      next if input_defaults[name] == value
       inputs[name] = value
     end
 
@@ -125,7 +127,7 @@ class Step
     end
     save_inputs(inputs, input_types, dir)
 
-    inputs.any?
+    inputs.keys
   end
 
   def name
@@ -437,11 +439,12 @@ class Step
     rec_dependencies = self.rec_dependencies
     return [] if rec_dependencies.empty?
     canfail_paths = self.canfail_paths
+
     dirty_files = rec_dependencies.reject{|dep|
       (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
         ! Open.exists?(dep.info_file) ||
         (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
-        ((dep.error? || dep.aborted? || dep.waiting?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
+        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
     }
   end
 
@@ -508,12 +511,12 @@ class Step
 
   def nopid?
     pid = info[:pid] || Open.exists?(pid_file)
-    ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error)
+    ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
   end
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/usage.rb

@@ -240,7 +240,7 @@ module Workflow
 
           inputs.each do |input, type, file|
             case type
-            when :tsv, :array, :text
+            when :tsv, :array, :text, :file
               lines = file.read.split("\n")
               head = lines[0..5].compact * "\n\n"
               head = head[0..500]

data/lib/rbbt/workflow/util/archive.rb

@@ -262,6 +262,7 @@ puts resource[path].find(search_path)
 
     job_files.each do |file|
       begin
+        Log.debug "Purging #{file}"
         Open.rm_rf file if Open.exists?(file)
       rescue
         Log.warn "Could not erase '#{file}': #{$!.message}"

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -0,0 +1,190 @@
+require 'rbbt/workflow'
+
+module Workflow
+  class Orchestrator
+
+    def self.job_workload(job)
+      workload = {job => []}
+      return workload if job.done?
+
+      job.dependencies.each do |dep|
+        next if dep.done?
+        workload.merge!(job_workload(dep))
+        workload[job] += workload[dep]
+        workload[job] << dep
+      end
+
+      job.input_dependencies.each do |dep|
+        next if dep.done?
+        workload.merge!(job_workload(dep))
+        workload[job] += workload[dep]
+        workload[job] << dep
+      end
+
+      workload
+    end
+
+    def self.job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+
+      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow]
+      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
+
+      job_rules = IndiferentHash.setup(rules[workflow][task_name])
+      rules["defaults"].each{|k,v| job_rules[k] ||= v } if rules["defaults"]
+      job_rules
+    end
+
+    def self.purge_duplicates(candidates)
+      seen = Set.new
+      candidates.select do |job|
+        if seen.include? job.path
+          false
+        else
+          seen << job.path
+          true
+        end
+      end
+    end
+
+    def self.job_resources(rules, job)
+      resources = (job_rules(rules, job) || {})["resources"] || {}
+
+      IndiferentHash.setup(resources)
+
+      default_resources = rules["default_resources"] || rules["defaults"]["resources"]
+      default_resources.each{|k,v| resources[k] ||= v } if default_resources
+
+      resources
+    end
+
+    def self.sort_candidates(candidates, rules)
+      seen = Set.new
+      candidates.sort_by do |job|
+        - job_resources(rules, job).values.inject(0){|acc,e| acc += e}
+      end
+    end
+
+    def self.candidates(workload, rules)
+      if rules.empty?
+        candidates = workload.select{|k,v| v.empty? }.
+          collect{|k,v| k}.
+          reject{|k| k.done? }
+      else
+        candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
+          select{|k,v| v.empty? }.
+          collect{|k,v| k }.
+          reject{|k| k.done? }
+      end
+
+      top_level = workload.keys - workload.values.flatten
+
+      candidates = purge_duplicates candidates
+      candidates = sort_candidates candidates, rules
+
+      candidates
+    end
+
+    attr_accessor :available_resources, :resources_requested, :resources_used, :timer
+
+    def initialize(timer = 5, available_resources = {})
+      @timer               = timer
+      @available_resources = IndiferentHash.setup(available_resources)
+      @resources_requested = IndiferentHash.setup({})
+      @resources_used      = IndiferentHash.setup({})
+    end
+
+    def release_resources(job)
+      if resources_used[job]
+        resources_used[job].each do |resource,value| 
+          next if resource == 'size'
+          resources_requested[resource] -= value.to_i
+        end
+        resources_used.delete job
+      end
+    end
+
+    def check_resources(rules, job)
+      resources = Orchestrator.job_resources(rules, job)
+
+      limit_resources = resources.select{|resource,value| available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource]  }.collect{|resource,v| resource }
+      if limit_resources.any?
+        Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}"
+      else
+
+        resources_used[job] = resources
+        resources.each do |resource,value| 
+          resources_requested[resource] ||= 0
+          resources_requested[resource] += value.to_i
+        end
+        Log.low "Orchestrator producing #{job.path} with resources #{resources}"
+
+        return yield
+      end
+    end
+
+    def run_with_rules(rules, job)
+      job_rules = Orchestrator.job_rules(rules, job)
+
+      Rbbt::Config.with_config do 
+        job_rules[:config_keys].each do |config|
+          Rbbt::Config.process_config config
+        end if job_rules && job_rules[:config_keys]
+
+        log = job_rules[:log] if job_rules 
+        log = Log.severity if log.nil?
+        Log.with_severity log do
+          job.produce(false, true)
+        end
+      end
+    end
+
+    def process(rules, jobs)
+      begin
+
+        workload = jobs.inject({}){|acc,job| acc.merge!(Orchestrator.job_workload(job)) }
+
+        while workload.values.flatten.any?
+
+          candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
+          raise "No candidates" if candidates.empty?
+
+          candidates.each do |job|
+            case 
+            when (job.error? || job.aborted?)
+              if job.recoverable_error?
+                job.clean
+                raise TryAgain
+              else
+                next
+              end
+              release_resources(job)
+            when job.done?
+              Log.debug "Orchestrator done #{job.path}"
+              release_resources(job)
+              raise TryAgain
+
+            when job.running?
+              next
+
+            else
+              check_resources(rules, job) do
+                run_with_rules(rules, job)
+              end
+            end
+          end
+
+          new_workload = {}
+          workload.each do |k,v|
+            next if k.done?
+            new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
+          end
+          sleep timer
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+  end
+end

data/lib/rbbt/workflow/util/trace.rb

@@ -0,0 +1,182 @@
+require 'rbbt/util/R'
+
+module Workflow
+  def self.trace(seed_jobs, options = {})
+
+    jobs = []
+    seed_jobs.each{|j| jobs << j; jobs += j.rec_dependencies}
+
+    data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
+    min_start = nil
+    max_done = nil
+    jobs.each do |job|
+      next unless job.info[:done]
+      started = job.info[:started]
+      ddone = job.info[:done]
+
+      code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
+      code = code + '.' + job.name
+
+      data[code] = [job.workflow.to_s, job.task_name, started, ddone]
+      if min_start.nil?
+        min_start = started
+      else
+        min_start = started if started < min_start
+      end
+
+      if max_done.nil?
+        max_done = ddone
+      else
+        max_done = ddone if ddone > max_done
+      end
+    end
+
+    data.add_field "Start.second" do |k,value|
+      value["Start"] - min_start
+    end
+
+    data.add_field "End.second" do |k,value|
+      value["End"] - min_start
+    end
+
+    if options[:fix_gap]
+      ranges = []
+      data.through do |k,values|
+        start, eend = values.values_at "Start.second", "End.second"
+
+        ranges << (start..eend)
+      end
+
+      gaps = {}
+      last = nil
+      Misc.collapse_ranges(ranges).each do |range|
+        start = range.begin
+        eend = range.end
+        if last
+          gaps[last] = start - last
+        end
+        last = eend
+      end
+
+      data.process "End.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      data.process "Start.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+    end
+
+    tasks_info = {}
+
+    jobs.each do |dep|
+      next unless dep.info[:done]
+      task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
+      info = tasks_info[task] ||= {}
+
+      time = dep.info[:done] - dep.info[:started]
+      info[:time] ||= []
+      info[:time] << time
+
+      cpus = nil
+      spark = false
+      shard = false
+      dep.info[:config_keys].select do |kinfo| 
+        key, value, tokens = kinfo
+        key = key.to_s
+        cpus = value if key.include? 'cpu'
+        spark = value if key == 'spark'
+        shard = value if key == 'shard'
+      end
+
+      info[:cpus] = cpus || 1
+      info[:spark] = spark
+      info[:shard] = shard
+    end
+
+    stats = TSV.setup({}, "Task~Calls,Avg. Time,Total Time,Cpus,Spark,Shard#:type=:list")
+
+    tasks_info.each do |task, info|
+      time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
+      avg_time = Misc.mean(time_lists)
+      total_time = Misc.sum(time_lists)
+      calls = time_lists.length
+      stats[task] = [calls, avg_time, total_time, cpus, spark, shard]
+    end
+
+    raise "No jobs to process" if data.size == 0
+
+    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
+    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
+    total = eend - start
+    Log.info "Total time elapsed: #{total} seconds"
+
+    if options[:fix_gap]
+      total_gaps = Misc.sum(gaps.collect{|k,v| v})
+      Log.info "Total gaps: #{total_gaps} seconds"
+    end
+
+    plot, width, height = options.values_at :plot, :width, :height
+    if plot
+      data.R <<-EOF, [:svg]
+    rbbt.require('tidyverse')
+    rbbt.require('ggplot2')
+
+    names(data) <- make.names(names(data))
+    data$id = rownames(data)
+    data$content = data$Task
+    data$start = data$Start
+    data$end = data$End
+    data$Project = data$Workflow
+
+    tasks = data
+
+    #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
+    theme_gantt <- function(base_size=11, base_family="Sans Serif") {
+      ret <- theme_bw(base_size, base_family) %+replace%
+        theme(panel.background = element_rect(fill="#ffffff", colour=NA),
+              axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
+              title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
+              panel.border = element_blank(), axis.line=element_blank(),
+              panel.grid.minor=element_blank(),
+              panel.grid.major.y = element_blank(),
+              panel.grid.major.x = element_line(size=0.5, colour="grey80"),
+              axis.ticks=element_blank(),
+              legend.position="bottom", 
+              axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
+              strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
+              strip.background=element_rect(fill="#ffffff", colour=NA),
+              panel.spacing.y=unit(1.5, "lines"),
+              legend.key = element_blank())
+
+      ret
+    }
+
+    tasks.long <- tasks %>%
+    gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
+    arrange(date.type, task.date) %>%
+    mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
+
+    x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
+
+    timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
+      geom_segment() + 
+      geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
+      guides(colour=guide_legend(title=NULL)) +
+      labs(x=NULL, y=NULL) + 
+      theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
+
+    rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
+      EOF
+    end
+
+    if options[:plot_data]
+      data
+    else
+      stats
+    end
+
+  end
+end