rbbt-util 5.28.6 → 5.28.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 900106fb1799c857a482187c7cb7d18a40c35dbeb5dd231bf8e1851235e28c63
-  data.tar.gz: e5ec2271b3ada0b282963f4ffc7e2b623d8bb84fcb72867a6848b93a80052463
+  metadata.gz: f13812c850239f7d78c94e74288ea7807663c72146002e13a13aa99babb18555
+  data.tar.gz: 3568f646d3095448ac6c8d99f32a5513421c80fd9d8b1f9a049a7ce512f3139d
 SHA512:
-  metadata.gz: 1a418d5b4fe4369f7c25dd063c60a65c2b6c0bd7e3dbf9fd156fe875f85ef1b2eeb4d2eb1a03472e191de98bbfbae20abdb86877d381193f86fe4c8325d2f391
-  data.tar.gz: 5225d6dbdbab08d0ed5239eba91bbe3d2a8dd16c92c24fe177b66ed3b9d4baef4e24818057de8bf694edf715aabd72b481926a2466c76492c27b75be25634b27
+  metadata.gz: 644bcc769e818310492993b15a024b83de399d65a0cfc74b9dc06ee233bfd66da3c0951a26899c66ce62a8410104f00b8b84c07f77a4a1159a7fb5149befb5a1
+  data.tar.gz: da5fd395014feded588f223686bd032dd86e46cac0e48e4369e9d1ea2e1f0d021e0fab02539ab2be1f483efc5463b9487952783edf2e00f22d7158879b5fd32d

data/lib/rbbt/persist.rb

@@ -25,6 +25,7 @@ module Persist
   MEMORY = {} unless defined? MEMORY
   MAX_FILE_LENGTH = 150
 
+  # Return non-false if the first file is newer than the second file
   def self.newer?(path, file)
     return true if not Open.exists?(file)
     path = path.find if Path === path

data/lib/rbbt/tsv/accessor.rb

@@ -550,9 +550,17 @@ module TSV
         "\t" << ([""] * fields.length) * "\t" << "\n"
       end
     when Array
-      "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      end
     else
-      "\t" << values.to_s << "\n"
+      if fields.nil? or fields.empty?
+        "\n"
+      else
+        "\t" << values.to_s << "\n"
+      end
     end
   end
 

data/lib/rbbt/tsv/dumper.rb

@@ -32,9 +32,17 @@ module TSV
           sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        end
       else
-        sep + values.to_s << "\n"
+        if fields.nil? or fields.empty?
+          "\n"
+        else
+          sep + values.to_s << "\n"
+        end
       end
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -624,6 +624,8 @@ module TSV
   def self.traverse(obj, options = {}, &block)
     into = options[:into]
 
+    into = options[:into] = Open.open(into, :mode => "w") if Misc.is_filename?(into)
+
     case into
     when :stream
       sout = Misc.open_pipe false, false do |sin|                                                                                                                                           

data/lib/rbbt/tsv/util.rb

@@ -184,7 +184,11 @@ module TSV
     str = "" 
     str << preamble.strip << "\n" if preamble and not preamble.empty?
     if fields
-      str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      if fields.empty?
+        str << header_hash << (key_field || "ID").to_s << "\n" 
+      else
+        str << header_hash << (key_field || "ID").to_s << sep << (fields * sep) << "\n" 
+      end
     end
 
     str

data/lib/rbbt/util/config.rb

@@ -15,7 +15,7 @@ module Rbbt::Config
   end
 
   def self.load_file(file)
-    Log.debug "Loading file: #{ file }"
+    Log.debug "Loading config file: #{ file }"
     TSV.traverse file, :type => :array do |line|
       next if line =~ /^#/
       key, value, *tokens = line.strip.split(/\s/)
@@ -85,6 +85,7 @@ module Rbbt::Config
     priorities
   end
 
+  # For equal priorities the matching prioritizes tokens ealier in the list
   def self.get(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     default = options.nil? ? nil : options[:default]

data/lib/rbbt/util/misc/system.rb

@@ -113,7 +113,7 @@ end
   end
 
   def self.is_filename?(string)
-    return true if defined? PATH and Path === string
+    return true if defined? Path and Path === string
     return true if string.respond_to? :exists
     return true if String === string and string.length < 265 and File.exist?(string)
     return false

data/lib/rbbt/workflow/definition.rb

@@ -83,7 +83,7 @@ module Workflow
         self.archive_deps
         self.dependencies = self.dependencies - [dep]
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
-        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist? dep.files_dir
+        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
         Open.ln_h dep.path, self.tmp_path
         case remove.to_s
         when 'true'
@@ -92,8 +92,10 @@ module Workflow
           dep.recursive_clean
         end
       else
-        Open.rm_rf self.files_dir 
-        Open.link dep.files_dir, self.files_dir
+        if Open.exists?(dep.files_dir)
+          Open.rm_rf self.files_dir 
+          Open.link dep.files_dir, self.files_dir
+        end
         Open.link dep.path, self.path
       end
       nil

data/lib/rbbt/workflow/step.rb

@@ -84,6 +84,7 @@ class Step
   end
 
   def load_dependencies_from_info
+    relocated = nil
     @dependencies = (self.info[:dependencies] || []).collect do |task,name,dep_path|
       if Open.exists?(dep_path) || Open.exists?(dep_path + '.info')
         Workflow._load_step dep_path
@@ -411,7 +412,7 @@ class Step
       return 
     end
 
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
 
       @result = nil
       @pid = nil
@@ -419,8 +420,8 @@ class Step
       Misc.insist do
         Open.rm info_file if Open.exists?(info_file)
         Open.rm md5_file if Open.exists?(md5_file)
-        Open.rm path if (Open.exists?(path) or Open.broken_link?(path))
-        Open.rm_rf files_dir if Open.exists?(files_dir)
+        Open.rm path if (Open.exists?(path) || Open.broken_link?(path))
+        Open.rm_rf files_dir if Open.exists?(files_dir) || Open.broken_link?(files_dir)
         Open.rm pid_file if Open.exists?(pid_file)
         Open.rm tmp_path if Open.exists?(tmp_path)
       end

data/lib/rbbt/workflow/util/archive.rb

@@ -262,6 +262,7 @@ puts resource[path].find(search_path)
 
     job_files.each do |file|
       begin
+        Log.debug "Purging #{file}"
         Open.rm_rf file if Open.exists?(file)
       rescue
         Log.warn "Could not erase '#{file}': #{$!.message}"

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -0,0 +1,190 @@
+require 'rbbt/workflow'
+
+module Workflow
+  class Orchestrator
+
+    def self.job_workload(job)
+      workload = {job => []}
+      return workload if job.done?
+
+      job.dependencies.each do |dep|
+        next if dep.done?
+        workload.merge!(job_workload(dep))
+        workload[job] += workload[dep]
+        workload[job] << dep
+      end
+
+      job.input_dependencies.each do |dep|
+        next if dep.done?
+        workload.merge!(job_workload(dep))
+        workload[job] += workload[dep]
+        workload[job] << dep
+      end
+
+      workload
+    end
+
+    def self.job_rules(rules, job)
+      workflow = job.workflow.to_s
+      task_name = job.task_name.to_s
+
+      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow]
+      return IndiferentHash.setup(rules["defaults"]) unless rules[workflow][task_name]
+
+      job_rules = IndiferentHash.setup(rules[workflow][task_name])
+      rules["defaults"].each{|k,v| job_rules[k] ||= v } if rules["defaults"]
+      job_rules
+    end
+
+    def self.purge_duplicates(candidates)
+      seen = Set.new
+      candidates.select do |job|
+        if seen.include? job.path
+          false
+        else
+          seen << job.path
+          true
+        end
+      end
+    end
+
+    def self.job_resources(rules, job)
+      resources = (job_rules(rules, job) || {})["resources"] || {}
+
+      IndiferentHash.setup(resources)
+
+      default_resources = rules["default_resources"] || rules["defaults"]["resources"]
+      default_resources.each{|k,v| resources[k] ||= v } if default_resources
+
+      resources
+    end
+
+    def self.sort_candidates(candidates, rules)
+      seen = Set.new
+      candidates.sort_by do |job|
+        - job_resources(rules, job).values.inject(0){|acc,e| acc += e}
+      end
+    end
+
+    def self.candidates(workload, rules)
+      if rules.empty?
+        candidates = workload.select{|k,v| v.empty? }.
+          collect{|k,v| k}.
+          reject{|k| k.done? }
+      else
+        candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
+          select{|k,v| v.empty? }.
+          collect{|k,v| k }.
+          reject{|k| k.done? }
+      end
+
+      top_level = workload.keys - workload.values.flatten
+
+      candidates = purge_duplicates candidates
+      candidates = sort_candidates candidates, rules
+
+      candidates
+    end
+
+    attr_accessor :available_resources, :resources_requested, :resources_used, :timer
+
+    def initialize(timer = 5, available_resources = {})
+      @timer               = timer
+      @available_resources = IndiferentHash.setup(available_resources)
+      @resources_requested = IndiferentHash.setup({})
+      @resources_used      = IndiferentHash.setup({})
+    end
+
+    def release_resources(job)
+      if resources_used[job]
+        resources_used[job].each do |resource,value| 
+          next if resource == 'size'
+          resources_requested[resource] -= value.to_i
+        end
+        resources_used.delete job
+      end
+    end
+
+    def check_resources(rules, job)
+      resources = Orchestrator.job_resources(rules, job)
+
+      limit_resources = resources.select{|resource,value| available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource]  }.collect{|resource,v| resource }
+      if limit_resources.any?
+        Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}"
+      else
+
+        resources_used[job] = resources
+        resources.each do |resource,value| 
+          resources_requested[resource] ||= 0
+          resources_requested[resource] += value.to_i
+        end
+        Log.low "Orchestrator producing #{job.path} with resources #{resources}"
+
+        return yield
+      end
+    end
+
+    def run_with_rules(rules, job)
+      job_rules = Orchestrator.job_rules(rules, job)
+
+      Rbbt::Config.with_config do 
+        job_rules[:config_keys].each do |config|
+          Rbbt::Config.process_config config
+        end if job_rules && job_rules[:config_keys]
+
+        log = job_rules[:log] if job_rules 
+        log = Log.severity if log.nil?
+        Log.with_severity log do
+          job.produce(false, true)
+        end
+      end
+    end
+
+    def process(rules, jobs)
+      begin
+
+        workload = jobs.inject({}){|acc,job| acc.merge!(Orchestrator.job_workload(job)) }
+
+        while workload.values.flatten.any?
+
+          candidates = resources_used.keys + Orchestrator.candidates(workload, rules)
+          raise "No candidates" if candidates.empty?
+
+          candidates.each do |job|
+            case 
+            when (job.error? || job.aborted?)
+              if job.recoverable_error?
+                job.clean
+                raise TryAgain
+              else
+                next
+              end
+              release_resources(job)
+            when job.done?
+              Log.debug "Orchestrator done #{job.path}"
+              release_resources(job)
+              raise TryAgain
+
+            when job.running?
+              next
+
+            else
+              check_resources(rules, job) do
+                run_with_rules(rules, job)
+              end
+            end
+          end
+
+          new_workload = {}
+          workload.each do |k,v|
+            next if k.done?
+            new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
+          end
+          sleep timer
+        end
+      rescue TryAgain
+        retry
+      end
+    end
+  end
+end

data/lib/rbbt/workflow/util/trace.rb

@@ -0,0 +1,182 @@
+require 'rbbt/util/R'
+
+module Workflow
+  def self.trace(seed_jobs, options = {})
+
+    jobs = []
+    seed_jobs.each{|j| jobs << j; jobs += j.rec_dependencies}
+
+    data = TSV.setup({}, "Job~Workflow,Task,Start,End#:type=:list")
+    min_start = nil
+    max_done = nil
+    jobs.each do |job|
+      next unless job.info[:done]
+      started = job.info[:started]
+      ddone = job.info[:done]
+
+      code = [job.workflow, job.task_name].compact.collect{|s| s.to_s} * "."
+      code = code + '.' + job.name
+
+      data[code] = [job.workflow.to_s, job.task_name, started, ddone]
+      if min_start.nil?
+        min_start = started
+      else
+        min_start = started if started < min_start
+      end
+
+      if max_done.nil?
+        max_done = ddone
+      else
+        max_done = ddone if ddone > max_done
+      end
+    end
+
+    data.add_field "Start.second" do |k,value|
+      value["Start"] - min_start
+    end
+
+    data.add_field "End.second" do |k,value|
+      value["End"] - min_start
+    end
+
+    if options[:fix_gap]
+      ranges = []
+      data.through do |k,values|
+        start, eend = values.values_at "Start.second", "End.second"
+
+        ranges << (start..eend)
+      end
+
+      gaps = {}
+      last = nil
+      Misc.collapse_ranges(ranges).each do |range|
+        start = range.begin
+        eend = range.end
+        if last
+          gaps[last] = start - last
+        end
+        last = eend
+      end
+
+      data.process "End.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+
+      data.process "Start.second" do |value,k,values|
+        gap = Misc.sum(gaps.select{|pos,size| pos < values["Start.second"]}.collect{|pos,size| size})
+        value - gap
+      end
+    end
+
+    tasks_info = {}
+
+    jobs.each do |dep|
+      next unless dep.info[:done]
+      task = [dep.workflow, dep.task_name].compact.collect{|s| s.to_s} * "#"
+      info = tasks_info[task] ||= {}
+
+      time = dep.info[:done] - dep.info[:started]
+      info[:time] ||= []
+      info[:time] << time
+
+      cpus = nil
+      spark = false
+      shard = false
+      dep.info[:config_keys].select do |kinfo| 
+        key, value, tokens = kinfo
+        key = key.to_s
+        cpus = value if key.include? 'cpu'
+        spark = value if key == 'spark'
+        shard = value if key == 'shard'
+      end
+
+      info[:cpus] = cpus || 1
+      info[:spark] = spark
+      info[:shard] = shard
+    end
+
+    stats = TSV.setup({}, "Task~Calls,Avg. Time,Total Time,Cpus,Spark,Shard#:type=:list")
+
+    tasks_info.each do |task, info|
+      time_lists, cpus, spark, shard = info.values_at :time, :cpus, :spark, :shard
+      avg_time = Misc.mean(time_lists)
+      total_time = Misc.sum(time_lists)
+      calls = time_lists.length
+      stats[task] = [calls, avg_time, total_time, cpus, spark, shard]
+    end
+
+    raise "No jobs to process" if data.size == 0
+
+    start = data.column("Start.second").values.flatten.collect{|v| v.to_f}.min
+    eend = data.column("End.second").values.flatten.collect{|v| v.to_f}.max
+    total = eend - start
+    Log.info "Total time elapsed: #{total} seconds"
+
+    if options[:fix_gap]
+      total_gaps = Misc.sum(gaps.collect{|k,v| v})
+      Log.info "Total gaps: #{total_gaps} seconds"
+    end
+
+    plot, width, height = options.values_at :plot, :width, :height
+    if plot
+      data.R <<-EOF, [:svg]
+    rbbt.require('tidyverse')
+    rbbt.require('ggplot2')
+
+    names(data) <- make.names(names(data))
+    data$id = rownames(data)
+    data$content = data$Task
+    data$start = data$Start
+    data$end = data$End
+    data$Project = data$Workflow
+
+    tasks = data
+
+    #theme_gantt <- function(base_size=11, base_family="Source Sans Pro Light") {
+    theme_gantt <- function(base_size=11, base_family="Sans Serif") {
+      ret <- theme_bw(base_size, base_family) %+replace%
+        theme(panel.background = element_rect(fill="#ffffff", colour=NA),
+              axis.title.x=element_text(vjust=-0.2), axis.title.y=element_text(vjust=1.5),
+              title=element_text(vjust=1.2, family="Source Sans Pro Semibold"),
+              panel.border = element_blank(), axis.line=element_blank(),
+              panel.grid.minor=element_blank(),
+              panel.grid.major.y = element_blank(),
+              panel.grid.major.x = element_line(size=0.5, colour="grey80"),
+              axis.ticks=element_blank(),
+              legend.position="bottom", 
+              axis.title=element_text(size=rel(1.2), family="Source Sans Pro Semibold"),
+              strip.text=element_text(size=rel(1.5), family="Source Sans Pro Semibold"),
+              strip.background=element_rect(fill="#ffffff", colour=NA),
+              panel.spacing.y=unit(1.5, "lines"),
+              legend.key = element_blank())
+
+      ret
+    }
+
+    tasks.long <- tasks %>%
+    gather(date.type, task.date, -c(Project, Task, id, Start.second, End.second)) %>%
+    arrange(date.type, task.date) %>%
+    mutate(id = factor(id, levels=rev(unique(id)), ordered=TRUE))
+
+    x.breaks <- seq(length(tasks$Task) + 0.5 - 3, 0, by=-3)
+
+    timeline <- ggplot(tasks.long, aes(y=id, yend=id, x=Start.second, xend=End.second, colour=Task)) + 
+      geom_segment() + 
+      geom_vline(xintercept=x.breaks, colour="grey80", linetype="dotted") + 
+      guides(colour=guide_legend(title=NULL)) +
+      labs(x=NULL, y=NULL) + 
+      theme_gantt() + theme(axis.text.x=element_text(angle=45, hjust=1))
+
+    rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, pointsize=6)
+      EOF
+    end
+
+    if options[:plot_data]
+      data
+    else
+      stats
+    end
+
+  end
+end

data/share/rbbt_commands/purge_job

@@ -3,7 +3,6 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
-require 'rbbt/workflow/remote/ssh/get'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -14,8 +13,7 @@ Remove a job and its dependencies
 $ rbbt purge [options] <job_path> 
 
 -h--help Print this help
--t--test Do a verbose dry run
--r--relocate Include relocated dependencies
+-r--recursive Remove recursively
 
 EOF
 if options[:help]
@@ -31,4 +29,4 @@ path, search_path, _sep, *other = ARGV
 
 raise ParameterException, "No path given" if path.nil? 
 
-Step.purge(path, options[:relocate])
+Step.purge(path, options[:recursive])

data/share/rbbt_commands/system/status

@@ -201,7 +201,7 @@ workflows.sort.each do |workflow,tasks|
                    Step::INFO_SERIALIZER.load(f)
                  end
                rescue
-                 Log.exception $!
+                 #Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -392,6 +392,20 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
+  def test_traverse_into_path
+    size = 100
+    array = (1..size).to_a.collect{|n| n.to_s}
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      io = TSV.traverse array, :into => tmpfile do |e|
+        e
+      end
+      io.join
+      assert_equal size, Open.read(tmpfile).split("\n").length
+    end
+  end
+
+
   def test_traverse_progress
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}

data/test/rbbt/tsv/test_manipulate.rb

@@ -117,6 +117,26 @@ row2 A B C
     end
   end
 
+  def test_slice_empty
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a b c
+row2 A B C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
+      tsv = tsv.slice []
+      assert tsv.fields.empty?
+      TmpFile.with_file do |tmpfile|
+        iii tsv.to_s
+        Open.write(tmpfile, tsv.to_s)
+        tsv = TSV.open tmpfile
+        assert tsv.fields.empty?
+      end
+    end
+  end
+
   def test_select
      content =<<-EOF
 #Id    ValueA    ValueB    OtherID

data/test/rbbt/workflow/util/test_orchestrator.rb

@@ -0,0 +1,136 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/util/orchestrator'
+require 'rbbt/workflow/util/trace'
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+module TestWF
+  extend Workflow 
+  
+  MULT = 0.1
+  task :a => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :a
+  task :b => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :b
+  task :c => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :c
+  task :d => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+end
+
+class TestClass < Test::Unit::TestCase
+  def _test_orchestrate
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+
+  def test_orchestrate_size
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.6
+  version: 5.28.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -327,7 +327,6 @@ files:
 - lib/rbbt/workflow/remote_workflow/remote_step.rb
 - lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
 - lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
-- lib/rbbt/workflow/schedule.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/accessor.rb
@@ -337,7 +336,9 @@ files:
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb
+- lib/rbbt/workflow/util/orchestrator.rb
 - lib/rbbt/workflow/util/provenance.rb
+- lib/rbbt/workflow/util/trace.rb
 - share/Rlib/plot.R
 - share/Rlib/svg.R
 - share/Rlib/util.R
@@ -516,8 +517,10 @@ files:
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
@@ -545,7 +548,9 @@ summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
 - test/rbbt/test_entity.rb
 - test/rbbt/workflow/test_remote_workflow.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb

data/lib/rbbt/workflow/schedule.rb

@@ -1,238 +0,0 @@
-class Step
-  class ResourceManager
-    class NotEnoughResources < Exception
-    end
-
-    attr_accessor :cpus, :memory
-    def initialize(cpus = nil, memory = nil)
-      @cpus = cpus
-      @memory = memory
-      @sem_file = "ResourceManager-" + rand(10000).to_s
-      @semaphore = RbbtSemaphore.create_semaphore(@sem_file, 1)
-    end
-
-    def allocate(cpus = nil, memory = nil, &block)
-      RbbtSemaphore.synchronize(@semaphore) do
-        if (@cpus && cpus && @cpus < cups)  ||
-          (@memory && memory && @memory < memory) 
-          raise NotEnoughResources
-        end
-        begin
-          @cpus -= cpus
-          @memory -= memory
-          yield
-        rescue
-          @cpus += cpus
-          @memory += memory
-        end
-      end
-    end
-
-    def finalize(manager)
-      RbbtSemaphore.delete_semaphore(@sem_file)
-    end
-
-    def self.finalize(manager)
-      proc { manager.finalize }
-    end
-  end
-
-  class Scheduler
-    attr_accessor :jobs, :cpus, :dep_jobs, :job_deps, :jobps
-    def initialize(jobs, cpus)
-      @jobs = jobs
-      @cpus = cpus
-      
-      @job_deps = {}
-
-      with_deps = jobs.dup
-      @dep_jobs = {}
-      @job_deps = {}
-      @jobps = {}
-      @missing = Set.new
-      while with_deps.any?
-        job = with_deps.pop
-        @jobps[job.path] = job
-        @missing << job.path unless job.done?
-
-        jdeps = job.dependencies
-        jdeps += job.inputs.flatten.select{|i| Step === i}
-
-        jdeps.reject!{|dep| dep.done? }
-        @job_deps[job.path] = []
-        jdeps.each do |dep|
-          #next if dep.done?
-          @dep_jobs[dep.path] ||= []
-          @job_deps[job.path] << dep.path
-          @dep_jobs[dep.path] << job.path
-          with_deps << dep unless @job_deps.include? dep.path
-        end
-      end
-
-      def ready
-        @job_deps.select do |jobp,deps|
-          (@missing & deps).empty?
-        end.collect{|jobp,deps| jobp}
-      end
-
-      def used
-        iii @dep_jobs
-        @dep_jobs.select do |dep,jobs|
-          iif [dep, @missing.to_a, jobs]
-          (@missing & jobs).empty?
-        end.keys
-      end
-
-      def next
-        priorities = {}
-        @jobs.each do |job|
-          priorities[job.path] = 1
-        end
-
-        @missing.each do |jobp|
-        end
-
-        ready.first
-      end
-    end
-  end
-
-  #def self._priorities(jobs)
-  #  job_level = {}
-  #  jobs.each do |job|
-  #    job_level[job.path] = 1.0
-  #  end
-
-  #  with_deps = jobs.dup
-  #  dep_jobs = {}
-  #  job_deps = {}
-  #  while with_deps.any?
-  #    job = with_deps.pop
-  #    level = job_level[job.path]
-  #    job_deps[job.path] = []
-  #    jdeps = job.dependencies
-  #    jdeps += job.inputs.flatten.select{|i| Step === i}
-
-  #    jdeps.reject!{|dep| dep.done? }
-  #    jdeps.each do |dep|
-  #      next if dep.done?
-  #      dep_jobs[dep.path] ||= []
-  #      job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
-  #      job_deps[job.path] << dep.path
-  #      dep_jobs[dep.path]  << job.path
-  #      with_deps << dep unless job_deps.include? dep.path
-  #    end
-  #  end
-  #  [job_level, job_deps, dep_jobs]
-  #end
-
-  #def self.produce_jobs(jobs, cpus, step_cpus = {})
-  #  require 'fc'
-
-  #  step_cpus = IndiferentHash.setup(step_cpus || {})
-
-  #  deps = []
-
-  #  jobs = [jobs] unless Array === jobs
-
-  #  job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-  #  jobps = {}
-  #  (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
-  #    jobps[job.path] = job
-  #  end
-
-  #  prio_queue = FastContainers::PriorityQueue.new :max
-
-  #  job_deps.each do |jobp,depps|
-  #    next if depps.any?
-  #    level = job_level[jobp]
-
-  #    prio_queue.push(jobp, level) 
-  #  end
-
-  #  queue = RbbtProcessQueue.new cpus
-
-  #  missing = job_deps.keys
-  #  queue.callback do |jobp|
-  #    Log.info "Done: #{jobp}"
-  #    missing -= [jobp]
-
-  #    job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-  #    parentsp = dep_jobs[jobp]
-
-  #    parentsp.each do |parentp|
-  #      next unless job_deps[parentp].include? jobp
-  #      job_deps[parentp] -= [jobp]
-  #      if job_deps[parentp].empty?
-  #        level = job_level[parentp]
-  #        prio_queue.push(parentp, level )
-  #      end
-  #    end if parentsp
-  #    prio_queue_new = FastContainers::PriorityQueue.new :max
-  #    while prio_queue.any?
-  #      elem = prio_queue.pop
-  #      prio_queue_new.push(elem, job_level[elem])
-  #    end
-  #    prio_queue = prio_queue_new
-  #  end
-  #  
-  #  queue.init do |jobp|
-  #    Log.info "Processing: #{jobp}"
-  #    job = jobps[jobp]
-  #    job_cpus = step_cpus[job.task_name] || 1
-  #    sleep 0.5
-  #    #job.produce
-  #    jobp
-  #  end
-
-  #  while missing.any?
-  #    while prio_queue.empty? && missing.any?
-  #      sleep 1
-  #    end
-  #    break if missing.empty?
-  #    jobp = prio_queue.pop
-  #    queue.process jobp
-  #  end
-
-  #  queue.join
-  #end
-end
-
-
-if __FILE__ == $0
-  require 'rbbt/workflow'
-
-  module TestWF
-    extend Workflow
-    input :num, :integer
-    task :dep => :integer do |num|
-      num
-    end
-    dep :dep, :num => 1
-    dep :dep, :num => 2
-    dep :dep, :num => 3
-    task :test do 
-      dependencies.collect{|d| d.load.to_s}  * ","
-    end
-  end
-  Log.severity = 0
-  job = TestWF.job(:test)
-  job.recursive_clean
-
-  Rbbt::Config.load_file Rbbt.etc.config_profile.HTS.find
-  Workflow.require_workflow "Sample"
-  Workflow.require_workflow "HTS"
-
-  jobs = []
-#  jobs << Sample.job(:mutect2, "QUINTANA-15")
-#  jobs << Sample.job(:mutect2, "QUINTANA-25")
-#  jobs << Sample.job(:mutect2, "QUINTANA-22")
-  jobs << Sample.job(:mutect2, "small")
-
-  sched = Step::Scheduler.new(jobs, 3)
-  iii sched.ready
-  iii sched.used
-  iii sched.next
-end