rbbt-util 5.26.1 → 5.26.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88ac452446d8a694bdb7d61c0bbd20b1c5f7e0d9
-  data.tar.gz: c6026be2d4aaa199b94cbfe5a3712bcf38577a42
+  metadata.gz: 81378fa8b108cf6280f0056bc0bcfcd2a4e380cb
+  data.tar.gz: 5ed2d44e0bb591608aab12753ff78997251b8828
 SHA512:
-  metadata.gz: 82a0ba5ee4b35a434a40376efd331278a7f2da8c1de248f55f55bd44625ce9bfd2c768e22521d6977a726b2ccd95205d4bbfc218aa2571268f553d1d91355183
-  data.tar.gz: 677e482567fe02fed13f84c263874b9315eef52bbbad45029873a0f56e83f6b07ad3847aa3e4ea12dc1e8fea7755129c7fd5087220e55582450990fd69f2f438
+  metadata.gz: 2930697a87736c4c3753a1dc87c91d544d28630e79ba261fcedf458e049de6097d87286f4c2aa293c735faa7b833a673e5a7be558bad064db0087fcfa4d986c1
+  data.tar.gz: 6d59e6b9993d498f62df6005b08dd68567676069f83ee7b22763529a452ee27a00e2289dbfbc6965593d98ae5411f6a6b86becaa625d5e45dca004d66583f07f

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -176,6 +176,44 @@ module TSV
     end
   end
 
+  def self.traverse_priority_queue(queue, options = {}, &block)
+    callback, bar, join = Misc.process_options options, :callback, :bar, :join
+
+    begin
+      error = false
+      if callback
+        bar.init if bar
+        while queue.any?
+          e = queue.pop
+          begin
+            callback.call yield(e)
+          ensure
+            bar.tick if bar
+          end
+        end
+      else
+        bar.init if bar
+        while queue.any?
+          e = queue.pop
+          begin
+            yield e
+          rescue Exception
+            Log.exception $!
+            raise $!
+          ensure
+            bar.tick if bar
+          end
+        end
+      end
+
+    rescue
+      error = true
+      raise $!
+    ensure
+      Log::ProgressBar.remove_bar(bar) if bar
+      join.call(error) if join
+    end
+  end
   def self.traverse_io_array(io, options = {}, &block)
     callback, bar, join = Misc.process_options options, :callback, :bar, :join
     begin
@@ -278,6 +316,8 @@ module TSV
     Log.low{"Traversing #{name} #{Log.color :green, "->"} #{stream_name(options[:into])}"}
     begin
       case obj
+      when (defined? FastContainers and FastContainers::PriorityQueue)
+        traverse_priority_queue(obj, options, &block)
       when TSV
         traverse_tsv(obj, options, &block)
       when Hash

data/lib/rbbt/workflow/schedule.rb

@@ -0,0 +1,227 @@
+class Step
+  class ResourceManager
+    class NotEoughResources < Exception
+    end
+
+    attr_accessor :cpus, :memory
+    def initialize(cpus = nil, memory = nil)
+      @cpus = cpus
+      @memory = memory
+      @sem_file = "ResourceManager-" + rand(10000).to_s
+      @semaphore = RbbtSemaphore.create_semaphore(@sem_file, 1)
+    end
+
+    def allocate(cpus = nil, memory = nil, &block)
+      RbbtSemaphore.synchronize(@semaphore) do
+        if (@cpus && cpus && @cpus < cups)  ||
+          (@memory && memory && @memory < memory) 
+          raise NotEoughResources
+        end
+        begin
+          @cpus -= cpus
+          @memory -= memory
+          yield
+        rescue
+          @cpus += cpus
+          @memory += memory
+        end
+      end
+    end
+
+    def finalize(manager)
+      RbbtSemaphore.delete_semaphore(@sem_file)
+    end
+
+    def self.finalize(manager)
+      proc { manager.finalize }
+    end
+  end
+
+  class Scheduler
+    attr_accessor :jobs, :cpus, :dep_jobs, :job_deps, :jobps
+    def initialize(jobs, cpus)
+      @jobs = jobs
+      @cpus = cpus
+      
+      @job_deps = {}
+
+      with_deps = jobs.dup
+      @dep_jobs = {}
+      @job_deps = {}
+      @jobps = {}
+      @missing = Set.new
+      while with_deps.any?
+        job = with_deps.pop
+        @jobps[job.path] = job
+        @missing << job.path unless job.done?
+
+        jdeps = job.dependencies
+        jdeps += job.inputs.flatten.select{|i| Step === i}
+
+        jdeps.reject!{|dep| dep.done? }
+        @job_deps[job.path] = []
+        jdeps.each do |dep|
+          next if dep.done?
+          @dep_jobs[dep.path] ||= []
+          @job_deps[job.path] << dep.path
+          @dep_jobs[dep.path] << job.path
+          with_deps << dep unless @job_deps.include? dep.path
+        end
+      end
+
+
+      def self.ready
+        @job_deps.select do |jobp,deps|
+          (@missing & deps).empty?
+        end.collect{|jobp,deps| jobp}
+      end
+
+      def self.next
+        priorities = {}
+        @jobs.each do |job|
+          priorities = 1
+        end
+
+        @missing.each do |jobp|
+        end
+
+        @dep_jobsb
+      end
+    end
+  end
+
+  def self._priorities(jobs)
+    job_level = {}
+    jobs.each do |job|
+      job_level[job.path] = 1.0
+    end
+
+    with_deps = jobs.dup
+    dep_jobs = {}
+    job_deps = {}
+    while with_deps.any?
+      job = with_deps.pop
+      level = job_level[job.path]
+      job_deps[job.path] = []
+      jdeps = job.dependencies
+      jdeps += job.inputs.flatten.select{|i| Step === i}
+
+      jdeps.reject!{|dep| dep.done? }
+      jdeps.each do |dep|
+        next if dep.done?
+        dep_jobs[dep.path] ||= []
+        job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
+        job_deps[job.path] << dep.path
+        dep_jobs[dep.path]  << job.path
+        with_deps << dep unless job_deps.include? dep.path
+      end
+    end
+    [job_level, job_deps, dep_jobs]
+  end
+
+  def self.produce_jobs(jobs, cpus, step_cpus = {})
+    require 'fc'
+
+    step_cpus = IndiferentHash.setup(step_cpus || {})
+
+    deps = []
+
+    jobs = [jobs] unless Array === jobs
+
+    job_level, job_deps, dep_jobs = self._priorities(jobs)
+
+    jobps = {}
+    (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
+      jobps[job.path] = job
+    end
+
+    prio_queue = FastContainers::PriorityQueue.new :max
+
+    job_deps.each do |jobp,depps|
+      next if depps.any?
+      level = job_level[jobp]
+
+      prio_queue.push(jobp, level) 
+    end
+
+    queue = RbbtProcessQueue.new cpus
+
+    missing = job_deps.keys
+    queue.callback do |jobp|
+      Log.info "Done: #{jobp}"
+      missing -= [jobp]
+
+      job_level, job_deps, dep_jobs = self._priorities(jobs)
+
+      parentsp = dep_jobs[jobp]
+
+      parentsp.each do |parentp|
+        next unless job_deps[parentp].include? jobp
+        job_deps[parentp] -= [jobp]
+        if job_deps[parentp].empty?
+          level = job_level[parentp]
+          prio_queue.push(parentp, level )
+        end
+      end if parentsp
+      prio_queue_new = FastContainers::PriorityQueue.new :max
+      while prio_queue.any?
+        elem = prio_queue.pop
+        prio_queue_new.push(elem, job_level[elem])
+      end
+      prio_queue = prio_queue_new
+    end
+    
+    queue.init do |jobp|
+      Log.info "Processing: #{jobp}"
+      job = jobps[jobp]
+      job_cpus = step_cpus[job.task_name] || 1
+      sleep 0.5
+      #job.produce
+      jobp
+    end
+
+    while missing.any?
+      while prio_queue.empty? && missing.any?
+        sleep 1
+      end
+      break if missing.empty?
+      jobp = prio_queue.pop
+      queue.process jobp
+    end
+
+    queue.join
+  end
+end
+
+
+if __FILE__ == $0
+  require 'rbbt/workflow'
+
+  module TestWF
+    extend Workflow
+    input :num, :integer
+    task :dep => :integer do |num|
+      num
+    end
+    dep :dep, :num => 1
+    dep :dep, :num => 2
+    dep :dep, :num => 3
+    task :test do 
+      dependencies.collect{|d| d.load.to_s}  * ","
+    end
+  end
+  Log.severity = 0
+  job = TestWF.job(:test)
+  job.recursive_clean
+
+  Rbbt::Config.load_file Rbbt.etc.config_profile.HTS.find
+  Workflow.require_workflow "Sample"
+  Workflow.require_workflow "HTS"
+
+  jobs = []
+  jobs << Sample.job(:mutect2, "QUINTANA-15")
+  jobs << Sample.job(:mutect2, "QUINTANA-25")
+  jobs << Sample.job(:mutect2, "QUINTANA-28")
+
+  sched = Step::Scheduler.new(jobs, 3)
+end

data/lib/rbbt/workflow/step.rb

@@ -298,6 +298,7 @@ class Step
     pid_file = Step.pid_file path
     md5_file = Step.md5_file path
     files_dir = Step.files_dir path
+    tmp_path = Step.tmp_path path
 
     if ! (Open.writable?(path) && Open.writable?(info_file))
       Log.warn "Could not clean #{path}: not writable"
@@ -315,6 +316,7 @@ class Step
         Open.rm path if (Open.exists?(path) or Open.broken_link?(path))
         Open.rm_rf files_dir if Open.exists?(files_dir)
         Open.rm pid_file if Open.exists?(pid_file)
+        Open.rm tmp_path if Open.exists?(tmp_path)
       end
     end
   end

data/share/install/software/lib/install_helpers

@@ -180,7 +180,7 @@ compile(){
     local extra="$@"
 
     if [ -f Makefile -o -f makefile ]; then
-        make || exit -1
+        make -j 4 || exit -1
         make install || echo "No install"
     fi
 
@@ -205,7 +205,6 @@ prepare_make(){
 
     [ -d src -a ! -e CMakeLists.txt -a ! -e Makefile -a ! -e configure ] && cd src
 
-
     if [ -f config/m4 ]; then
         libtoolize --force
         aclocal
@@ -257,7 +256,9 @@ build_make(){
         echo "Extra params: $extra"
     fi
 
-    prepare_make $name
+    if [ ! -f configure ]; then 
+     prepare_make $name
+    fi
 
     if [ -f configure ]; then 
         ./configure --prefix="$(opt_dir "$name")" $extra

data/share/rbbt_commands/workflow/trace

@@ -151,6 +151,8 @@ tasks_info.each do |task, info|
   stats[task] = [calls,avg_time, cpus, spark, shard]
 end
 
+raise "No jobs to process" if data.size == 0
+
 start = data.column("Start.second").values.flatten.collect{|v| v.to_i}.min
 eend = data.column("End.second").values.flatten.collect{|v| v.to_i}.max
 total = eend - start

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -120,6 +120,21 @@ class TestTSVParallelThrough < Test::Unit::TestCase
 
     assert_equal array, res
   end
+  
+  def test_traverse_priority
+    require 'fc'
+
+    queue = FastContainers::PriorityQueue.new(:min)
+
+    array = []
+    100.times do e = rand(1000).to_i; array << e; queue.push(e,e) end
+
+    res = TSV.traverse queue, :into => [] do |v|
+      v
+    end
+
+    assert_equal array.sort, res
+  end
 
   def test_traverse_array_threads
     require 'rbbt/sources/organism'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.1
+  version: 5.26.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-18 00:00:00.000000000 Z
+date: 2019-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -325,6 +325,7 @@ files:
 - lib/rbbt/workflow/examples.rb
 - lib/rbbt/workflow/prepare.rb
 - lib/rbbt/workflow/provenance.rb
+- lib/rbbt/workflow/schedule.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/dependencies.rb