rbbt-util 5.28.2 → 5.28.8

data/share/rbbt_commands/app/start

@@ -54,9 +54,9 @@ app = ARGV.shift
 
 ENV["RServe-session"] = options[:RServe_session] || app
 
-app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps.find
+app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
 
-app_dir = app_dir[app]
+app_dir = app_dir[app].find
 
 server = options[:server] || 'puma'
 Misc.in_dir(app_dir) do

data/share/rbbt_commands/purge_job

@@ -3,7 +3,6 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
-require 'rbbt/workflow/remote/ssh/get'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -14,8 +13,7 @@ Remove a job and its dependencies
 $ rbbt purge [options] <job_path> 
 
 -h--help Print this help
--t--test Do a verbose dry run
--r--relocate Include relocated dependencies
+-r--recursive Remove recursively
 
 EOF
 if options[:help]
@@ -31,4 +29,4 @@ path, search_path, _sep, *other = ARGV
 
 raise ParameterException, "No path given" if path.nil? 
 
-Step.purge(path, options[:relocate])
+Step.purge(path, options[:recursive])

data/share/rbbt_commands/system/status

@@ -201,7 +201,7 @@ workflows.sort.each do |workflow,tasks|
                    Step::INFO_SERIALIZER.load(f)
                  end
                rescue
-                 Log.exception $!
+                 #Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)

data/share/rbbt_commands/workflow/forget_deps

@@ -29,8 +29,6 @@ path = ARGV[0]
 raise ParameterException, "No path given" if path.nil?
 step = Workflow.load_step path
 
-
-iif step.rec_dependencies
-
 step.archive_deps
+step.copy_files_dir
 step.set_info :dependencies, []

data/share/rbbt_commands/workflow/server

@@ -25,6 +25,7 @@ $ rbbt workflow server [options] <Workflow>
 --stream Activate streaming of workflow tasks
 -fs--file_server Activate file serving for resources
 -mj--monitor_jobs Monitor jobs (UNSAFE)
+-a--app_dir* Application execution directory
 --export_all Export all workflow tasks (use with caution!)
 --export* Export workflow tasks (asynchronous)
 --export_asynchronous* Export workflow tasks as asynchronous
@@ -74,6 +75,7 @@ sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_syn
 exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
 
 TmpFile.with_file do |app_dir|
+  app_dir = options[:app_dir] if options[:app_dir]
   Misc.in_dir(app_dir) do
     app_dir = Path.setup(app_dir.dup)
     Open.write(app_dir.etc.target_workflow.find, workflow)

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -392,6 +392,20 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal size, stream.read.split("\n").length
   end
 
+  def test_traverse_into_path
+    size = 100
+    array = (1..size).to_a.collect{|n| n.to_s}
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      io = TSV.traverse array, :into => tmpfile do |e|
+        e
+      end
+      io.join
+      assert_equal size, Open.read(tmpfile).split("\n").length
+    end
+  end
+
+
   def test_traverse_progress
     size = 1000
     array = (1..size).to_a.collect{|n| n.to_s}

data/test/rbbt/tsv/test_manipulate.rb

@@ -117,6 +117,26 @@ row2 A B C
     end
   end
 
+  def test_slice_empty
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a b c
+row2 A B C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
+      tsv = tsv.slice []
+      assert tsv.fields.empty?
+      TmpFile.with_file do |tmpfile|
+        iii tsv.to_s
+        Open.write(tmpfile, tsv.to_s)
+        tsv = TSV.open tmpfile
+        assert tsv.fields.empty?
+      end
+    end
+  end
+
   def test_select
      content =<<-EOF
 #Id    ValueA    ValueB    OtherID

data/test/rbbt/workflow/util/test_orchestrator.rb

@@ -0,0 +1,136 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/util/orchestrator'
+require 'rbbt/workflow/util/trace'
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+module TestWF
+  extend Workflow 
+  
+  MULT = 0.1
+  task :a => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :a
+  task :b => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :b
+  task :c => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :c
+  task :d => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+end
+
+class TestClass < Test::Unit::TestCase
+  def _test_orchestrate
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+
+  def test_orchestrate_size
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.2
+  version: 5.28.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-12 00:00:00.000000000 Z
+date: 2020-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -327,7 +327,6 @@ files:
 - lib/rbbt/workflow/remote_workflow/remote_step.rb
 - lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
 - lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
-- lib/rbbt/workflow/schedule.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/accessor.rb
@@ -337,7 +336,9 @@ files:
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb
+- lib/rbbt/workflow/util/orchestrator.rb
 - lib/rbbt/workflow/util/provenance.rb
+- lib/rbbt/workflow/util/trace.rb
 - share/Rlib/plot.R
 - share/Rlib/svg.R
 - share/Rlib/util.R
@@ -516,8 +517,10 @@ files:
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
@@ -545,7 +548,9 @@ summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
 - test/rbbt/test_entity.rb
 - test/rbbt/workflow/test_remote_workflow.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb

data/lib/rbbt/workflow/schedule.rb

@@ -1,238 +0,0 @@
-class Step
-  class ResourceManager
-    class NotEnoughResources < Exception
-    end
-
-    attr_accessor :cpus, :memory
-    def initialize(cpus = nil, memory = nil)
-      @cpus = cpus
-      @memory = memory
-      @sem_file = "ResourceManager-" + rand(10000).to_s
-      @semaphore = RbbtSemaphore.create_semaphore(@sem_file, 1)
-    end
-
-    def allocate(cpus = nil, memory = nil, &block)
-      RbbtSemaphore.synchronize(@semaphore) do
-        if (@cpus && cpus && @cpus < cups)  ||
-          (@memory && memory && @memory < memory) 
-          raise NotEnoughResources
-        end
-        begin
-          @cpus -= cpus
-          @memory -= memory
-          yield
-        rescue
-          @cpus += cpus
-          @memory += memory
-        end
-      end
-    end
-
-    def finalize(manager)
-      RbbtSemaphore.delete_semaphore(@sem_file)
-    end
-
-    def self.finalize(manager)
-      proc { manager.finalize }
-    end
-  end
-
-  class Scheduler
-    attr_accessor :jobs, :cpus, :dep_jobs, :job_deps, :jobps
-    def initialize(jobs, cpus)
-      @jobs = jobs
-      @cpus = cpus
-      
-      @job_deps = {}
-
-      with_deps = jobs.dup
-      @dep_jobs = {}
-      @job_deps = {}
-      @jobps = {}
-      @missing = Set.new
-      while with_deps.any?
-        job = with_deps.pop
-        @jobps[job.path] = job
-        @missing << job.path unless job.done?
-
-        jdeps = job.dependencies
-        jdeps += job.inputs.flatten.select{|i| Step === i}
-
-        jdeps.reject!{|dep| dep.done? }
-        @job_deps[job.path] = []
-        jdeps.each do |dep|
-          #next if dep.done?
-          @dep_jobs[dep.path] ||= []
-          @job_deps[job.path] << dep.path
-          @dep_jobs[dep.path] << job.path
-          with_deps << dep unless @job_deps.include? dep.path
-        end
-      end
-
-      def ready
-        @job_deps.select do |jobp,deps|
-          (@missing & deps).empty?
-        end.collect{|jobp,deps| jobp}
-      end
-
-      def used
-        iii @dep_jobs
-        @dep_jobs.select do |dep,jobs|
-          iif [dep, @missing.to_a, jobs]
-          (@missing & jobs).empty?
-        end.keys
-      end
-
-      def next
-        priorities = {}
-        @jobs.each do |job|
-          priorities[job.path] = 1
-        end
-
-        @missing.each do |jobp|
-        end
-
-        ready.first
-      end
-    end
-  end
-
-  #def self._priorities(jobs)
-  #  job_level = {}
-  #  jobs.each do |job|
-  #    job_level[job.path] = 1.0
-  #  end
-
-  #  with_deps = jobs.dup
-  #  dep_jobs = {}
-  #  job_deps = {}
-  #  while with_deps.any?
-  #    job = with_deps.pop
-  #    level = job_level[job.path]
-  #    job_deps[job.path] = []
-  #    jdeps = job.dependencies
-  #    jdeps += job.inputs.flatten.select{|i| Step === i}
-
-  #    jdeps.reject!{|dep| dep.done? }
-  #    jdeps.each do |dep|
-  #      next if dep.done?
-  #      dep_jobs[dep.path] ||= []
-  #      job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
-  #      job_deps[job.path] << dep.path
-  #      dep_jobs[dep.path]  << job.path
-  #      with_deps << dep unless job_deps.include? dep.path
-  #    end
-  #  end
-  #  [job_level, job_deps, dep_jobs]
-  #end
-
-  #def self.produce_jobs(jobs, cpus, step_cpus = {})
-  #  require 'fc'
-
-  #  step_cpus = IndiferentHash.setup(step_cpus || {})
-
-  #  deps = []
-
-  #  jobs = [jobs] unless Array === jobs
-
-  #  job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-  #  jobps = {}
-  #  (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
-  #    jobps[job.path] = job
-  #  end
-
-  #  prio_queue = FastContainers::PriorityQueue.new :max
-
-  #  job_deps.each do |jobp,depps|
-  #    next if depps.any?
-  #    level = job_level[jobp]
-
-  #    prio_queue.push(jobp, level) 
-  #  end
-
-  #  queue = RbbtProcessQueue.new cpus
-
-  #  missing = job_deps.keys
-  #  queue.callback do |jobp|
-  #    Log.info "Done: #{jobp}"
-  #    missing -= [jobp]
-
-  #    job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-  #    parentsp = dep_jobs[jobp]
-
-  #    parentsp.each do |parentp|
-  #      next unless job_deps[parentp].include? jobp
-  #      job_deps[parentp] -= [jobp]
-  #      if job_deps[parentp].empty?
-  #        level = job_level[parentp]
-  #        prio_queue.push(parentp, level )
-  #      end
-  #    end if parentsp
-  #    prio_queue_new = FastContainers::PriorityQueue.new :max
-  #    while prio_queue.any?
-  #      elem = prio_queue.pop
-  #      prio_queue_new.push(elem, job_level[elem])
-  #    end
-  #    prio_queue = prio_queue_new
-  #  end
-  #  
-  #  queue.init do |jobp|
-  #    Log.info "Processing: #{jobp}"
-  #    job = jobps[jobp]
-  #    job_cpus = step_cpus[job.task_name] || 1
-  #    sleep 0.5
-  #    #job.produce
-  #    jobp
-  #  end
-
-  #  while missing.any?
-  #    while prio_queue.empty? && missing.any?
-  #      sleep 1
-  #    end
-  #    break if missing.empty?
-  #    jobp = prio_queue.pop
-  #    queue.process jobp
-  #  end
-
-  #  queue.join
-  #end
-end
-
-
-if __FILE__ == $0
-  require 'rbbt/workflow'
-
-  module TestWF
-    extend Workflow
-    input :num, :integer
-    task :dep => :integer do |num|
-      num
-    end
-    dep :dep, :num => 1
-    dep :dep, :num => 2
-    dep :dep, :num => 3
-    task :test do 
-      dependencies.collect{|d| d.load.to_s}  * ","
-    end
-  end
-  Log.severity = 0
-  job = TestWF.job(:test)
-  job.recursive_clean
-
-  Rbbt::Config.load_file Rbbt.etc.config_profile.HTS.find
-  Workflow.require_workflow "Sample"
-  Workflow.require_workflow "HTS"
-
-  jobs = []
-#  jobs << Sample.job(:mutect2, "QUINTANA-15")
-#  jobs << Sample.job(:mutect2, "QUINTANA-25")
-#  jobs << Sample.job(:mutect2, "QUINTANA-22")
-  jobs << Sample.job(:mutect2, "small")
-
-  sched = Step::Scheduler.new(jobs, 3)
-  iii sched.ready
-  iii sched.used
-  iii sched.next
-end