rbbt-util 5.28.6 → 5.28.11

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.6
+  version: 5.28.11
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -318,6 +318,8 @@ files:
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
+- lib/rbbt/workflow/integration/ansible.rb
+- lib/rbbt/workflow/integration/ansible/workflow.rb
 - lib/rbbt/workflow/integration/cromwell.rb
 - lib/rbbt/workflow/integration/nextflow.rb
 - lib/rbbt/workflow/remote_workflow.rb
@@ -327,7 +329,6 @@ files:
 - lib/rbbt/workflow/remote_workflow/remote_step.rb
 - lib/rbbt/workflow/remote_workflow/remote_step/rest.rb
 - lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb
-- lib/rbbt/workflow/schedule.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/accessor.rb
@@ -337,7 +338,9 @@ files:
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb
+- lib/rbbt/workflow/util/orchestrator.rb
 - lib/rbbt/workflow/util/provenance.rb
+- lib/rbbt/workflow/util/trace.rb
 - share/Rlib/plot.R
 - share/Rlib/svg.R
 - share/Rlib/util.R
@@ -346,6 +349,7 @@ files:
 - share/config.ru
 - share/install/software/lib/install_helpers
 - share/rbbt_commands/alias
+- share/rbbt_commands/ansible
 - share/rbbt_commands/app/install
 - share/rbbt_commands/app/start
 - share/rbbt_commands/app/template
@@ -512,12 +516,13 @@ files:
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_tmpfile.rb
-- test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
@@ -545,9 +550,10 @@ summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
 - test/rbbt/test_entity.rb
 - test/rbbt/workflow/test_remote_workflow.rb
+- test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
-- test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb

data/lib/rbbt/workflow/schedule.rb

@@ -1,238 +0,0 @@
-class Step
-  class ResourceManager
-    class NotEnoughResources < Exception
-    end
-
-    attr_accessor :cpus, :memory
-    def initialize(cpus = nil, memory = nil)
-      @cpus = cpus
-      @memory = memory
-      @sem_file = "ResourceManager-" + rand(10000).to_s
-      @semaphore = RbbtSemaphore.create_semaphore(@sem_file, 1)
-    end
-
-    def allocate(cpus = nil, memory = nil, &block)
-      RbbtSemaphore.synchronize(@semaphore) do
-        if (@cpus && cpus && @cpus < cups)  ||
-          (@memory && memory && @memory < memory) 
-          raise NotEnoughResources
-        end
-        begin
-          @cpus -= cpus
-          @memory -= memory
-          yield
-        rescue
-          @cpus += cpus
-          @memory += memory
-        end
-      end
-    end
-
-    def finalize(manager)
-      RbbtSemaphore.delete_semaphore(@sem_file)
-    end
-
-    def self.finalize(manager)
-      proc { manager.finalize }
-    end
-  end
-
-  class Scheduler
-    attr_accessor :jobs, :cpus, :dep_jobs, :job_deps, :jobps
-    def initialize(jobs, cpus)
-      @jobs = jobs
-      @cpus = cpus
-      
-      @job_deps = {}
-
-      with_deps = jobs.dup
-      @dep_jobs = {}
-      @job_deps = {}
-      @jobps = {}
-      @missing = Set.new
-      while with_deps.any?
-        job = with_deps.pop
-        @jobps[job.path] = job
-        @missing << job.path unless job.done?
-
-        jdeps = job.dependencies
-        jdeps += job.inputs.flatten.select{|i| Step === i}
-
-        jdeps.reject!{|dep| dep.done? }
-        @job_deps[job.path] = []
-        jdeps.each do |dep|
-          #next if dep.done?
-          @dep_jobs[dep.path] ||= []
-          @job_deps[job.path] << dep.path
-          @dep_jobs[dep.path] << job.path
-          with_deps << dep unless @job_deps.include? dep.path
-        end
-      end
-
-      def ready
-        @job_deps.select do |jobp,deps|
-          (@missing & deps).empty?
-        end.collect{|jobp,deps| jobp}
-      end
-
-      def used
-        iii @dep_jobs
-        @dep_jobs.select do |dep,jobs|
-          iif [dep, @missing.to_a, jobs]
-          (@missing & jobs).empty?
-        end.keys
-      end
-
-      def next
-        priorities = {}
-        @jobs.each do |job|
-          priorities[job.path] = 1
-        end
-
-        @missing.each do |jobp|
-        end
-
-        ready.first
-      end
-    end
-  end
-
-  #def self._priorities(jobs)
-  #  job_level = {}
-  #  jobs.each do |job|
-  #    job_level[job.path] = 1.0
-  #  end
-
-  #  with_deps = jobs.dup
-  #  dep_jobs = {}
-  #  job_deps = {}
-  #  while with_deps.any?
-  #    job = with_deps.pop
-  #    level = job_level[job.path]
-  #    job_deps[job.path] = []
-  #    jdeps = job.dependencies
-  #    jdeps += job.inputs.flatten.select{|i| Step === i}
-
-  #    jdeps.reject!{|dep| dep.done? }
-  #    jdeps.each do |dep|
-  #      next if dep.done?
-  #      dep_jobs[dep.path] ||= []
-  #      job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
-  #      job_deps[job.path] << dep.path
-  #      dep_jobs[dep.path]  << job.path
-  #      with_deps << dep unless job_deps.include? dep.path
-  #    end
-  #  end
-  #  [job_level, job_deps, dep_jobs]
-  #end
-
-  #def self.produce_jobs(jobs, cpus, step_cpus = {})
-  #  require 'fc'
-
-  #  step_cpus = IndiferentHash.setup(step_cpus || {})
-
-  #  deps = []
-
-  #  jobs = [jobs] unless Array === jobs
-
-  #  job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-  #  jobps = {}
-  #  (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
-  #    jobps[job.path] = job
-  #  end
-
-  #  prio_queue = FastContainers::PriorityQueue.new :max
-
-  #  job_deps.each do |jobp,depps|
-  #    next if depps.any?
-  #    level = job_level[jobp]
-
-  #    prio_queue.push(jobp, level) 
-  #  end
-
-  #  queue = RbbtProcessQueue.new cpus
-
-  #  missing = job_deps.keys
-  #  queue.callback do |jobp|
-  #    Log.info "Done: #{jobp}"
-  #    missing -= [jobp]
-
-  #    job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-  #    parentsp = dep_jobs[jobp]
-
-  #    parentsp.each do |parentp|
-  #      next unless job_deps[parentp].include? jobp
-  #      job_deps[parentp] -= [jobp]
-  #      if job_deps[parentp].empty?
-  #        level = job_level[parentp]
-  #        prio_queue.push(parentp, level )
-  #      end
-  #    end if parentsp
-  #    prio_queue_new = FastContainers::PriorityQueue.new :max
-  #    while prio_queue.any?
-  #      elem = prio_queue.pop
-  #      prio_queue_new.push(elem, job_level[elem])
-  #    end
-  #    prio_queue = prio_queue_new
-  #  end
-  #  
-  #  queue.init do |jobp|
-  #    Log.info "Processing: #{jobp}"
-  #    job = jobps[jobp]
-  #    job_cpus = step_cpus[job.task_name] || 1
-  #    sleep 0.5
-  #    #job.produce
-  #    jobp
-  #  end
-
-  #  while missing.any?
-  #    while prio_queue.empty? && missing.any?
-  #      sleep 1
-  #    end
-  #    break if missing.empty?
-  #    jobp = prio_queue.pop
-  #    queue.process jobp
-  #  end
-
-  #  queue.join
-  #end
-end
-
-
-if __FILE__ == $0
-  require 'rbbt/workflow'
-
-  module TestWF
-    extend Workflow
-    input :num, :integer
-    task :dep => :integer do |num|
-      num
-    end
-    dep :dep, :num => 1
-    dep :dep, :num => 2
-    dep :dep, :num => 3
-    task :test do 
-      dependencies.collect{|d| d.load.to_s}  * ","
-    end
-  end
-  Log.severity = 0
-  job = TestWF.job(:test)
-  job.recursive_clean
-
-  Rbbt::Config.load_file Rbbt.etc.config_profile.HTS.find
-  Workflow.require_workflow "Sample"
-  Workflow.require_workflow "HTS"
-
-  jobs = []
-#  jobs << Sample.job(:mutect2, "QUINTANA-15")
-#  jobs << Sample.job(:mutect2, "QUINTANA-25")
-#  jobs << Sample.job(:mutect2, "QUINTANA-22")
-  jobs << Sample.job(:mutect2, "small")
-
-  sched = Step::Scheduler.new(jobs, 3)
-  iii sched.ready
-  iii sched.used
-  iii sched.next
-end

data/test/rbbt/workflow/remote/test_client.rb

@@ -1,56 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/workflow'
-require 'rbbt/workflow/remote/client'
-
-class TestWFRest
-  extend Workflow
-
-  input :name, :string, "Name to greet", "World"
-  task :hi => :string do |name|
-    "Hello #{name}"
-  end
-
-  dep :hi
-  task :intro => :string do 
-    step(:hi).load + ", nice to meet you"
-  end
-end
-
-class TestRemote < Test::Unit::TestCase
-
-  def test_ssh
-    Log.severity = 0
-    client = WorkflowRemoteClient.new "ssh://turbo:Translation", "Translation"
-    job = client.job("translate", "SSH-TEST", :genes => ["TP53","KRAS"])
-    assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
-  end
-
-  def test_rest
-    Log.with_severity 0 do
-
-      workflow_server(TestWFRest) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
-      end
-
-      workflow_server(TestWFRest, :Port => 1902) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
-      end
-    end
-  end
-
-  def test_rest_clean
-    real_job = TestWFRest.job(:hi)
-    real_job.run
-    first_time = File.ctime(real_job.path)
-    workflow_server(TestWFRest) do |client|
-      assert File.ctime(real_job.path) == first_time
-      assert_equal "Hello World", client.job(:hi, nil, {}).clean.run
-      assert File.ctime(real_job.path) > first_time
-    end
-  end
-end
-