rbbt-util 5.26.6 → 5.26.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a856a792ffcb0f6e395e2452f75624050653ec72
-  data.tar.gz: 193c1d2cf21224d5233151417f61d3e8b1cb52fa
+SHA256:
+  metadata.gz: 73dc573b119d00638cd05d906db7e13ae1d4c1f402bc447192127bf42a2538de
+  data.tar.gz: f18941095a6cd114cffc2140bd5272b9a624bf83b2dd423da91efdb4381c82a6
 SHA512:
-  metadata.gz: 5c58296853107056685a6a12045eed29b92027972997ccde07b1498dd569bb92093199fa0b74f4d56b894f2dddd647947f8b14275e2ee312ead5eee350511717
-  data.tar.gz: e19d29cb5e5694dcd5f95f4944ea287cc390b10bcde624553824b4e4072aca245edd2857e1f99b6dec1091264200076f0e41fc5c8e1653e68f8a9273e550f315
+  metadata.gz: 0231c8e2b307671ad8abea117ad86d086fe21a9db202ca8486a16e97b352671f447f67a74fc7bbf0f8fc622f85e996fbacd20eb054e749f11a9a13b0650e0cfd
+  data.tar.gz: 4b347b870203aa41f520cd767a30df6a1ab84b4278d6454923f59f4d8ba30137a8577cf7a1f22aa4b4e5c6a30ec1a5f0884edb79a84706bf988eb76b81589f04

data/lib/rbbt/monitor.rb

@@ -173,7 +173,7 @@ module Rbbt
                     end
                   else
                     #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
-                    cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+                    cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
 
                     CMD.cmd(cmd, :pipe => true)
                   end

data/lib/rbbt/tsv/manipulate.rb

@@ -395,7 +395,7 @@ module TSV
           new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
         end
       end
-    when String === method
+    when (String === method || Symbol === method)
       if block_given?
         case 
         when block.arity == 1

data/lib/rbbt/util/log/progress/report.rb

@@ -75,7 +75,7 @@ module Log
     def thr_msg
       if @history.nil?
         @history ||= [[@ticks, Time.now] ]
-      else
+      elsif @last_ticks != @ticks
         @history << [@ticks, Time.now]
         max_history ||= case 
                       when @ticks > 20
@@ -96,13 +96,16 @@ module Log
         @history.shift if @history.length > max_history
       end
 
+      @last_ticks = @ticks
+
       @mean_max ||= 0
-      if @history.length > 3
+      if @history.length > 2
 
         sticks, stime = @history.first
         ssticks, sstime = @history[-3]
         lticks, ltime = @history.last
 
+
         mean = @mean = (lticks - sticks).to_f / (ltime - stime)
         short_mean = (lticks - ssticks).to_f / (ltime - sstime)
 

data/lib/rbbt/util/misc/omics.rb

@@ -328,8 +328,8 @@ module Misc
   end
 
   def self.chr_cmp_strict(chr1, chr2)
-    chr1 = chr1.sub(/^chr/)
-    chr2 = chr2.sub(/^chr/)
+    chr1 = chr1.sub(/^chr/, '')
+    chr2 = chr2.sub(/^chr/, '')
     if (m1 = chr1.match(/(\d+)$/)) && (m2 = chr2.match(/(\d+)$/))
       m1[1].to_i <=> m2[1].to_i
     elsif chr1 =~ /\d+$/

data/lib/rbbt/util/tmpfile.rb

@@ -17,13 +17,13 @@ module TmpFile
 
   # Creates a random file name, with the given suffix and a random number
   # up to +max+
-  def self.random_name(s = "", max = 10000000)
+  def self.random_name(s = "tmp-", max = 10000000)
     n = rand(max)
     s + n.to_s
   end
 
   # Creates a random filename in the temporary directory
-  def self.tmp_file(s = "", max=10000000, dir = TMPDIR)
+  def self.tmp_file(s = "tmp-", max=10000000, dir = TMPDIR)
     File.expand_path(File.join(dir, random_name(s,max)))
   end
 
@@ -31,7 +31,7 @@ module TmpFile
     options, content, erase = content, nil, true if Hash === content
     options, erase = erase, true if Hash === erase 
 
-    prefix = options[:prefix] || ""
+    prefix = options[:prefix] || "tmp-"
     tmpdir = options[:tmpdir] || TMPDIR
     max = options[:max] || 10000000
     tmpfile = tmp_file prefix, max, tmpdir
@@ -53,7 +53,7 @@ module TmpFile
   end
 
   def self.with_dir(erase = true, options = {})
-    prefix = options[:prefix] || ""
+    prefix = options[:prefix] || "tmpdir-"
     tmpdir = tmp_file prefix
 
     FileUtils.mkdir_p tmpdir

data/lib/rbbt/workflow/definition.rb

@@ -73,8 +73,9 @@ module Workflow
       if forget
         self.dependencies = self.dependencies - [dep]
         self.set_info :dependency, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+        Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist? dep.files_dir
-        Open.cp dep.path, self.tmp_path
+        Open.ln_h dep.path, self.tmp_path
       else
         Open.link dep.path, self.path
       end

data/lib/rbbt/workflow/examples.rb

@@ -42,7 +42,7 @@ module Workflow
       dir = Path.setup(dir.dup)
       input_names.each do |input|
         file = dir[input].find
-        file = dir.glob(input.to_s + ".*").first if file.nil? or not file.exists?
+        file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not file.exists?
         Log.debug "Trying #{ input }: #{file}"
         next unless file and file.exists?
 

data/lib/rbbt/workflow/schedule.rb

@@ -1,6 +1,6 @@
 class Step
   class ResourceManager
-    class NotEoughResources < Exception
+    class NotEnoughResources < Exception
     end
 
     attr_accessor :cpus, :memory
@@ -15,7 +15,7 @@ class Step
       RbbtSemaphore.synchronize(@semaphore) do
         if (@cpus && cpus && @cpus < cups)  ||
           (@memory && memory && @memory < memory) 
-          raise NotEoughResources
+          raise NotEnoughResources
         end
         begin
           @cpus -= cpus
@@ -61,7 +61,7 @@ class Step
         jdeps.reject!{|dep| dep.done? }
         @job_deps[job.path] = []
         jdeps.each do |dep|
-          next if dep.done?
+          #next if dep.done?
           @dep_jobs[dep.path] ||= []
           @job_deps[job.path] << dep.path
           @dep_jobs[dep.path] << job.path
@@ -69,128 +69,135 @@ class Step
         end
       end
 
-
-      def self.ready
+      def ready
         @job_deps.select do |jobp,deps|
           (@missing & deps).empty?
         end.collect{|jobp,deps| jobp}
       end
 
-      def self.next
+      def used
+        iii @dep_jobs
+        @dep_jobs.select do |dep,jobs|
+          iif [dep, @missing.to_a, jobs]
+          (@missing & jobs).empty?
+        end.keys
+      end
+
+      def next
         priorities = {}
         @jobs.each do |job|
-          priorities = 1
+          priorities[job.path] = 1
         end
 
         @missing.each do |jobp|
         end
 
-        @dep_jobsb
-      end
-    end
-  end
-
-  def self._priorities(jobs)
-    job_level = {}
-    jobs.each do |job|
-      job_level[job.path] = 1.0
-    end
-
-    with_deps = jobs.dup
-    dep_jobs = {}
-    job_deps = {}
-    while with_deps.any?
-      job = with_deps.pop
-      level = job_level[job.path]
-      job_deps[job.path] = []
-      jdeps = job.dependencies
-      jdeps += job.inputs.flatten.select{|i| Step === i}
-
-      jdeps.reject!{|dep| dep.done? }
-      jdeps.each do |dep|
-        next if dep.done?
-        dep_jobs[dep.path] ||= []
-        job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
-        job_deps[job.path] << dep.path
-        dep_jobs[dep.path]  << job.path
-        with_deps << dep unless job_deps.include? dep.path
+        ready.first
       end
     end
-    [job_level, job_deps, dep_jobs]
   end
 
-  def self.produce_jobs(jobs, cpus, step_cpus = {})
-    require 'fc'
-
-    step_cpus = IndiferentHash.setup(step_cpus || {})
-
-    deps = []
-
-    jobs = [jobs] unless Array === jobs
-
-    job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-    jobps = {}
-    (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
-      jobps[job.path] = job
-    end
-
-    prio_queue = FastContainers::PriorityQueue.new :max
-
-    job_deps.each do |jobp,depps|
-      next if depps.any?
-      level = job_level[jobp]
-
-      prio_queue.push(jobp, level) 
-    end
-
-    queue = RbbtProcessQueue.new cpus
-
-    missing = job_deps.keys
-    queue.callback do |jobp|
-      Log.info "Done: #{jobp}"
-      missing -= [jobp]
-
-      job_level, job_deps, dep_jobs = self._priorities(jobs)
-
-      parentsp = dep_jobs[jobp]
-
-      parentsp.each do |parentp|
-        next unless job_deps[parentp].include? jobp
-        job_deps[parentp] -= [jobp]
-        if job_deps[parentp].empty?
-          level = job_level[parentp]
-          prio_queue.push(parentp, level )
-        end
-      end if parentsp
-      prio_queue_new = FastContainers::PriorityQueue.new :max
-      while prio_queue.any?
-        elem = prio_queue.pop
-        prio_queue_new.push(elem, job_level[elem])
-      end
-      prio_queue = prio_queue_new
-    end
-    
-    queue.init do |jobp|
-      Log.info "Processing: #{jobp}"
-      job = jobps[jobp]
-      job_cpus = step_cpus[job.task_name] || 1
-      sleep 0.5
-      #job.produce
-      jobp
-    end
-
-    while missing.any?
-      while prio_queue.empty? && missing.any?
-        sleep 1
-      end
-      break if missing.empty?
-      jobp = prio_queue.pop
-      queue.process jobp
-    end
-
-    queue.join
-  end
+  #def self._priorities(jobs)
+  #  job_level = {}
+  #  jobs.each do |job|
+  #    job_level[job.path] = 1.0
+  #  end
+
+  #  with_deps = jobs.dup
+  #  dep_jobs = {}
+  #  job_deps = {}
+  #  while with_deps.any?
+  #    job = with_deps.pop
+  #    level = job_level[job.path]
+  #    job_deps[job.path] = []
+  #    jdeps = job.dependencies
+  #    jdeps += job.inputs.flatten.select{|i| Step === i}
+
+  #    jdeps.reject!{|dep| dep.done? }
+  #    jdeps.each do |dep|
+  #      next if dep.done?
+  #      dep_jobs[dep.path] ||= []
+  #      job_level[dep.path] = level / (10 * jdeps.length) if job_level[dep.path].nil? || job_level[dep.path] < level / (10 * jdeps.length)
+  #      job_deps[job.path] << dep.path
+  #      dep_jobs[dep.path]  << job.path
+  #      with_deps << dep unless job_deps.include? dep.path
+  #    end
+  #  end
+  #  [job_level, job_deps, dep_jobs]
+  #end
+
+  #def self.produce_jobs(jobs, cpus, step_cpus = {})
+  #  require 'fc'
+
+  #  step_cpus = IndiferentHash.setup(step_cpus || {})
+
+  #  deps = []
+
+  #  jobs = [jobs] unless Array === jobs
+
+  #  job_level, job_deps, dep_jobs = self._priorities(jobs)
+
+  #  jobps = {}
+  #  (jobs + jobs.collect{|job| job.rec_dependencies}).flatten.uniq.each do |job|
+  #    jobps[job.path] = job
+  #  end
+
+  #  prio_queue = FastContainers::PriorityQueue.new :max
+
+  #  job_deps.each do |jobp,depps|
+  #    next if depps.any?
+  #    level = job_level[jobp]
+
+  #    prio_queue.push(jobp, level) 
+  #  end
+
+  #  queue = RbbtProcessQueue.new cpus
+
+  #  missing = job_deps.keys
+  #  queue.callback do |jobp|
+  #    Log.info "Done: #{jobp}"
+  #    missing -= [jobp]
+
+  #    job_level, job_deps, dep_jobs = self._priorities(jobs)
+
+  #    parentsp = dep_jobs[jobp]
+
+  #    parentsp.each do |parentp|
+  #      next unless job_deps[parentp].include? jobp
+  #      job_deps[parentp] -= [jobp]
+  #      if job_deps[parentp].empty?
+  #        level = job_level[parentp]
+  #        prio_queue.push(parentp, level )
+  #      end
+  #    end if parentsp
+  #    prio_queue_new = FastContainers::PriorityQueue.new :max
+  #    while prio_queue.any?
+  #      elem = prio_queue.pop
+  #      prio_queue_new.push(elem, job_level[elem])
+  #    end
+  #    prio_queue = prio_queue_new
+  #  end
+  #  
+  #  queue.init do |jobp|
+  #    Log.info "Processing: #{jobp}"
+  #    job = jobps[jobp]
+  #    job_cpus = step_cpus[job.task_name] || 1
+  #    sleep 0.5
+  #    #job.produce
+  #    jobp
+  #  end
+
+  #  while missing.any?
+  #    while prio_queue.empty? && missing.any?
+  #      sleep 1
+  #    end
+  #    break if missing.empty?
+  #    jobp = prio_queue.pop
+  #    queue.process jobp
+  #  end
+
+  #  queue.join
+  #end
 end
 
 
@@ -219,9 +226,13 @@ if __FILE__ == $0
   Workflow.require_workflow "HTS"
 
   jobs = []
-  jobs << Sample.job(:mutect2, "QUINTANA-15")
-  jobs << Sample.job(:mutect2, "QUINTANA-25")
-  jobs << Sample.job(:mutect2, "QUINTANA-28")
+#  jobs << Sample.job(:mutect2, "QUINTANA-15")
+#  jobs << Sample.job(:mutect2, "QUINTANA-25")
+#  jobs << Sample.job(:mutect2, "QUINTANA-22")
+  jobs << Sample.job(:mutect2, "small")
 
   sched = Step::Scheduler.new(jobs, 3)
+  iii sched.ready
+  iii sched.used
+  iii sched.next
 end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -119,6 +119,11 @@ class Step
     Log.info str
   end
 
+  def input_dependencies
+    inputs.flatten.select{|i| Step === i}
+  end
+
+
   def execute_dependency(dependency, log = true)
     task_name = self.task_name
     canfail_paths = self.canfail_paths
@@ -247,7 +252,7 @@ class Step
       respawn = rest && rest.include?(:always_respawn)
       respawn = :always if respawn.nil?
 
-      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path} [#{cpus}]", :respawn => respawn) do |dep|
+      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{self.short_path} [#{cpus}]", :respawn => respawn) do |dep|
         begin
           Signal.trap(:INT) do
             dep.abort
@@ -338,7 +343,7 @@ class Step
 
     compute_deps = rec_dependencies.collect do |dep|
       next unless ComputeDependency === dep
-      dep.rec_dependencies
+      dep.rec_dependencies + dep.inputs.flatten.select{|i| Step === i}
     end.compact.flatten.uniq
 
     canfail_paths = self.canfail_paths
@@ -353,24 +358,15 @@ class Step
         raise $! unless canfail_paths.include? step.path
       end
       next unless step.dependencies and step.dependencies.any?
-      step.dependencies.each do |step_dep|
+      (step.dependencies + step.input_dependencies).each do |step_dep|
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step_dep
       end
-      step.inputs.each do |inputs|
-        inputs = [inputs] unless Array === inputs
-        inputs.each do |step_dep|
-          next unless Step === step_dep
-          next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
-          dep_step[step_dep.path] ||= []
-          dep_step[step_dep.path] << step_dep
-        end
-      end
     end
 
     produced = []
-    dependencies.each do |dep|
+    (dependencies + input_dependencies).each do |dep|
       next unless ComputeDependency === dep
       if dep.compute == :produce
         dep.produce 
@@ -382,16 +378,16 @@ class Step
 
     required_dep_paths = []
     dep_step.each do |path,list|
-      required_dep_paths << path if list.length > 1
+      #required_dep_paths << path if list.length > 1
+      required_dep_paths << path if (list & dependencies).any?
     end
 
     required_dep_paths.concat dependencies.collect{|dep| dep.path}
-    required_dep_paths.concat(rec_dependencies.collect do |dep| 
-      dep.inputs.flatten.select{|i| Step === i}.collect{|d| d.path}
-    end.flatten)
+
+    required_dep_paths.concat input_dependencies.collect{|dep| dep.path}
 
     required_dep_paths.concat(dependencies.collect do |dep| 
-      [dep.path] + dep.inputs.flatten.select{|i| Step === i}.collect{|d| d.path}
+      [dep.path] + dep.input_dependencies
     end.flatten)
 
     log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"

data/lib/rbbt/workflow/step/run.rb

@@ -238,15 +238,6 @@ class Step
             :versions => Rbbt.versions
           })
 
-
-          begin
-            run_dependencies
-          rescue Exception
-            Open.rm pid_file if Open.exists?(pid_file)
-            stop_dependencies
-            raise $!
-          end
-
           new_inputs = []
           @inputs.each_with_index do |input,i|
             name = @task.inputs[i]
@@ -305,6 +296,14 @@ class Step
             set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, info_inputs)) 
           end
 
+          begin
+            run_dependencies
+          rescue Exception
+            Open.rm pid_file if Open.exists?(pid_file)
+            stop_dependencies
+            raise $!
+          end
+
           set_info :started, (start_time = Time.now)
           log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
 

data/share/rbbt_commands/workflow/prov

@@ -117,7 +117,7 @@ if options[:plot]
       nodes <- read.csv("#{node_info_file}", header=T, as.is=T)
       links <- read.csv("#{edge_file}", header=T, as.is=T)
 
-      library(igraph)
+      rbbt.require('igraph')
 
       net <- graph.data.frame(links, nodes, directed=T)
       net <- simplify(net, remove.multiple = F, remove.loops = T)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.6
+  version: 5.26.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-29 00:00:00.000000000 Z
+date: 2019-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -517,8 +517,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)