rbbt-util 5.32.9 → 5.32.15

data/lib/rbbt/util/log/progress/report.rb

@@ -12,32 +12,35 @@ module Log
     attr_accessor :history, :mean_max, :max_history
     def thr_msg
       if @history.nil?
-        @history ||= [[@ticks, Time.now] ]
+        @history ||= [[0, @start], [@ticks, Time.now] ]
       elsif @last_ticks != @ticks
         @history << [@ticks, Time.now]
-        max_history ||= case 
-                      when @ticks > 20
-                        count = @ticks - @last_count
-                        count = 1 if count == 0
-                        if @max
-                          times = @max / count
-                          num = times / 20
-                          num = 2 if num < 2
-                        else
-                          num = 10
+        max_history ||= begin
+                          max_history = case 
+                                        when @ticks > 20
+                                          count = @ticks - @last_count
+                                          count = 1 if count == 0
+                                          if @max
+                                            times = @max / count
+                                            num = times / 20
+                                            num = 2 if num < 2
+                                          else
+                                            num = 10
+                                          end
+                                          count * num
+                                        else
+                                          20
+                                        end
+                          max_history = 30 if max_history > 30
+                          max_history
                         end
-                        count * num
-                      else
-                        20
-                      end
-        max_history = 30 if max_history > 30
         @history.shift if @history.length > max_history
       end
 
       @last_ticks = @ticks
 
       @mean_max ||= 0
-      if @history.length > 2
+      if @history.length > 3
 
         sticks, stime = @history.first
         ssticks, sstime = @history[-3]

data/lib/rbbt/util/log/progress/util.rb

@@ -80,10 +80,11 @@ module Log
 
     def self.with_bar(max, options = {})
       bar = new_bar(max, options)
+      res = nil
       begin
         error = false
-        yield bar
         keep = false
+        yield bar
       rescue KeepBar
         keep = true
       rescue

data/lib/rbbt/util/misc/omics.rb

@@ -511,9 +511,9 @@ module Misc
 
            case chr_prefix.to_s.downcase
            when "remove"
-             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
-           when "true", "add"
              sort_order = sort_order.collect{|chr| chr.sub('chr', '') }  if sort_order.first.include?('chr') 
+           when "true", "add"
+             sort_order = sort_order.collect{|chr| "chr" + chr }  unless sort_order.first.include?('chr')
            end
 
            sort_genomic_locations_by_contig(mutations, sort_order)

data/lib/rbbt/util/misc/system.rb

@@ -112,10 +112,10 @@ end
     res
   end
 
-  def self.is_filename?(string)
+  def self.is_filename?(string, need_to_exists = true)
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and File.exist?(string)
+    return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
     return false
   end
 

data/lib/rbbt/util/python.rb

@@ -3,10 +3,56 @@ require 'pycall/import'
 
 module RbbtPython
   extend PyCall::Import
+
+  def self.exec(script)
+    PyCall.exec(script)
+  end
+
+  def self.iterate(iterator, options = {})
+    bar = options[:bar]
+
+    case bar
+    when TrueClass
+      bar = Log::ProgressBar.new nil, :desc => "RbbtPython iterate"
+    when String
+      bar = Log::ProgressBar.new nil, :desc => bar
+    end
+
+    while true
+      begin
+        yield iterator.__next__
+        bar.tick if bar
+      rescue PyCall::PyError
+        if $!.type.to_s == "<class 'StopIteration'>"
+          break
+        else
+          raise $!
+        end
+      rescue
+        bar.error if bar
+        raise $!
+      end
+    end
+
+    Log::ProgressBar.remove_bar bar if bar
+    nil
+  end
+
+  def self.collect(iterator, options = {}, &block)
+    acc = []
+    self.iterate(iterator, options) do |elem|
+      res = block.call elem
+      acc << res
+    end
+    acc
+  end
+
   def self.run(mod = nil, imports = nil, &block)
     if mod
-      if imports
+      if Array === imports
         pyfrom mod, :import => imports
+      elsif Hash === imports
+        pyimport mod, imports
       else
         pyimport mod 
       end
@@ -17,8 +63,10 @@ module RbbtPython
 
   def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
     if mod
-      if imports
+      if Array === imports
         pyfrom mod, :import => imports
+      elsif Hash === imports
+        pyimport mod, imports
       else
         pyimport mod 
       end
@@ -31,8 +79,10 @@ module RbbtPython
 
   def self.run_log_stderr(mod = nil, imports = nil, severity = 0, &block)
     if mod
-      if imports
+      if Array === imports
         pyfrom mod, :import => imports
+      elsif Hash === imports
+        pyimport mod, imports
       else
         pyimport mod 
       end
@@ -49,4 +99,14 @@ module RbbtPython
     end
   end
 
+  def self.add_paths(paths)
+    self.run 'sys' do
+      paths.each do |path|
+        sys.path.append path
+      end
+    end
+  end
+
+  RbbtPython.add_paths Rbbt.python.find_all
+  RbbtPython.pyimport "rbbt"
 end

data/lib/rbbt/util/simpleDSL.rb

@@ -38,13 +38,13 @@ module SimpleDSL
     hook_method(method)
 
     # Execute
+    @config ||= {}
     if actions.is_a? Proc
       begin
-        require 'parse_tree_extensions'
-        require 'parse_tree'
-        require 'ruby2ruby'
-        @config[@@method_name] = actions.to_ruby.collect[1..-2].join
+        require 'method_source'
+        @config[@@method_name] = actions.source.split("\n")[1..-2] * "\n"
       rescue Exception
+        Log.exception $!
         @config[@@method_name] = NoRuby2Ruby.new "The gem ruby2ruby is not installed. It will not work on ruby 1.9."
       end
       

data/lib/rbbt/workflow.rb

@@ -347,6 +347,11 @@ module Workflow
 
     inputs = IndiferentHash.setup(inputs)
 
+    not_overriden = inputs.delete :not_overriden 
+    if not_overriden
+      inputs[:not_overriden] = :not_overriden_dep
+    end
+
     Workflow.resolve_locals(inputs)
 
     task_info = task_info(taskname)
@@ -400,7 +405,11 @@ module Workflow
     jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
 
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
-    overriden = has_overriden_inputs || dependencies.select{|dep| dep.overriden }.any?
+
+    overriden_deps = dependencies.select{|d| d.overriden }
+    true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
+
+    overriden = has_overriden_inputs || overriden_deps.any?
 
     if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
       step_path = step_path taskname, jobname, [], [], task.extension
@@ -413,7 +422,16 @@ module Workflow
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
-    job.overriden = overriden
+
+    case not_overriden 
+    when TrueClass
+      job.overriden = has_overriden_inputs || true_overriden_deps.any?
+    when :not_overriden_dep
+      job.overriden = true if has_overriden_inputs || true_overriden_deps.any?
+    else
+      job.overriden = true if has_overriden_inputs || overriden_deps.any?
+    end
+
     job.real_inputs = real_inputs.keys
     job
   end

data/lib/rbbt/workflow/step.rb

@@ -34,13 +34,40 @@ class Step
                     end
   end
 
+
+  def overriden?
+    return true if @overriden
+    return true if dependencies.select{|dep| dep.overriden? }.any?
+    info[:archived_info].each do |f,i|
+      return true if i[:overriden] || i["overriden"]
+    end if info[:archived_info]
+    return false
+  end
+
   def overriden
-    if @overriden.nil? 
-      return [] if dependencies.nil?
-      dependencies.select{|dep| dep.overriden }.any?
-    else
-      @overriden
+    @overriden
+    #if @overriden.nil? 
+    #  return false if dependencies.nil?
+    #  dependencies.select{|dep| dep.overriden? }.any?
+    #else
+    #  @overriden
+    #end
+  end
+
+  def overriden_deps
+    ord = []
+    deps = dependencies.dup
+    while dep = deps.shift
+      case dep.overriden
+      when FalseClass
+        next
+      when Symbol
+        ord << dep
+      else
+        deps += dep.dependencies
+      end
     end
+    ord
   end
 
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
@@ -134,7 +161,11 @@ class Step
 
     archived_info = {}
     dependencies.each do |dep|
-      archived_info[dep.path] = dep.info
+      if Symbol === dep.overriden && ! Open.exists?(dep.info_file)
+        archived_info[dep.path] = dep.overriden
+      else
+        archived_info[dep.path] = dep.info
+      end
       archived_info.merge!(dep.archived_info)
     end if dependencies
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -248,7 +248,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -551,7 +551,7 @@ class Step
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :ending && status != :dependencies && status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/util/data.rb

@@ -0,0 +1,35 @@
+require 'rbbt/workflow'
+require 'rbbt/workflow/examples'
+
+module Workflow
+  module Data
+    def data(directory)
+      @@data_directory = directory
+    end
+
+    def get_datadir(clean_name)
+      data_dir = File.join(@@data_directory, clean_name)
+      raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
+      if Path === @@data_directory
+        @@data_directory.annotate data_dir
+      else
+        Path.setup(data_dir)
+      end
+    end
+
+    def data_task(name, workflow, oname, *rest, &block)
+      dep_task(name, workflow, oname, *rest) do |jobname, options|
+        data_dir = self.get_datadir(jobname)
+        task_info = workflow.task_info(oname)
+        dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
+        data_options = block.call data_dir, dir_options, task_info
+        if data_options.include?(:inputs)
+          data_options
+        else
+          {:inputs => data_options.merge(options)}
+        end
+      end
+    end
+  end
+
+end

data/lib/rbbt/workflow/util/trace.rb

@@ -188,6 +188,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
     seed_jobs.each do |step|
       jobs += step.rec_dependencies + [step]
       step.info[:archived_info].each do |path,ainfo|
+        next unless Hash === ainfo
         archived_step = Step.new path
 
         archived_step.define_singleton_method :info do
@@ -205,7 +206,7 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
 
     end
 
-    jobs = jobs.uniq.sort_by{|job| t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
+    jobs = jobs.uniq.sort_by{|job| [job, job.info]; t = job.info[:started] || Open.mtime(job.path) || Time.now; Time === t ? t : Time.parse(t) }
 
     data = trace_job_times(jobs, options[:fix_gap])
 

data/python/rbbt.py

@@ -0,0 +1,7 @@
+import warnings
+import sys
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+def rbbt():
+    print("Rbbt")

data/share/install/software/lib/install_helpers

@@ -499,7 +499,7 @@ install_jar(){
     local url="$2"
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
-    wget "$url" -O "$OPT_DIR/jars/$name.jar" || wget "$url" -O "$OPT_DIR/jars/$name.jar" --no-check-certificate || (rm "$OPT_DIR/jars/$name.jar"; exit -1)
+    wget "$url" -O "$OPT_DIR/$name/$name.jar" || wget "$url" -O "$OPT_DIR/$name/$name.jar" --no-check-certificate || (rm "$OPT_DIR/$name/$name.jar"; exit -1)
     link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 

data/share/rbbt_commands/hpc/list

@@ -173,7 +173,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
    select = true if jobid && jobid.split(",").include?(id)
-   select = select && cmd.match(/#{search}/) if search
+   select = select && step_path.match(/#{search}/) if search
    next unless select 
  elsif search
    select = false
@@ -206,9 +206,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
    puts Log.color(:magenta, "BATCH parameters: ")
    case job_batch_system 
    when 'slurm'
-     text = CMD.cmd('grep "^#SBATCH" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
    when 'lsf'
-     text = CMD.cmd('grep "^#BSUB" |tail -n +5', :in => Open.read(fcmd)).read.strip
+     text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
    else
      text = ""
    end
@@ -292,13 +292,17 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      step_path = step_line.split(": ").last.strip
      step = Step.new step_path
      step.load_dependencies_from_info
+     has_bar = false
      (step.rec_dependencies + [step]).reverse.each do |j|
        next if j.done? 
-       next unless j.file(:progress).exists?
-       bar = Log::ProgressBar.new 
-       bar.load(j.file(:progress).yaml)
-       puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+       if j.file(:progress).exists?
+         bar = Log::ProgressBar.new 
+         bar.load(j.file(:progress).yaml)
+         puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
+         has_bar = true
+       end
      end
+     puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
    end
  end