rbbt-util 5.28.9 → 5.28.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2fa2f71df9aef8810599ab3660c847c084dea75ccb2485d294cfa1962e341162
-  data.tar.gz: 7eb004ddf8bcf30b00325dc5251d9ed73a307de433c18c0847eb4d70a4bea1e2
+  metadata.gz: 443812f3080048a2e39b95ceda4fda2b3f403788c4d32ae4822b0b6920ac1d1f
+  data.tar.gz: e586076cca9dbd72b9dae8ba8be0061269e9bef5d04163f382f2d3b3f353d1b8
 SHA512:
-  metadata.gz: 6ba23322f9ac768f1782c4e26605c50171b6d35d1d3c2acc0ef195e26ca39af851a232e1581ca2ec220899b7780f35f8d0d36e1f61f0b0e6500c883999c3760b
-  data.tar.gz: afa0952f228a951ea78506ad0776fd96dbfbd2b91bac73310ee6807e4a7d0a74c7ca8f582880c5abd28bbf709c8be3d9dff1797056be756962091b2e97031f30
+  metadata.gz: ca6ea9a139a48dfcc76fb04b889d45881d023e9f6fda362da0bb7bd4c63b31217ca0a9bb9f4702186c78b741a1eb1b5c7ca2607512e6dced51026081bdcb8aa7
+  data.tar.gz: 76f021c8d565e2833506f17fb54436d0fe80d29d8bface099cece0445cb70718f8f2126e6fa6eb139f6555e33fe9382d1401339aacb100073bb4964c62f1a260

data/lib/rbbt/entity.rb

@@ -38,7 +38,7 @@ module Entity
                                                    if value.to_s == k.to_s
                                                      found = k
                                                      break
-                                                   elsif value =~ /\(#{Regexp.quote k}\)/
+                                                   elsif value.to_s =~ /\(#{Regexp.quote k}\)/
                                                      found = k
                                                      break
                                                    end

data/lib/rbbt/fix_width_table.rb

@@ -67,7 +67,7 @@ class FixWidthTable
 
   def format(pos, value)
     padding = value_size - value.length
-    if range
+    if @range
       (pos  + [padding, value + ("\0" * padding)]).pack("llll#{mask}")
     else
       [pos, padding, value + ("\0" * padding)].pack("ll#{mask}")
@@ -105,7 +105,7 @@ class FixWidthTable
 
   def idx_value(index)
     return nil if index < 0 or index >= size
-    @file.seek((range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
+    @file.seek((@range ? 17 : 9 ) + (record_size) * index, IO::SEEK_SET)
     padding = @file.read(4).unpack("l").first+1
     txt = @file.read(value_size)
     str = txt.unpack(mask).first
@@ -277,7 +277,8 @@ class FixWidthTable
 
   def [](pos)
     return [] if size == 0
-    if range
+    self.read
+    if @range
       get_range(pos)
     else
       get_point(pos)
@@ -286,7 +287,7 @@ class FixWidthTable
   
   def overlaps(pos, value = false)
     return [] if size == 0
-    idxs = if range
+    idxs = if @range
       get_range(pos, true)
     else
       get_point(pos, true)

data/lib/rbbt/persist.rb

@@ -25,7 +25,7 @@ module Persist
   MEMORY = {} unless defined? MEMORY
   MAX_FILE_LENGTH = 150
 
-  # Return non-false if the first file is newer than the second file
+  # Is 'file' newer than 'path'? return non-true if path is newer than file
   def self.newer?(path, file)
     return true if not Open.exists?(file)
     path = path.find if Path === path

data/lib/rbbt/persist/tsv/adapter.rb

@@ -52,7 +52,6 @@ module Persist
     def lock
       return yield if @locked
       lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Log.stack caller if $LOG
       Misc.lock(lock_filename) do
         begin
           @locked = true

data/lib/rbbt/persist/tsv/fix_width_table.rb

@@ -52,7 +52,7 @@ module Persist
       if TSV::ENTRY_KEYS.include? key
         set_metadata(key, value)
       else
-        if range
+        if @range
           add_range_point key, value
         else
           add key, value
@@ -61,7 +61,7 @@ module Persist
     end
      
     def add(key, value)
-      key = pos_function.call(key) if pos_function and not (range and Array === key)
+      key = pos_function.call(key) if pos_function and not (@range and Array === key)
       super(key, value)
     end
 
@@ -85,8 +85,10 @@ module Persist
     end
 
     def each
+      read
       @size.times do |i|
-        yield i, value(i)
+        v = idx_value(i)
+        yield i, v
       end
     end
 

data/lib/rbbt/tsv/dumper.rb

@@ -32,13 +32,17 @@ module TSV
           sep + ([""] * fields.length) * sep << "\n"
         end
       when Array
-        if fields.nil? or fields.empty?
+        if fields.nil? 
+          sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
+        elsif fields.empty?
           "\n"
         else
           sep + (values.collect{|v| Array === v ? v * "|" : v} * sep) << "\n"
         end
       else
-        if fields.nil? or fields.empty?
+        if fields.nil?
+          sep + values.to_s + "\n"
+        elsif fields.empty?
           "\n"
         else
           sep + values.to_s << "\n"

data/lib/rbbt/util/cmd.rb

@@ -98,6 +98,7 @@ module CMD
     post       = options.delete(:post)
     log        = options.delete(:log)
     no_fail    = options.delete(:no_fail)
+    no_fail    = options.delete(:nofail) if no_fail.nil?
     no_wait    = options.delete(:no_wait)
 
     dont_close_in  = options.delete(:dont_close_in)

data/lib/rbbt/util/misc/bgzf.rb

@@ -7,7 +7,7 @@ module Bgzf
 
   def self.bgzip_cmd
     @@bgzip_cmd ||= begin
-                      path = `bash -c "type -p bgzips"`.strip
+                      path = `bash -c "type -p bgzip"`.strip
                       if path.empty?
                         Rbbt.claim Rbbt.software.opt.htslib, :install, Rbbt.share.install.software.HTSLIB.find(:lib)
                         Rbbt.software.opt.htslib.produce

data/lib/rbbt/util/named_array.rb

@@ -117,7 +117,7 @@ module NamedArray
   #end
 
   def each(&block)
-    if defined?(Entity) and not @fields.nil? and not @fields.empty?
+    if defined?(Entity) && ! (@fields.nil? || @fields.empty?)
       i = 0
       super do |elem|
         field = @fields[i]

data/lib/rbbt/util/open.rb

@@ -16,9 +16,9 @@ module Open
   GREP_CMD = begin
                if ENV["GREP_CMD"] 
                  ENV["GREP_CMD"]
-               elsif File.exists?('/bin/grep')
+               elsif File.exist?('/bin/grep')
                  "/bin/grep"
-               elsif File.exists?('/usr/bin/grep')
+               elsif File.exist?('/usr/bin/grep')
                  "/usr/bin/grep"
                else
                  "grep"
@@ -262,7 +262,7 @@ module Open
     if (dir_sub_path = find_repo_dir(file))
       remove_from_repo(*dir_sub_path)
     else
-      FileUtils.rm(file) if File.exists?(file) or Open.broken_link?(file)
+      FileUtils.rm(file) if File.exist?(file) or Open.broken_link?(file)
     end
   end
 
@@ -333,7 +333,7 @@ module Open
       nil
     else
       target = target.find if Path === target
-      if ! File.exists?(target)
+      if ! File.exist?(target)
         FileUtils.mkdir_p target
       end
     end
@@ -344,8 +344,8 @@ module Open
     target = target.find if Path === target
 
     target = File.join(target, File.basename(source)) if File.directory? target
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     FileUtils.ln_s source, target
   end
 
@@ -353,8 +353,8 @@ module Open
     source = source.find if Path === source
     target = target.find if Path === target
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     FileUtils.ln source, target
   end
 
@@ -362,8 +362,8 @@ module Open
     source = source.find if Path === source
     target = target.find if Path === target
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
+    FileUtils.mkdir_p File.dirname(target) unless File.exist?(File.dirname(target))
+    FileUtils.rm target if File.exist?(target)
     begin
       CMD.cmd("ln -L '#{ source }' '#{ target }'")
     rescue ProcessFailed
@@ -475,6 +475,7 @@ module Open
       File.exist?(file) #|| File.symlink?(file)
     end
   end
+
   class << self
     alias exist? exists?
   end
@@ -747,7 +748,7 @@ module Open
     if (dir_sub_path = find_repo_dir(path))
       writable_repo?(*dir_sub_path)
     else
-      if File.exists?(path)
+      if File.exist?(path)
         File.writable?(path)
       else
         File.writable?(File.dirname(File.expand_path(path)))
@@ -776,14 +777,14 @@ module Open
       file = file.find if Path === file
       begin
         if File.symlink?(file) || File.stat(file).nlink > 1
-          if File.exists?(file + '.info') && defined?(Step)
+          if File.exist?(file + '.info') && defined?(Step)
             done = Step::INFO_SERIALIZER.load(Open.open(file + '.info'))[:done]
             return done if done
           end
 
           file = Pathname.new(file).realpath.to_s 
         end
-        return nil unless File.exists?(file)
+        return nil unless File.exist?(file)
         File.mtime(file)
       rescue
         nil
@@ -793,7 +794,7 @@ module Open
 
   def self.update_mtime(path, target)
     if File.symlink?(target) || File.stat(target).nlink > 1
-      if File.exists?(target + '.info')
+      if File.exist?(target + '.info')
         target = target + '.info'
       else
         target = Pathname.new(target).realpath.to_s 
@@ -801,7 +802,7 @@ module Open
     end
 
     CMD.cmd("touch -r '#{path}' '#{target}'")
-    CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exists?(path + '.info')
+    CMD.cmd("touch -r '#{path}.info' '#{target}'") if File.exist?(path + '.info')
   end
 
   def self.atime(file)
@@ -823,7 +824,7 @@ module Open
   end
 
   def self.broken_link?(path)
-    File.symlink?(path) && ! File.exists?(File.readlink(path))
+    File.symlink?(path) && ! File.exist?(File.readlink(path))
   end
 
   def self.download(url, path)

data/lib/rbbt/workflow/definition.rb

@@ -44,6 +44,7 @@ module Workflow
 
   def dep(*dependency, &block)
     @dependencies ||= []
+    dependency = [tasks.keys.last] if dependency.empty? && ! block_given?
     if block_given?
       if dependency.any?
 
@@ -129,7 +130,7 @@ module Workflow
       :resumable          => consume_resumable,
       :input_options      => consume_input_options
     }
-
+     
     task_info[:extension] = case task_info[:result_type].to_s
                             when "tsv"
                               "tsv"

data/lib/rbbt/workflow/integration/ansible.rb

@@ -0,0 +1,53 @@
+require_relative 'ansible/workflow'
+require 'rbbt/workflow/usage'
+
+module Ansible
+  def self.play(playbook, inventory = nil)
+    inventory = Rbbt.etc.ansible_inventory.find
+    Log.with_severity 0 do
+      TmpFile.with_file do |tmp|
+        if Hash === playbook
+          Open.write(tmp, [playbook].to_yaml)
+          playbook = tmp
+        end
+        CMD.cmd_log("ansible-playbook -i #{inventory} #{playbook}")
+      end
+    end
+  end
+
+  def self.clean_symbols(hash)
+    new = {}
+    hash.each do |key,value|
+      key = key.to_s
+      value = case value
+              when Symbol
+                value.to_s
+              when Hash
+                self.clean_symbols(value)
+              else
+                value
+              end
+      new[key] = value
+    end
+    new
+  end
+
+  def self.workflow2playbook(workflow, task, options = {})
+    job_options = workflow.get_SOPT(workflow.tasks[task])
+
+    tasks = workflow.job(task, nil, job_options).exec
+
+    hosts = options[:hosts] || 'localhost'
+
+    clean_tasks = tasks.collect{|task| self.clean_symbols task }
+    {"hosts" => hosts, "tasks" => clean_tasks}
+  end
+
+  def self.playbook(file, task = nil, options = {})
+    task = 'default' if task.nil?
+
+    workflow = Workflow.require_workflow file
+    task = workflow.tasks.keys.last if workflow.tasks[task].nil?
+    workflow2playbook workflow, task, options
+  end
+end

data/lib/rbbt/workflow/integration/ansible/workflow.rb

@@ -0,0 +1,60 @@
+require 'rbbt/workflow'
+
+module Ansible
+  module AnsibleWorkflow
+    def self.extended(object)
+      class << object
+        attr_accessor :ans_tasks
+      end
+
+      object.helper :register do |task_info|
+        desc = task.description if task
+        name ||= desc || short_path
+        task_info = {"name" => name}.merge(task_info)
+        @ans_tasks ||= []
+        @ans_tasks << task_info
+        task
+      end
+
+      object.helper :ans do |name, info|
+        register({ name => info})
+      end
+
+      object.helper :add do |name, info|
+        @ans_tasks.last[name.to_s] = info
+      end
+
+      object.helper :shell do |cmd|
+        register({"shell" => cmd.strip})
+      end
+
+      object.helper :sudo do |cmd|
+        register({"shell" => cmd.strip, "become" => 'yes'})
+      end
+
+      object.helper :singularity do |scmd|
+        img = config :singularity_img, :build, :test, :small, :default => '/data/img/singularity/rbbt/rbbt.simg'
+        container = config :singularity_container, :build, :test, :small, :default => '/data/img/sandbox/mvazque2/'
+        cmd = <<-EOF
+singularity exec -C -H '#{container}' '#{img}' #{scmd}
+        EOF
+        register({"shell" => cmd.strip, "name" => short_path})
+      end
+
+
+      object.helper :produce_task do
+        @ans_tasks
+      end
+    end
+
+    def play(name = nil, &block)
+      name = Misc.snake_case(@description) if name.nil?
+      task name => :yaml do |*args|
+        self.instance_exec *args, &block
+        dependencies.inject([]){|acc,dep| acc += dep.load } + produce_task
+      end
+    end
+
+  end
+end
+

data/lib/rbbt/workflow/step.rb

@@ -154,6 +154,11 @@ class Step
     all_inputs
   end
 
+  def dependencies=(dependencies)
+    @dependencies = dependencies
+    set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
+  end
+
   def recursive_inputs
     if NamedArray === inputs
       i = {}

data/lib/rbbt/workflow/step/accessor.rb

@@ -121,7 +121,7 @@ class Step
       inputs[name] = value
     end
 
-    if options.include? 'override_dependencies'
+    if options && options.include?('override_dependencies')
       inputs.merge!(:override_dependencies => open[:override_dependencies])
       input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
     end

data/lib/rbbt/workflow/util/archive.rb

@@ -67,8 +67,10 @@ class Step
       next unless File.exists?(step.path)
       job_files << step.path
       job_files << step.info_file if File.exists?(step.info_file)
+      job_files << Step.md5_file(step.path) if File.exists?(Step.md5_file step.path)
       job_file_dir_content = Dir.glob(step.files_dir + '/**/*')
       job_files += job_file_dir_content
+      job_files << step.files_dir if File.exists?(step.files_dir)
       rec_dependencies = Set.new
 
       next unless recursive

data/lib/rbbt/workflow/util/orchestrator.rb

@@ -143,18 +143,25 @@ module Workflow
       end
     end
 
-    def erase_job_dependencies(job, rules, workload, top_level_jobs)
+    def erase_job_dependencies(job, rules, all_jobs, top_level_jobs)
       job.dependencies.each do |dep|
         next if top_level_jobs.include? dep.path
         next unless Orchestrator.job_rules(rules, dep)["erase"].to_s == 'true'
 
-        list = (workload.keys - [job]).collect{|pending| pending.dependencies}.flatten
-        next if list.include?(dep)
+        dep_path = dep.path
+        parents = all_jobs.select do |parent| 
+          paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| d.last }
+          paths.include? dep_path
+        end
+
+        next unless parents.reject{|parent| parent.done? }.empty?
 
-        Log.high "Erasing #{dep.path} from #{job.path}"
-        job.archive_deps
-        job.copy_files_dir
-        job.dependencies = job.dependencies - [dep]
+        parents.each do |parent|
+          Log.high "Erasing #{dep.path} from #{parent.path}"
+          parent.archive_deps
+          parent.copy_files_dir
+          parent.dependencies = parent.dependencies - [dep]
+        end
         dep.clean
       end
     end
@@ -162,7 +169,13 @@ module Workflow
     def process(rules, jobs)
       begin
 
-        workload = jobs.inject({}){|acc,job| acc.merge!(Orchestrator.job_workload(job)) }
+        workload = jobs.inject({}) do |acc,job| 
+          Orchestrator.job_workload(job).each do |j,d|
+            acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path
+          end
+          acc
+        end
+        all_jobs = workload.keys
 
         top_level_jobs = jobs.collect{|job| job.path }
         while workload.any? 
@@ -187,7 +200,7 @@ module Workflow
             when job.done?
               Log.debug "Orchestrator done #{job.path}"
               release_resources(job)
-              erase_job_dependencies(job, rules, workload, top_level_jobs)
+              erase_job_dependencies(job, rules, all_jobs, top_level_jobs)
 
             when job.running?
               next