miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/lib/enveomics_rb/utils.rb

@@ -0,0 +1,73 @@
+
+require 'enveomics_rb/errors'
+require 'zlib'
+
+def use(gems, mandatory = true)
+  gems = [gems] unless gems.is_a? Array
+  begin
+    require 'rubygems'
+    while !gems.empty?
+      require gems.shift
+    end
+    return true
+  rescue LoadError
+    abort "\nUnmet requirements, please install required gems:" +
+      gems.map{ |gem| "\n   gem install #{gem}" }.join + "\n\n" if mandatory
+    return false
+  end
+end
+
+def say(*msg)
+  return if $QUIET ||= false
+
+  o = '[%s] %s' % [Time.now, msg.join('')]
+  $stderr.puts(o)
+end
+
+##
+# Returns an open reading file handler for the file,
+# supporting .gz and '-' for STDIN
+def reader(file)
+  file == '-' ? $stdin :
+    file =~ /\.gz$/ ? Zlib::GzipReader.open(file) :
+    File.open(file, 'r')
+end
+
+##
+# Returns an open writing file handler for the file,
+# supporting .gz and '-' for STDOUT
+def writer(file)
+  file == '-' ? $stdout :
+    file =~ /\.gz$/ ? Zlib::GzipWriter.open(file) :
+    File.open(file, 'w')
+end
+
+##
+# Run a command +cmd+ that can be a ready-to-go string or an Array to escape
+# 
+# Supported symbol key options in Hash +opts+:
+# - wait: Boolean, should I wait for the command to complete? Default: true
+# - stdout: Path to redirect the standard output
+# - stderr: Path to redirect the standard error
+# - mergeout: Send stderr to stdout
+#
+# Return the process ID. If wait is true (default), check for the exit
+# status and throw an Enveomics::CommandError if non-zero
+def run_cmd(cmd, opts = {})
+  opts[:wait] = true if opts[:wait].nil?
+  cmd = cmd.shelljoin if cmd.is_a? Array
+  cmd += "  > #{opts[:stdout].shellescape}" if opts[:stdout]
+  cmd += " 2> #{opts[:stderr].shellescape}" if opts[:stderr]
+  cmd += ' 2>&1' if opts[:mergeout]
+  pid = spawn(cmd)
+  return pid unless opts[:wait]
+
+  Process.wait(pid)
+  unless $?.success?
+    raise Enveomics::CommandError.new(
+      "Command failed with status #{$?.exitstatus}:\n#{cmd}"
+    )
+  end
+  pid
+end
+

data/utils/enveomics/Scripts/lib/enveomics_rb/vcf.rb

@@ -0,0 +1,135 @@
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+##
+# Enveomics representation of a Variant Call Format (VCF) file.
+class VCF
+  
+  ##
+  # File-handler, a File object.
+  attr_reader :fh
+  def initialize(file)
+    @fh = (file.is_a?(String) ? File.open(file, "r") : file )
+  end
+  
+  ##
+  # Iterate through each variant (i.e., each non-comment line), passing a
+  # VCF::Variant object to +blk+.
+  def each_variant(&blk)
+    fh.rewind
+    fh.each_line do |ln|
+      next if ln =~ /^#/
+      blk.call VCF::Variant.new(ln)
+    end
+  end
+
+  ##
+  # Iterate through each header (i.e., each comment line), passing a String to
+  # +blk+.
+  def each_header(&blk)
+    fh.rewind
+    fh.each_line do |ln|
+      next unless ln =~ /^#/
+      blk.call ln
+    end
+  end
+end
+
+class VCF::Variant
+  
+  ##
+  # Column definitions in VCF.
+  @@COLUMNS = [:chrom,:pos,:id,:ref,:alt,:qual,:filter,:info,:format,:bam]
+  
+  ##
+  # An Array of String, containing each of the VCF entrie's columns.
+  attr_reader :data
+
+  ##
+  # Initialize VCF::Variant from String +line+, a non-comment line in the VCF.
+  def initialize(line)
+    @data = line.chomp.split("\t")
+    # Qual as float
+    @data[5] = data[5].to_f
+    # Split info
+    info = data[7].split(";").map{ |i| i=~/=/ ? i.split("=", 2) : [i, true] }
+    @data[7] = Hash[*info.map{ |i| [i[0].to_sym, i[1]] }.flatten]
+    # Read formatted data
+    unless data[9].nil? or data[9].empty?
+      f = format.split(":")
+      b = bam.split(":")
+      f.each_index{ |i| @data[7][f[i].to_sym] = b[i] }
+    end
+    @data[7][:INDEL] = true if ref.size != alt.split(",").first.size
+  end
+
+  ##
+  # Named functions for each column.
+  @@COLUMNS.each_index do |i|
+    define_method(@@COLUMNS[i]) { @@COLUMNS[i]==:pos ? data[i].to_i : data[i] }
+  end
+
+  ##
+  # Sequencing depth.
+  def dp
+    return nil if info[:DP].nil?
+    info[:DP].to_i
+  end
+
+  ##
+  # Sequencing depth of FWD-REF, REV-REF, FWD-ALT, and REV-ALT.
+  def dp4
+    return nil if info[:DP4].nil?
+    @dp4 ||= info[:DP4].split(",").map{ |i| i.to_i }
+    @dp4
+  end
+
+  ##
+  ## Sequencing depth of REF and ALT.
+  def ad
+    return nil if info[:AD].nil?
+    @ad ||= info[:AD].split(",").map{ |i| i.to_i }
+    @ad
+  end
+  
+  ##
+  # Sequencing depth of the REF allele.
+  def ref_dp
+    return dp4[0] + dp4[1] unless dp4.nil?
+    return ad[0] unless ad.nil?
+    nil
+  end
+
+  ##
+  # Sequencing depth of the ALT allele.
+  def alt_dp
+    return dp4[2] + dp4[3] unless dp4.nil?
+    return ad[1] unless ad.nil?
+    nil
+  end
+
+  ##
+  # Information content of the variant in bits (from 0 to 1).
+  def shannon
+    return @shannon unless @shannon.nil?
+    a = ref_dp
+    b = alt_dp
+    ap = a.to_f/(a+b)
+    bp = b.to_f/(a+b)
+    @shannon = -(ap*Math.log(ap,2) + bp*Math.log(bp,2))
+    @shannon
+  end
+
+  ##
+  # Is it an indel?
+  def indel? ; !info[:INDEL].nil? and info[:INDEL] ; end
+
+  ##
+  # Return as String.
+  def to_s ; (data[0..6] + [info_to_s] + data[8..-1].to_a).join("\t") + "\n" ; end
+
+  ##
+  # Returns the INFO entry as String.
+  def info_to_s ; data[7].to_a.map{ |i| i.join("=") }.join(";") ; end
+  
+end

data/utils/enveomics/Scripts/ogs.annotate.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'enveomics_rb/og'
+require 'optparse'
+
+o = {:q=>FALSE, :f=>"(\\S+)\\.txt", :consolidate=>TRUE, :pre=>[]}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Annotates Orthology Groups (OGs) using one or more reference genomes.
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE", "Input file containing the OGs (as generated by ogs.rb)."){ |v| o[:in]=v }
+   opts.on("-o", "--out FILE", "Output file containing the annotated OGs."){ |v| o[:out]=v }
+   opts.on("-a FILE1,FILE2,...", Array, "Input file(s) containing the annotations.One or more tab-delimited files",
+   		"with the gene names in the first column and the annotation in the second."){ |v| o[:annotations]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-f","--format STRING", "Format of the filenames for the annotation files, using regex syntax.",
+   		"By default: '#{o[:f]}'."){ |v| o[:f]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+abort "-a is mandatory" if o[:annotations].nil?
+
+##### MAIN:
+begin
+   # Read the pre-computed OGs
+   collection = OGCollection.new
+   $stderr.puts "Reading pre-computed OGs in '#{o[:in]}'." unless o[:q]
+   f = File.open(o[:in], 'r')
+   h = f.gets.chomp.split /\t/
+   while ln = f.gets
+      collection << OG.new(h, ln.chomp.split(/\t/))
+   end
+   f.close
+   $stderr.puts " Loaded OGs: #{collection.ogs.length}." unless o[:q]
+
+   # Read annotations
+   o[:annotations].each do |annot|
+      m = /#{o[:f]}/.match(annot)
+      if m.nil? or m[1].nil?
+	 warn "Cannot parse filename: #{rbm_file} (doesn't match /#{o[:f]}/)."
+	 next
+      end
+      f = File.open(annot, 'r')
+      no_og = 0
+      collection.add_note_src m[1]+' annotation'
+      while ln=f.gets
+	 r = ln.chomp.split /\t/
+	 g = Gene.new m[1], r[0]
+	 og = collection.get_og g
+	 if og.nil?
+	    no_og += 1
+	 else
+	    og.add_note g.id + ': ' + r[1], collection.note_srcs.length-1
+	 end
+      end
+      warn "Warning: Cannot find #{no_og} genes from #{m[1]} in OG collection." if no_og>0
+   end 
+   # Save the output matrix
+   $stderr.puts "Saving annotated OGs into '#{o[:out]}'." unless o[:q]
+   f = File.open(o[:out], "w")
+   f.puts collection.to_s
+   f.close
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.core-pan.rb

@@ -0,0 +1,160 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @license: artistic-2.0
+#
+
+$:.push File.expand_path("../lib", __FILE__)
+require "optparse"
+require "json"
+require "tmpdir"
+
+o = {q:false, n:100, thr:2}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+  opts.banner = "
+Subsamples the genomes in a set of Orthology Groups (OGs) and estimates the
+trend of core genome and pangenome sizes.
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Mandatory"
+  opts.on("-o", "--ogs FILE",
+    "Input file containing the precomputed OGs."){ |v| o[:ogs]=v }
+  opts.separator ""
+  opts.separator "Output Options"
+  opts.on("-s", "--summary FILE",
+    "Output file in tabular format with summary statistics."){ |v| o[:summ]=v }
+  opts.on("-t", "--tab FILE","Output file in tabular format."){ |v| o[:tab]=v }
+  opts.on("-j", "--json FILE", "Output file in JSON format."){ |v| o[:json]=v }
+  opts.separator ""
+  opts.separator "Other Options"
+  opts.on("-n", "--replicates INT",
+    "Number of replicates to estimate. By default: #{o[:n]}."
+    ){ |v| o[:n]=v.to_i }
+  opts.on("--threads INT",
+    "Children threads to spawn. By default: #{o[:thr]}."){ |v| o[:thr]=v.to_i}
+  opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:ogs].nil?
+
+##### MAIN:
+begin
+  # Read the pre-computed OGs
+  $stderr.puts "Reading pre-computed OGs in '#{o[:ogs]}'." unless o[:q]
+  bool_a = []
+  genomes_n = nil
+  File.open(o[:ogs], "r") do |f|
+    h = f.gets.chomp.split "\t"
+    genomes_n = h.size
+    while ln = f.gets
+      bool_a << ln.chomp.split("\t").map{ |g| g!="-" }
+    end
+  end
+  $stderr.puts " Loaded OGs: #{bool_a.size}." unless o[:q]
+  bool_a_b = bool_a.map{ |og| og.map{ |g| g ? "1" : "0" }.join("").to_i(2) }
+
+  # Generate subsamples
+  size = {core:[], pan:[]}
+  Dir.mktmpdir do |dir|
+    children = 0
+    (0 .. o[:n]-1).each do |i|
+      fork do
+        # Generate trajectory
+        genomes = (0 .. genomes_n-1).to_a.shuffle
+        genomes_b = (2 ** genomes_n) - 1
+        core = []
+        pan = []
+        while not genomes.empty?
+          core.unshift 0
+          pan.unshift 0
+          bool_a_b.map! do |og|
+            r_og = og & genomes_b
+            if r_og>0
+              core[0] += 1 if r_og==genomes_b
+              pan[0]  += 1
+              og
+            else
+              nil
+            end
+          end
+          bool_a_b.compact!
+          genomes_b ^= 2 ** genomes.pop
+        end
+        abort "UNEXPECTED ERROR: Final genomes_b=#{genomes_b}." if genomes_b>0
+        # Store trajectory
+        File.open("#{dir}/#{i}", "w") do |tfh|
+          tfh.puts JSON.generate({core:core, pan:pan})
+        end
+      end # fork
+      children += 1
+      if children >= o[:thr]
+        Process.wait
+        children -= 1
+      end
+    end
+    Process.waitall
+    # Recover trajectories
+    (0 .. o[:n]-1).each do |i|
+      s = JSON.parse(File.read("#{dir}/#{i}"), {:symbolize_names=>true})
+      size[:core][i] = s[:core]
+      size[:pan][i] = s[:pan]
+    end
+  end # Dir.mktmpdir
+
+  # Show result
+  $stderr.puts "Generating reports." unless o[:q]
+
+  # Save results in JSON
+  unless o[:json].nil?
+    ofh = File.open(o[:json], "w")
+    ofh.puts JSON.pretty_generate(size)
+    ofh.close
+  end
+
+  # Save results in tab
+  unless o[:tab].nil?
+    ofh = File.open(o[:tab], "w")
+    ofh.puts (%w{replicate metric}+(1 .. genomes_n).to_a).join("\t")
+    (0 .. o[:n]-1).each do |i|
+      ofh.puts ([i+1,"core"] + size[:core][i]).join("\t")
+      ofh.puts ([i+1,"pan"] + size[:pan][i]).join("\t")
+    end
+    ofh.close
+  end
+
+  # Save summary results in tab
+  unless o[:summ].nil?
+    ofh = File.open(o[:summ], "w")
+    ofh.puts %w{genomes core_avg core_sd core_q1 core_q2 core_q3
+      pan_avg pan_sd pan_q1 pan_q2 pan_q3}.join("\t")
+    (0 .. genomes_n-1).each do |i|
+      res = [ i+1 ]
+      [:core, :pan].each do |met|
+        a = size[met].map{ |r| r[i] }.sort
+        avg = a.inject(0,:+).to_f / a.size
+        var = a.map{ |v| v**2 }.inject(0,:+).to_f/a.size - avg**2
+        sd = Math.sqrt(var)
+        q1 = a[ a.size*1/4 ]
+        q2 = a[ a.size*2/4 ]
+        q3 = a[ a.size*3/4 ]
+        res += [avg,sd,q1,q2,q3]
+      end
+      ofh.puts res.join("\t")
+    end
+    ofh.close
+  end
+
+  $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+