miga-base 0.7.26.0 → 0.7.26.1

data/utils/enveomics/Scripts/lib/enveomics_rb/stat.rb

@@ -0,0 +1,30 @@
+
+module Enve
+  class Stat
+    # Generates a random number from the +dist+ distribution with +params+
+    # parameters. This is simply a wrapper to the r_* functions below.
+    def self.rand(dist=:unif, *params)
+      send("r_#{dist}", *params)
+    end
+
+    # Generates a random number from the uniform distribution between +min+ and
+    # +max+. By default generates random numbers between 0.0 and 1.0.
+    def self.r_unif(min=0.0, max=1.0)
+      min + (max-min)*Random::rand
+    end
+    
+    # Generates a random number from the geometric distribution with support
+    # {0, 1, 2, ...} and probability of success +p+.
+    def self.r_geom(p)
+      (Math::log(1.0 - rand)/Math::log(1.0-p) - 1.0).ceil
+    end
+
+    # Generates a random number from the shifted geometric distribution with
+    # support {1, 2, 3, ...} and probability of success +p+.
+    def self.r_sgeom(p)
+      (Math::log(1.0 - rand)/Math::log(1.0-p)).ceil
+    end
+    
+  end
+end
+

data/utils/enveomics/Scripts/lib/enveomics_rb/vcf.rb

@@ -0,0 +1,135 @@
+# @author  Luis M. Rodriguez-R
+# @license Artistic-2.0
+
+##
+# Enveomics representation of a Variant Call Format (VCF) file.
+class VCF
+  
+  ##
+  # File-handler, a File object.
+  attr_reader :fh
+  def initialize(file)
+    @fh = (file.is_a?(String) ? File.open(file, "r") : file )
+  end
+  
+  ##
+  # Iterate through each variant (i.e., each non-comment line), passing a
+  # VCF::Variant object to +blk+.
+  def each_variant(&blk)
+    fh.rewind
+    fh.each_line do |ln|
+      next if ln =~ /^#/
+      blk.call VCF::Variant.new(ln)
+    end
+  end
+
+  ##
+  # Iterate through each header (i.e., each comment line), passing a String to
+  # +blk+.
+  def each_header(&blk)
+    fh.rewind
+    fh.each_line do |ln|
+      next unless ln =~ /^#/
+      blk.call ln
+    end
+  end
+end
+
+class VCF::Variant
+  
+  ##
+  # Column definitions in VCF.
+  @@COLUMNS = [:chrom,:pos,:id,:ref,:alt,:qual,:filter,:info,:format,:bam]
+  
+  ##
+  # An Array of String, containing each of the VCF entrie's columns.
+  attr_reader :data
+
+  ##
+  # Initialize VCF::Variant from String +line+, a non-comment line in the VCF.
+  def initialize(line)
+    @data = line.chomp.split("\t")
+    # Qual as float
+    @data[5] = data[5].to_f
+    # Split info
+    info = data[7].split(";").map{ |i| i=~/=/ ? i.split("=", 2) : [i, true] }
+    @data[7] = Hash[*info.map{ |i| [i[0].to_sym, i[1]] }.flatten]
+    # Read formatted data
+    unless data[9].nil? or data[9].empty?
+      f = format.split(":")
+      b = bam.split(":")
+      f.each_index{ |i| @data[7][f[i].to_sym] = b[i] }
+    end
+    @data[7][:INDEL] = true if ref.size != alt.split(",").first.size
+  end
+
+  ##
+  # Named functions for each column.
+  @@COLUMNS.each_index do |i|
+    define_method(@@COLUMNS[i]) { @@COLUMNS[i]==:pos ? data[i].to_i : data[i] }
+  end
+
+  ##
+  # Sequencing depth.
+  def dp
+    return nil if info[:DP].nil?
+    info[:DP].to_i
+  end
+
+  ##
+  # Sequencing depth of FWD-REF, REV-REF, FWD-ALT, and REV-ALT.
+  def dp4
+    return nil if info[:DP4].nil?
+    @dp4 ||= info[:DP4].split(",").map{ |i| i.to_i }
+    @dp4
+  end
+
+  ##
+  ## Sequencing depth of REF and ALT.
+  def ad
+    return nil if info[:AD].nil?
+    @ad ||= info[:AD].split(",").map{ |i| i.to_i }
+    @ad
+  end
+  
+  ##
+  # Sequencing depth of the REF allele.
+  def ref_dp
+    return dp4[0] + dp4[1] unless dp4.nil?
+    return ad[0] unless ad.nil?
+    nil
+  end
+
+  ##
+  # Sequencing depth of the ALT allele.
+  def alt_dp
+    return dp4[2] + dp4[3] unless dp4.nil?
+    return ad[1] unless ad.nil?
+    nil
+  end
+
+  ##
+  # Information content of the variant in bits (from 0 to 1).
+  def shannon
+    return @shannon unless @shannon.nil?
+    a = ref_dp
+    b = alt_dp
+    ap = a.to_f/(a+b)
+    bp = b.to_f/(a+b)
+    @shannon = -(ap*Math.log(ap,2) + bp*Math.log(bp,2))
+    @shannon
+  end
+
+  ##
+  # Is it an indel?
+  def indel? ; !info[:INDEL].nil? and info[:INDEL] ; end
+
+  ##
+  # Return as String.
+  def to_s ; (data[0..6] + [info_to_s] + data[8..-1].to_a).join("\t") + "\n" ; end
+
+  ##
+  # Returns the INFO entry as String.
+  def info_to_s ; data[7].to_a.map{ |i| i.join("=") }.join(";") ; end
+  
+end

data/utils/enveomics/Scripts/ogs.annotate.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'enveomics_rb/og'
+require 'optparse'
+
+o = {:q=>FALSE, :f=>"(\\S+)\\.txt", :consolidate=>TRUE, :pre=>[]}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Annotates Orthology Groups (OGs) using one or more reference genomes.
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE", "Input file containing the OGs (as generated by ogs.rb)."){ |v| o[:in]=v }
+   opts.on("-o", "--out FILE", "Output file containing the annotated OGs."){ |v| o[:out]=v }
+   opts.on("-a FILE1,FILE2,...", Array, "Input file(s) containing the annotations.One or more tab-delimited files",
+   		"with the gene names in the first column and the annotation in the second."){ |v| o[:annotations]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-f","--format STRING", "Format of the filenames for the annotation files, using regex syntax.",
+   		"By default: '#{o[:f]}'."){ |v| o[:f]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+abort "-a is mandatory" if o[:annotations].nil?
+
+##### MAIN:
+begin
+   # Read the pre-computed OGs
+   collection = OGCollection.new
+   $stderr.puts "Reading pre-computed OGs in '#{o[:in]}'." unless o[:q]
+   f = File.open(o[:in], 'r')
+   h = f.gets.chomp.split /\t/
+   while ln = f.gets
+      collection << OG.new(h, ln.chomp.split(/\t/))
+   end
+   f.close
+   $stderr.puts " Loaded OGs: #{collection.ogs.length}." unless o[:q]
+
+   # Read annotations
+   o[:annotations].each do |annot|
+      m = /#{o[:f]}/.match(annot)
+      if m.nil? or m[1].nil?
+	 warn "Cannot parse filename: #{rbm_file} (doesn't match /#{o[:f]}/)."
+	 next
+      end
+      f = File.open(annot, 'r')
+      no_og = 0
+      collection.add_note_src m[1]+' annotation'
+      while ln=f.gets
+	 r = ln.chomp.split /\t/
+	 g = Gene.new m[1], r[0]
+	 og = collection.get_og g
+	 if og.nil?
+	    no_og += 1
+	 else
+	    og.add_note g.id + ': ' + r[1], collection.note_srcs.length-1
+	 end
+      end
+      warn "Warning: Cannot find #{no_og} genes from #{m[1]} in OG collection." if no_og>0
+   end 
+   # Save the output matrix
+   $stderr.puts "Saving annotated OGs into '#{o[:out]}'." unless o[:q]
+   f = File.open(o[:out], "w")
+   f.puts collection.to_s
+   f.close
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.core-pan.rb

@@ -0,0 +1,160 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @license: artistic-2.0
+#
+
+$:.push File.expand_path("../lib", __FILE__)
+require "optparse"
+require "json"
+require "tmpdir"
+
+o = {q:false, n:100, thr:2}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+  opts.banner = "
+Subsamples the genomes in a set of Orthology Groups (OGs) and estimates the
+trend of core genome and pangenome sizes.
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Mandatory"
+  opts.on("-o", "--ogs FILE",
+    "Input file containing the precomputed OGs."){ |v| o[:ogs]=v }
+  opts.separator ""
+  opts.separator "Output Options"
+  opts.on("-s", "--summary FILE",
+    "Output file in tabular format with summary statistics."){ |v| o[:summ]=v }
+  opts.on("-t", "--tab FILE","Output file in tabular format."){ |v| o[:tab]=v }
+  opts.on("-j", "--json FILE", "Output file in JSON format."){ |v| o[:json]=v }
+  opts.separator ""
+  opts.separator "Other Options"
+  opts.on("-n", "--replicates INT",
+    "Number of replicates to estimate. By default: #{o[:n]}."
+    ){ |v| o[:n]=v.to_i }
+  opts.on("--threads INT",
+    "Children threads to spawn. By default: #{o[:thr]}."){ |v| o[:thr]=v.to_i}
+  opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:ogs].nil?
+
+##### MAIN:
+begin
+  # Read the pre-computed OGs
+  $stderr.puts "Reading pre-computed OGs in '#{o[:ogs]}'." unless o[:q]
+  bool_a = []
+  genomes_n = nil
+  File.open(o[:ogs], "r") do |f|
+    h = f.gets.chomp.split "\t"
+    genomes_n = h.size
+    while ln = f.gets
+      bool_a << ln.chomp.split("\t").map{ |g| g!="-" }
+    end
+  end
+  $stderr.puts " Loaded OGs: #{bool_a.size}." unless o[:q]
+  bool_a_b = bool_a.map{ |og| og.map{ |g| g ? "1" : "0" }.join("").to_i(2) }
+
+  # Generate subsamples
+  size = {core:[], pan:[]}
+  Dir.mktmpdir do |dir|
+    children = 0
+    (0 .. o[:n]-1).each do |i|
+      fork do
+        # Generate trajectory
+        genomes = (0 .. genomes_n-1).to_a.shuffle
+        genomes_b = (2 ** genomes_n) - 1
+        core = []
+        pan = []
+        while not genomes.empty?
+          core.unshift 0
+          pan.unshift 0
+          bool_a_b.map! do |og|
+            r_og = og & genomes_b
+            if r_og>0
+              core[0] += 1 if r_og==genomes_b
+              pan[0]  += 1
+              og
+            else
+              nil
+            end
+          end
+          bool_a_b.compact!
+          genomes_b ^= 2 ** genomes.pop
+        end
+        abort "UNEXPECTED ERROR: Final genomes_b=#{genomes_b}." if genomes_b>0
+        # Store trajectory
+        File.open("#{dir}/#{i}", "w") do |tfh|
+          tfh.puts JSON.generate({core:core, pan:pan})
+        end
+      end # fork
+      children += 1
+      if children >= o[:thr]
+        Process.wait
+        children -= 1
+      end
+    end
+    Process.waitall
+    # Recover trajectories
+    (0 .. o[:n]-1).each do |i|
+      s = JSON.parse(File.read("#{dir}/#{i}"), {:symbolize_names=>true})
+      size[:core][i] = s[:core]
+      size[:pan][i] = s[:pan]
+    end
+  end # Dir.mktmpdir
+
+  # Show result
+  $stderr.puts "Generating reports." unless o[:q]
+
+  # Save results in JSON
+  unless o[:json].nil?
+    ofh = File.open(o[:json], "w")
+    ofh.puts JSON.pretty_generate(size)
+    ofh.close
+  end
+
+  # Save results in tab
+  unless o[:tab].nil?
+    ofh = File.open(o[:tab], "w")
+    ofh.puts (%w{replicate metric}+(1 .. genomes_n).to_a).join("\t")
+    (0 .. o[:n]-1).each do |i|
+      ofh.puts ([i+1,"core"] + size[:core][i]).join("\t")
+      ofh.puts ([i+1,"pan"] + size[:pan][i]).join("\t")
+    end
+    ofh.close
+  end
+
+  # Save summary results in tab
+  unless o[:summ].nil?
+    ofh = File.open(o[:summ], "w")
+    ofh.puts %w{genomes core_avg core_sd core_q1 core_q2 core_q3
+      pan_avg pan_sd pan_q1 pan_q2 pan_q3}.join("\t")
+    (0 .. genomes_n-1).each do |i|
+      res = [ i+1 ]
+      [:core, :pan].each do |met|
+        a = size[met].map{ |r| r[i] }.sort
+        avg = a.inject(0,:+).to_f / a.size
+        var = a.map{ |v| v**2 }.inject(0,:+).to_f/a.size - avg**2
+        sd = Math.sqrt(var)
+        q1 = a[ a.size*1/4 ]
+        q2 = a[ a.size*2/4 ]
+        q3 = a[ a.size*3/4 ]
+        res += [avg,sd,q1,q2,q3]
+      end
+      ofh.puts res.join("\t")
+    end
+    ofh.close
+  end
+
+  $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+

data/utils/enveomics/Scripts/ogs.extract.rb

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+#
+# @author Luis M. Rodriguez-R
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path("lib", File.dirname(__FILE__))
+require "enveomics_rb/enveomics"
+require "enveomics_rb/og"
+
+o = {q:false, pergenome:false, prefix:false, first:false, rand:false,
+   core:0.0, dups:0}
+OptionParser.new do |opts|
+   opts.banner = "
+Extracts sequences of Orthology Groups (OGs) from genomes (proteomes).
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE",
+      "Input file containing the OGs (as generated by ogs.rb)."){ |v| o[:in]=v }
+   opts.on("-o", "--out FILE",
+      "Output directory where to place extracted sequences."){ |v| o[:out]=v }
+   opts.on("-s", "--seqs STRING",
+      "Path to the proteomes in FastA format, using '%s' to denote the genome.",
+      "For example: /path/to/seqs/%s.faa."){ |v| o[:seqs]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-c", "--core FLOAT",
+      "Use only OGs present in at least this fraction of the genomes.",
+      "To use only the strict core genome*, use -c 1."){ |v| o[:core]=v.to_f }
+   opts.on("-d", "--duplicates INT",
+      "Use only OGs with less than this number of in-paralogs in a genome.",
+      "To use only genes without in-paralogs*, use -d 1."
+      ){ |v| o[:dups]=v.to_i }
+   opts.on("-g", "--per-genome",
+      "If set, the output is generated per genome.",
+      "By default, the output is per OG."){ |v| o[:pergenome]=v }
+   opts.on("-p", "--prefix",
+      "If set, each sequence is prefixed with the genome name",
+      "(or OG number, if --per-genome) and a dash."){ |v| o[:prefix]=v }
+   opts.on("-r", "--rand",
+      "Get only one gene per genome per OG (random) regardless of in-paralogs.",
+      "By default all genes are extracted."){ |v| o[:rand]=v }
+   opts.on("-f", "--first",
+      "Get only one gene per genome per OG (first) regardless of in-paralogs.",
+      "By default all genes are extracted. Takes precendece over --rand."
+      ){ |v| o[:first]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+   opts.separator "    * To use only the unus genome (OGs with exactly one " +
+      "gene per genome), use: -c 1 -d 1."
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+abort "-s is mandatory" if o[:seqs].nil?
+
+##### MAIN:
+begin
+   # Read the pre-computed OGs
+   collection = OGCollection.new
+   $stderr.puts "Reading pre-computed OGs in '#{o[:in]}'." unless o[:q]
+   f = File.open(o[:in], "r")
+   h = f.gets.chomp.split /\t/
+   while ln = f.gets
+      collection << OG.new(h, ln.chomp.split(/\t/))
+   end
+   f.close
+   $stderr.puts " Loaded OGs: #{collection.ogs.size}." unless o[:q]
+   $stderr.puts " Reported Genomes: #{Gene.genomes.size}." unless o[:q]
+
+   # Filter core/in-paralog genes
+   collection.filter_core! o[:core] unless o[:core]==0.0
+   collection.remove_inparalogs! o[:dups] unless o[:dups]==0
+   $stderr.puts " Filtered OGs: #{collection.ogs.size}." unless
+      o[:q] or o[:core]==0.0
+
+   # Open outputs
+   $stderr.puts "Initializing output files." unless o[:q]
+   Dir.mkdir(o[:out]) unless Dir.exist? o[:out]
+   ofhs = o[:pergenome] ?
+      Gene.genomes.map{|g| File.open("#{o[:out]}/#{g}.fa", "w")} :
+      ( (1 .. collection.ogs.size).map do |og|
+	 File.open("#{o[:out]}/OG#{og}.fa", "w")
+      end )
+   $stderr.puts " Created files: #{ofhs.size}." unless o[:q]
+   
+   # Read genomes
+   $stderr.puts "Filtering genes." unless o[:q]
+   genome_i = -1
+   Gene.genomes.each do |genome|
+      genome_i = Gene.genomes.index(genome)
+      $stderr.print "  Genome #{genome_i+1}.   \r" unless o[:q]
+      genes = ( collection.get_genome_genes(genome).map do |og|
+	    o[:first] ? [og.first] : (o[:rand] ? [og.sample] : og)
+	 end )
+      hand = nil
+      File.open(sprintf(o[:seqs], genome), "r").each do |ln|
+	 if ln =~ /^>(\S+)/
+	    og = genes.index{|g| g.include? $1}
+	    hand = og.nil? ? nil : ( o[:pergenome] ? genome_i : og )
+	    ln.sub!(/^>/, ">#{o[:pergenome] ? "OG#{og}" : genome}-") if
+	       o[:prefix] and not hand.nil?
+	 end
+	 ofhs[hand].puts(ln) unless hand.nil?
+      end
+   end
+   $stderr.puts "  #{genome_i+1} genomes processed." unless o[:q]
+
+   # Close outputs
+   $stderr.puts "Closing output files." unless o[:q]
+   ofhs.each{|h| h.close}
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+