miga-base 0.7.26.0 → 1.0.0.1

data/utils/enveomics/Scripts/ogs.extract.rb

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+#
+# @author Luis M. Rodriguez-R
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path("lib", File.dirname(__FILE__))
+require "enveomics_rb/enveomics"
+require "enveomics_rb/og"
+
+o = {q:false, pergenome:false, prefix:false, first:false, rand:false,
+   core:0.0, dups:0}
+OptionParser.new do |opts|
+   opts.banner = "
+Extracts sequences of Orthology Groups (OGs) from genomes (proteomes).
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE",
+      "Input file containing the OGs (as generated by ogs.rb)."){ |v| o[:in]=v }
+   opts.on("-o", "--out FILE",
+      "Output directory where to place extracted sequences."){ |v| o[:out]=v }
+   opts.on("-s", "--seqs STRING",
+      "Path to the proteomes in FastA format, using '%s' to denote the genome.",
+      "For example: /path/to/seqs/%s.faa."){ |v| o[:seqs]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-c", "--core FLOAT",
+      "Use only OGs present in at least this fraction of the genomes.",
+      "To use only the strict core genome*, use -c 1."){ |v| o[:core]=v.to_f }
+   opts.on("-d", "--duplicates INT",
+      "Use only OGs with less than this number of in-paralogs in a genome.",
+      "To use only genes without in-paralogs*, use -d 1."
+      ){ |v| o[:dups]=v.to_i }
+   opts.on("-g", "--per-genome",
+      "If set, the output is generated per genome.",
+      "By default, the output is per OG."){ |v| o[:pergenome]=v }
+   opts.on("-p", "--prefix",
+      "If set, each sequence is prefixed with the genome name",
+      "(or OG number, if --per-genome) and a dash."){ |v| o[:prefix]=v }
+   opts.on("-r", "--rand",
+      "Get only one gene per genome per OG (random) regardless of in-paralogs.",
+      "By default all genes are extracted."){ |v| o[:rand]=v }
+   opts.on("-f", "--first",
+      "Get only one gene per genome per OG (first) regardless of in-paralogs.",
+      "By default all genes are extracted. Takes precendece over --rand."
+      ){ |v| o[:first]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+   opts.separator "    * To use only the unus genome (OGs with exactly one " +
+      "gene per genome), use: -c 1 -d 1."
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+abort "-s is mandatory" if o[:seqs].nil?
+
+##### MAIN:
+begin
+   # Read the pre-computed OGs
+   collection = OGCollection.new
+   $stderr.puts "Reading pre-computed OGs in '#{o[:in]}'." unless o[:q]
+   f = File.open(o[:in], "r")
+   h = f.gets.chomp.split /\t/
+   while ln = f.gets
+      collection << OG.new(h, ln.chomp.split(/\t/))
+   end
+   f.close
+   $stderr.puts " Loaded OGs: #{collection.ogs.size}." unless o[:q]
+   $stderr.puts " Reported Genomes: #{Gene.genomes.size}." unless o[:q]
+
+   # Filter core/in-paralog genes
+   collection.filter_core! o[:core] unless o[:core]==0.0
+   collection.remove_inparalogs! o[:dups] unless o[:dups]==0
+   $stderr.puts " Filtered OGs: #{collection.ogs.size}." unless
+      o[:q] or o[:core]==0.0
+
+   # Open outputs
+   $stderr.puts "Initializing output files." unless o[:q]
+   Dir.mkdir(o[:out]) unless Dir.exist? o[:out]
+   ofhs = o[:pergenome] ?
+      Gene.genomes.map{|g| File.open("#{o[:out]}/#{g}.fa", "w")} :
+      ( (1 .. collection.ogs.size).map do |og|
+	 File.open("#{o[:out]}/OG#{og}.fa", "w")
+      end )
+   $stderr.puts " Created files: #{ofhs.size}." unless o[:q]
+   
+   # Read genomes
+   $stderr.puts "Filtering genes." unless o[:q]
+   genome_i = -1
+   Gene.genomes.each do |genome|
+      genome_i = Gene.genomes.index(genome)
+      $stderr.print "  Genome #{genome_i+1}.   \r" unless o[:q]
+      genes = ( collection.get_genome_genes(genome).map do |og|
+	    o[:first] ? [og.first] : (o[:rand] ? [og.sample] : og)
+	 end )
+      hand = nil
+      File.open(sprintf(o[:seqs], genome), "r").each do |ln|
+	 if ln =~ /^>(\S+)/
+	    og = genes.index{|g| g.include? $1}
+	    hand = og.nil? ? nil : ( o[:pergenome] ? genome_i : og )
+	    ln.sub!(/^>/, ">#{o[:pergenome] ? "OG#{og}" : genome}-") if
+	       o[:prefix] and not hand.nil?
+	 end
+	 ofhs[hand].puts(ln) unless hand.nil?
+      end
+   end
+   $stderr.puts "  #{genome_i+1} genomes processed." unless o[:q]
+
+   # Close outputs
+   $stderr.puts "Closing output files." unless o[:q]
+   ofhs.each{|h| h.close}
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.mcl.rb

@@ -0,0 +1,186 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Sep-11-2015
+# @license: artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require 'enveomics_rb/og'
+require 'optparse'
+require 'tmpdir'
+
+o = {q:false, f:"(\\S+)-(\\S+)\\.rbm", mcl:"", inflation:1.5, blind:false,
+   evalue:false, thr:2, identity:false, bestmatch:false}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Identifies Orthology Groups (OGs) in Reciprocal Best Matches (RBM)
+between all pairs in a collection of genomes, using the Markov Cluster
+Algorithm.
+
+Requires MCL (see http://www.micans.org/mcl).
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-o", "--out FILE",
+      "Output file containing the detected OGs."){ |v| o[:out]=v }
+   opts.on("-d", "--dir DIR",
+      "Directory containing the RBM files.",
+      "Becomes optional iff --abc is set to a non-empty file."){ |v| o[:dir]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-f", "--format STRING",
+      "Format of the filenames for the RBM files (within -d), using regex " +
+      "syntax.", "By default: '#{o[:f]}'."){ |v| o[:f]=v }
+   opts.on("-I", "--inflation FLOAT",
+      "Inflation parameter for MCL clustering. By default: #{o[:inflation]}."
+      ){ |v| o[:inflation]=v.to_f }
+   opts.on("-b", "--blind",
+      "If set, computes clusters without taking bitscore into account."
+      ){ |v| o[:blind]=v }
+   opts.on("-e", "--evalue",
+      "If set, uses the e-value to weight edges, instead of the default " +
+      "Bit-Score."){ |v| o[:evalue]=v }
+   opts.on("-i", "--identity",
+      "If set, uses the identity to weight edges, instead of the default " +
+      "Bit-Score."){ |v| o[:identity]=v }
+   opts.on("-B", "--best-match",
+      "If set, it assumes best-matches instead reciprocal best matches."
+      ){ |v| o[:bestmatch]=v }
+   opts.on("-m", "--mcl-bin DIR",
+      "Path to the directory containing the mcl binaries.",
+      "By default, assumed to be in the PATH."){ |v| o[:mcl]=v+"/" }
+   opts.on("--abc FILE",
+      "Use this abc file instead of a temporal file."){ |v| o[:abc] = v }
+   opts.on("-t", "--threads INT",
+      "Number of threads to use. By default: #{o[:thr]}."){ |v| o[:thr]=v.to_i }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:out].nil?
+o[:evalue] = false if o[:identity]
+o[:evalue] = false if o[:blind]
+o[:identity] = false if o[:blind]
+
+##### MAIN:
+begin
+   Dir.mktmpdir do |dir|
+      o[:abc] = "#{dir}/rbms.abc" if o[:abc].nil?
+      abort "-d must exist and be a directory" unless
+	 File.size?(o[:abc]) or
+	 (!o[:dir].nil? and File.exists?(o[:dir]) and File.directory?(o[:dir]))
+      # Traverse the whole directory
+      if File.size? o[:abc]
+	 $stderr.puts "Reusing existing abc file '#{o[:abc]}'." unless o[:q]
+      else
+	 file_i = 0
+	 ln_i = 0
+	 $stderr.puts "Reading RBM files within '#{o[:dir]}'." unless o[:q]
+	 abc = File.open(o[:abc] + ".tmp", "w")
+	 Dir.entries(o[:dir]).each do |rbm_file|
+	    next unless File.file?(o[:dir]+"/"+rbm_file)
+	    # Parse the filename to identify the genomes
+	    m = /#{o[:f]}/.match(rbm_file)
+	    if m.nil? or m[2].nil?
+	       warn "Ignoring #{rbm_file}: doesn't match /#{o[:f]}/."
+	       next
+	    end
+	    file_i += 1
+	    # Read the RBMs list
+	    f = File.open(o[:dir]+"/"+rbm_file, "r")
+	    while ln = f.gets
+	       # Add the RBM to the abc file
+	       row = ln.split(/\t/)
+	       abc.puts [m[1]+">"+row[0], m[2]+">"+row[1],
+		  (o[:blind] ? "1" :
+		  (o[:evalue] ? row[10] :
+		  (o[:identity] ? row[2] : row[11])))].join("\t")
+	       ln_i += 1
+	    end
+	    f.close
+	    $stderr.print " Scanned files: #{file_i}. " +
+	       "Found RBMs: #{ln_i}.   \r" unless o[:q]
+	 end
+	 abc.close
+	 File.rename(o[:abc] + ".tmp", o[:abc])
+	 $stderr.print "\n" unless o[:q]
+      end # if File.size? o[:abc] ... else
+
+      # Build .mci file (mcxload) and compute .mccl file (mcl)
+      $stderr.puts "Markov-Clustering" unless o[:q]
+      `'#{o[:mcl]}mcxload' #{"--stream-mirror" unless o[:bestmatch]} \
+	 -abc '#{o[:abc]}' -o '#{dir}/rbms.mci' --write-binary \
+	 -write-tab '#{dir}/genes.tab' #{"--stream-neg-log10" if o[:evalue]} \
+	 &>/dev/null`
+      `'#{o[:mcl]}mcl' '#{dir}/rbms.mci' -V all -I #{o[:inflation].to_s} \
+	 -o '#{dir}/ogs.mcl' -te #{o[:thr].to_s}`
+
+      # Load .tab as Gene objects
+      $stderr.puts "Loading gene table from '#{dir}/genes.tab'." unless o[:q]
+      genes = []
+      tab = File.open("#{dir}/genes.tab", "r")
+      while ln = tab.gets
+	 ln.chomp!
+	 r = ln.split /\t|>/
+	 genes[ r[0].to_i ] = Gene.new(r[1], r[2])
+      end
+      tab.close
+      $stderr.puts " Got " + genes.size.to_s + " genes in " +
+	 Gene.genomes.size.to_s + " genomes." unless o[:q]
+      
+      # Load .mcl file as OGCollection
+      $stderr.puts "Loading clusters from '#{dir}/ogs.mcl'." unless o[:q]
+      collection = OGCollection.new
+      mcl = File.open("#{dir}/ogs.mcl", "r")
+      in_matrix = false
+      my_genes = nil
+      while ln = mcl.gets
+         ln.chomp!
+	 if ln =~ /^\(mclmatrix$/
+	    in_matrix = true
+	    next
+	 end
+	 next if ln =~ /^begin$/
+	 if in_matrix
+	    break if ln =~ /^\)$/
+	    if ln =~ /^\d+\s+/
+	       ln.sub!(/^\d+\s+/, "")
+	       my_genes = []
+	    end
+	    ln.sub!(/^\s+/, "")
+	    raise "Incomplete mcl matrix, offending line: #{$.}: #{ln}" if
+	       my_genes.nil?
+	    my_genes += ln.split(/\s/)
+	    if my_genes.last == "$"
+	       my_genes.pop
+	       og = OG.new
+	       my_genes.each{|i| og << genes[ i.to_i ]}
+	       collection << og
+	       my_genes = nil
+	    end
+	 end
+      end
+      mcl.close
+      $stderr.puts " Got #{collection.ogs.size} clusters." unless o[:q]
+      
+      # Save the output matrix
+      $stderr.puts "Saving matrix into '#{o[:out]}'." unless o[:q]
+      f = File.open(o[:out], "w")
+      f.puts collection.to_s
+      f.close
+      $stderr.puts "Done.\n" unless o[:q] 
+   end
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.rb

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Apr-29-2015
+# @license: artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'enveomics_rb/og'
+require 'optparse'
+
+o = {:q=>FALSE, :f=>"(\\S+)-(\\S+)\\.rbm", :consolidate=>TRUE, :pre=>[]}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+***IMPORTANT NOTE***
+This script suffers from chaining effect and is very sensitive to spurious connections,
+because it applies a greedy clustering algorithm. For most practical purposes, the use
+of this script is discouraged and `ogs.mcl.rb` should be preferred. [ Apr-29-2015 ]
+
+Identifies Orthology Groups (OGs) in Reciprocal Best Matches (RBM)
+between all pairs in a collection of genomes.
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-o", "--out FILE", "Output file containing the detected OGs."){ |v| o[:out]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-d", "--dir DIR", "Directory containing the RBM files."){ |v| o[:dir]=v }
+   opts.on("-p", "--pre-ogs FILE1,FILE2,...", Array, "Pre-computed OGs file(s), separated by commas."){ |v| o[:pre]=v }
+   opts.on("-n", "--unchecked", "Do not check internal redundancy in OGs."){ o[:consolidate]=FALSE }
+   opts.on("-f","--format STRING", "Format of the filenames for the RBM files (within -d), using regex syntax. By default: '#{o[:f]}'."){ |v| o[:f]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:out].nil?
+
+##### MAIN:
+begin
+   # Initialize the collection of OGs.
+   collection = OGCollection.new
+   # Read the pre-computed OGs (if -p is passed).
+   o[:pre].each do |pre|
+      $stderr.puts "Reading pre-computed OGs in '#{pre}'." unless o[:q]
+      f = File.open(pre, 'r')
+      h = f.gets.chomp.split /\t/
+      while ln = f.gets
+	 collection << OG.new(h, ln.chomp.split(/\t/))
+      end
+      f.close
+      $stderr.puts " Loaded OGs: #{collection.ogs.length}." unless o[:q]
+   end
+   # Read the RBM files in the directory (if -d is passed).
+   unless o[:dir].nil?
+      abort "-d must exist and be a directory" unless File.exists?(o[:dir]) and File.directory?(o[:dir])
+      # Traverse the whole directory.
+      file_i = 0
+      $stderr.puts "Reading RBM files within '#{o[:dir]}'." unless o[:q] 
+      Dir.entries(o[:dir]).each do |rbm_file|
+	 next unless File.file?(o[:dir]+"/"+rbm_file)
+	 # Parse the filename to identify the genomes.
+	 m = /#{o[:f]}/.match(rbm_file)
+	 if m.nil? or m[2].nil?
+	    warn "Cannot parse filename: #{rbm_file} (doesn't match /#{o[:f]}/)."
+	    next
+	 end
+	 file_i += 1
+	 # Read the RBMs list
+	 f = File.open(o[:dir]+"/"+rbm_file, "r")
+	 while ln = f.gets
+	    # Add the RBM to the collection of OGs. Only the first two columns are used.
+	    row = ln.split(/\t/)
+	    collection.add_rbm( Gene.new(m[1],row[0]), Gene.new(m[2],row[1]) )
+	 end
+	 f.close
+	 $stderr.print " Scanned files: #{file_i}. Found OGs: #{collection.ogs.length}.   \r" unless o[:q]
+      end
+      $stderr.print "\n" unless o[:q]
+   end
+   # Evaluate internal consistency merging linked OGs (unless -n is passed).
+   if o[:consolidate]
+      $stderr.puts "Evaluating internal consistency." unless o[:q] 
+      collection.consolidate!
+      $stderr.puts " Final OGs: #{collection.ogs.length}." unless o[:q]
+   end
+   # Save the output matrix
+   $stderr.puts "Saving matrix into '#{o[:out]}'." unless o[:q]
+   f = File.open(o[:out], "w")
+   f.puts collection.to_s
+   f.close
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.stats.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @license: Artistic-2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'enveomics_rb/og'
+require 'optparse'
+require 'json'
+
+o = {q:false, a:false}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+  opts.banner = "
+Estimates some descriptive statistics on a set of Orthology Groups (OGs).
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Mandatory"
+  opts.on("-o", "--ogs FILE",
+    "Input file containing the precomputed OGs."){ |v| o[:ogs]=v }
+  opts.separator ""
+  opts.separator "Other Options"
+  opts.on("-j", "--json FILE", "Output file in JSON format."){ |v| o[:json]=v }
+  opts.on("-t", "--tab FILE","Output file in tabular format."){ |v| o[:tab]=v }
+  opts.on("-T", "--transposed-tab FILE",
+    "Output file in transposed tabular format."){ |v| o[:ttab]=v }
+  opts.on("-a", "--auto", "Run completely quietly (no STDERR or STDOUT)") do
+    o[:q] = true
+    o[:a] = true
+  end
+  opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:ogs].nil?
+
+##### MAIN:
+begin
+  # Initialize the collection of OGs.
+  collection = OGCollection.new
+   
+  # Read the pre-computed OGs
+  $stderr.puts "Reading pre-computed OGs in '#{o[:ogs]}'." unless o[:q]
+  f = File.open(o[:ogs], "r")
+  h = f.gets.chomp.split /\t/
+  while ln = f.gets
+    collection << OG.new(h, ln.chomp.split(/\t/))
+  end
+  f.close
+  $stderr.puts " Loaded OGs: #{collection.ogs.length}." unless o[:q]
+   
+  # Estimate descriptive stats
+  stat_name = {
+    genomes: "Number of genomes",
+    pan: "Pangenome (OGs)",
+    core: "Core genome (OGs)",
+    core90pc: "OGs in 90% of the genomes",
+    core80pc: "OGs in 80% of the genomes",
+    unus: "Unus genome, core genome discarding paralogs (OGs)",
+    avg: "Average number of OGs in a genome",
+    avg_pan: "Average genome (OGs) / Pangenome (OGs)",
+    core_avg: "Core genome (OGs) / Average genome (OGs)",
+    core_pan: "Core genome (OGs) / Pangenome (OGs)",
+    ogs_shannon: "Entropy of the OG frequencies (bits)"
+  }
+  stats = {}
+  stats[:genomes] = Gene.genomes.length
+  stats[:pan] = collection.ogs.length
+  stats[:core] = collection.ogs.map do |og|
+    (og.genomes.length == Gene.genomes.length) ? 1 : 0
+  end.inject(0,:+)
+  stats[:core90pc] = collection.ogs.map do |og|
+    (og.genomes.length >= 0.9*Gene.genomes.length) ? 1 : 0
+  end.inject(0,:+)
+  stats[:core80pc] = collection.ogs.map do |og|
+    (og.genomes.length >= 0.8*Gene.genomes.length) ? 1 : 0
+  end.inject(0,:+)
+  stats[:unus] = collection.ogs.map do |og|
+    (og.genomes.length != Gene.genomes.length) ? 0 :
+      (og.genes.all?{ |i| i.size==1 }) ? 1 : 0
+  end.inject(0,:+)
+  og_genomes = collection.ogs.map{ |og| og.genomes.length }.inject(0,:+)
+  stats[:avg] = og_genomes.to_f/Gene.genomes.length
+  stats[:avg_pan] = stats[:avg]/stats[:pan]
+  stats[:core_avg] = stats[:core].to_f/stats[:avg]
+  stats[:core_pan] = stats[:core].to_f/stats[:pan]
+  stats[:ogs_shannon] = -1 * collection.ogs.map do |og|
+    pi = og.genomes.length.to_f/Gene.genomes.length
+    pi * Math.log(pi)
+  end.inject(0.0,:+)
+
+  # Show result
+  $stderr.puts "Generating reports." unless o[:q]
+  stats.each_pair{ |k,v| puts " #{stat_name[k]}: #{v}" } unless o[:a]
+
+  # Save results in JSON
+  unless o[:json].nil?
+    ohf = File.open(o[:json], "w")
+    ohf.puts JSON.pretty_generate(stats)
+    ohf.close
+  end
+
+  # Save results in tab
+  unless o[:tab].nil?
+    ohf = File.open(o[:tab], "w")
+    stats.each_pair{ |k,v| ohf.puts "#{k}\t#{v}" }
+    ohf.close
+  end
+
+  # Save results in T(tab)
+  unless o[:ttab].nil?
+    ohf = File.open(o[:ttab], "w")
+    ohf.puts stats.keys.join("\t")
+    ohf.puts stats.values.join("\t")
+    ohf.close
+  end
+
+  $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+
+