miga-base 0.3.0.0 → 0.3.0.1

data/utils/enveomics/Scripts/ogs.annotate.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Mar-23-2015
+# @license: artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'enveomics_rb/og'
+require 'optparse'
+
+o = {:q=>FALSE, :f=>"(\\S+)\\.txt", :consolidate=>TRUE, :pre=>[]}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Annotates Orthology Groups (OGs) using one or more reference genomes.
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE", "Input file containing the OGs (as generated by ogs.rb)."){ |v| o[:in]=v }
+   opts.on("-o", "--out FILE", "Output file containing the annotated OGs."){ |v| o[:out]=v }
+   opts.on("-a FILE1,FILE2,...", Array, "Input file(s) containing the annotations.One or more tab-delimited files",
+   		"with the gene names in the first column and the annotation in the second."){ |v| o[:annotations]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-f","--format STRING", "Format of the filenames for the annotation files, using regex syntax.",
+   		"By default: '#{o[:f]}'."){ |v| o[:f]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+abort "-a is mandatory" if o[:annotations].nil?
+
+##### MAIN:
+begin
+   # Read the pre-computed OGs
+   collection = OGCollection.new
+   $stderr.puts "Reading pre-computed OGs in '#{o[:in]}'." unless o[:q]
+   f = File.open(o[:in], 'r')
+   h = f.gets.chomp.split /\t/
+   while ln = f.gets
+      collection << OG.new(h, ln.chomp.split(/\t/))
+   end
+   f.close
+   $stderr.puts " Loaded OGs: #{collection.ogs.length}." unless o[:q]
+
+   # Read annotations
+   o[:annotations].each do |annot|
+      m = /#{o[:f]}/.match(annot)
+      if m.nil? or m[1].nil?
+	 warn "Cannot parse filename: #{rbm_file} (doesn't match /#{o[:f]}/)."
+	 next
+      end
+      f = File.open(annot, 'r')
+      no_og = 0
+      collection.add_note_src m[1]+' annotation'
+      while ln=f.gets
+	 r = ln.chomp.split /\t/
+	 g = Gene.new m[1], r[0]
+	 og = collection.get_og g
+	 if og.nil?
+	    no_og += 1
+	 else
+	    og.add_note g.id + ': ' + r[1], collection.note_srcs.length-1
+	 end
+      end
+      warn "Warning: Cannot find #{no_og} genes from #{m[1]} in OG collection." if no_og>0
+   end 
+   # Save the output matrix
+   $stderr.puts "Saving annotated OGs into '#{o[:out]}'." unless o[:q]
+   f = File.open(o[:out], "w")
+   f.puts collection.to_s
+   f.close
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.core-pan.rb

@@ -0,0 +1,160 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @license: artistic-2.0
+#
+
+$:.push File.expand_path("../lib", __FILE__)
+require "optparse"
+require "json"
+require "tmpdir"
+
+o = {q:false, n:100, thr:2}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+  opts.banner = "
+Subsamples the genomes in a set of Orthology Groups (OGs) and estimates the
+trend of core genome and pangenome sizes.
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Mandatory"
+  opts.on("-o", "--ogs FILE",
+    "Input file containing the precomputed OGs."){ |v| o[:ogs]=v }
+  opts.separator ""
+  opts.separator "Output Options"
+  opts.on("-s", "--summary FILE",
+    "Output file in tabular format with summary statistics."){ |v| o[:summ]=v }
+  opts.on("-t", "--tab FILE","Output file in tabular format."){ |v| o[:tab]=v }
+  opts.on("-j", "--json FILE", "Output file in JSON format."){ |v| o[:json]=v }
+  opts.separator ""
+  opts.separator "Other Options"
+  opts.on("-n", "--replicates INT",
+    "Number of replicates to estimate. By default: #{o[:n]}."
+    ){ |v| o[:n]=v.to_i }
+  opts.on("--threads INT",
+    "Children threads to spawn. By default: #{o[:thr]}."){ |v| o[:thr]=v.to_i}
+  opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:ogs].nil?
+
+##### MAIN:
+begin
+  # Read the pre-computed OGs
+  $stderr.puts "Reading pre-computed OGs in '#{o[:ogs]}'." unless o[:q]
+  bool_a = []
+  genomes_n = nil
+  File.open(o[:ogs], "r") do |f|
+    h = f.gets.chomp.split "\t"
+    genomes_n = h.size
+    while ln = f.gets
+      bool_a << ln.chomp.split("\t").map{ |g| g!="-" }
+    end
+  end
+  $stderr.puts " Loaded OGs: #{bool_a.size}." unless o[:q]
+  bool_a_b = bool_a.map{ |og| og.map{ |g| g ? "1" : "0" }.join("").to_i(2) }
+
+  # Generate subsamples
+  size = {core:[], pan:[]}
+  Dir.mktmpdir do |dir|
+    children = 0
+    (0 .. o[:n]-1).each do |i|
+      fork do
+        # Generate trajectory
+        genomes = (0 .. genomes_n-1).to_a.shuffle
+        genomes_b = (2 ** genomes_n) - 1
+        core = []
+        pan = []
+        while not genomes.empty?
+          core.unshift 0
+          pan.unshift 0
+          bool_a_b.map! do |og|
+            r_og = og & genomes_b
+            if r_og>0
+              core[0] += 1 if r_og==genomes_b
+              pan[0]  += 1
+              og
+            else
+              nil
+            end
+          end
+          bool_a_b.compact!
+          genomes_b ^= 2 ** genomes.pop
+        end
+        abort "UNEXPECTED ERROR: Final genomes_b=#{genomes_b}." if genomes_b>0
+        # Store trajectory
+        File.open("#{dir}/#{i}", "w") do |tfh|
+          tfh.puts JSON.generate({core:core, pan:pan})
+        end
+      end # fork
+      children += 1
+      if children >= o[:thr]
+        Process.wait
+        children -= 1
+      end
+    end
+    Process.waitall
+    # Recover trajectories
+    (0 .. o[:n]-1).each do |i|
+      s = JSON.parse(File.read("#{dir}/#{i}"), {:symbolize_names=>true})
+      size[:core][i] = s[:core]
+      size[:pan][i] = s[:pan]
+    end
+  end # Dir.mktmpdir
+
+  # Show result
+  $stderr.puts "Generating reports." unless o[:q]
+
+  # Save results in JSON
+  unless o[:json].nil?
+    ofh = File.open(o[:json], "w")
+    ofh.puts JSON.pretty_generate(size)
+    ofh.close
+  end
+
+  # Save results in tab
+  unless o[:tab].nil?
+    ofh = File.open(o[:tab], "w")
+    ofh.puts (%w{replicate metric}+(1 .. genomes_n).to_a).join("\t")
+    (0 .. o[:n]-1).each do |i|
+      ofh.puts ([i+1,"core"] + size[:core][i]).join("\t")
+      ofh.puts ([i+1,"pan"] + size[:pan][i]).join("\t")
+    end
+    ofh.close
+  end
+
+  # Save summary results in tab
+  unless o[:summ].nil?
+    ofh = File.open(o[:summ], "w")
+    ofh.puts %w{genomes core_avg core_sd core_q1 core_q2 core_q3
+      pan_avg pan_sd pan_q1 pan_q2 pan_q3}.join("\t")
+    (0 .. genomes_n-1).each do |i|
+      res = [ i+1 ]
+      [:core, :pan].each do |met|
+        a = size[met].map{ |r| r[i] }.sort
+        avg = a.inject(0,:+).to_f / a.size
+        var = a.map{ |v| v**2 }.inject(0,:+).to_f/a.size - avg**2
+        sd = Math.sqrt(var)
+        q1 = a[ a.size*1/4 ]
+        q2 = a[ a.size*2/4 ]
+        q3 = a[ a.size*3/4 ]
+        res += [avg,sd,q1,q2,q3]
+      end
+      ofh.puts res.join("\t")
+    end
+    ofh.close
+  end
+
+  $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+

data/utils/enveomics/Scripts/ogs.extract.rb

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+#
+# @author Luis M. Rodriguez-R
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path("lib", File.dirname(__FILE__))
+require "enveomics_rb/enveomics"
+require "enveomics_rb/og"
+
+o = {q:false, pergenome:false, prefix:false, first:false, rand:false,
+   core:0.0, dups:0}
+OptionParser.new do |opts|
+   opts.banner = "
+Extracts sequences of Orthology Groups (OGs) from genomes (proteomes).
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-i", "--in FILE",
+      "Input file containing the OGs (as generated by ogs.rb)."){ |v| o[:in]=v }
+   opts.on("-o", "--out FILE",
+      "Output directory where to place extracted sequences."){ |v| o[:out]=v }
+   opts.on("-s", "--seqs STRING",
+      "Path to the proteomes in FastA format, using '%s' to denote the genome.",
+      "For example: /path/to/seqs/%s.faa."){ |v| o[:seqs]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-c", "--core FLOAT",
+      "Use only OGs present in at least this fraction of the genomes.",
+      "To use only the strict core genome*, use -c 1."){ |v| o[:core]=v.to_f }
+   opts.on("-d", "--duplicates INT",
+      "Use only OGs with less than this number of in-paralogs in a genome.",
+      "To use only genes without in-paralogs*, use -d 1."
+      ){ |v| o[:dups]=v.to_i }
+   opts.on("-g", "--per-genome",
+      "If set, the output is generated per genome.",
+      "By default, the output is per OG."){ |v| o[:pergenome]=v }
+   opts.on("-p", "--prefix",
+      "If set, each sequence is prefixed with the genome name",
+      "(or OG number, if --per-genome) and a dash."){ |v| o[:prefix]=v }
+   opts.on("-r", "--rand",
+      "Get only one gene per genome per OG (random) regardless of in-paralogs.",
+      "By default all genes are extracted."){ |v| o[:rand]=v }
+   opts.on("-f", "--first",
+      "Get only one gene per genome per OG (first) regardless of in-paralogs.",
+      "By default all genes are extracted. Takes precendece over --rand."
+      ){ |v| o[:first]=v }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = TRUE }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+   opts.separator "    * To use only the unus genome (OGs with exactly one " +
+      "gene per genome), use: -c 1 -d 1."
+   opts.separator ""
+end.parse!
+abort "-i is mandatory" if o[:in].nil?
+abort "-o is mandatory" if o[:out].nil?
+abort "-s is mandatory" if o[:seqs].nil?
+
+##### MAIN:
+begin
+   # Read the pre-computed OGs
+   collection = OGCollection.new
+   $stderr.puts "Reading pre-computed OGs in '#{o[:in]}'." unless o[:q]
+   f = File.open(o[:in], "r")
+   h = f.gets.chomp.split /\t/
+   while ln = f.gets
+      collection << OG.new(h, ln.chomp.split(/\t/))
+   end
+   f.close
+   $stderr.puts " Loaded OGs: #{collection.ogs.size}." unless o[:q]
+   $stderr.puts " Reported Genomes: #{Gene.genomes.size}." unless o[:q]
+
+   # Filter core/in-paralog genes
+   collection.filter_core! o[:core] unless o[:core]==0.0
+   collection.remove_inparalogs! o[:dups] unless o[:dups]==0
+   $stderr.puts " Filtered OGs: #{collection.ogs.size}." unless
+      o[:q] or o[:core]==0.0
+
+   # Open outputs
+   $stderr.puts "Initializing output files." unless o[:q]
+   Dir.mkdir(o[:out]) unless Dir.exist? o[:out]
+   ofhs = o[:pergenome] ?
+      Gene.genomes.map{|g| File.open("#{o[:out]}/#{g}.fa", "w")} :
+      ( (1 .. collection.ogs.size).map do |og|
+	 File.open("#{o[:out]}/OG#{og}.fa", "w")
+      end )
+   $stderr.puts " Created files: #{ofhs.size}." unless o[:q]
+   
+   # Read genomes
+   $stderr.puts "Filtering genes." unless o[:q]
+   genome_i = -1
+   Gene.genomes.each do |genome|
+      genome_i = Gene.genomes.index(genome)
+      $stderr.print "  Genome #{genome_i+1}.   \r" unless o[:q]
+      genes = ( collection.get_genome_genes(genome).map do |og|
+	    o[:first] ? [og.first] : (o[:rand] ? [og.sample] : og)
+	 end )
+      hand = nil
+      File.open(sprintf(o[:seqs], genome), "r").each do |ln|
+	 if ln =~ /^>(\S+)/
+	    og = genes.index{|g| g.include? $1}
+	    hand = og.nil? ? nil : ( o[:pergenome] ? genome_i : og )
+	    ln.sub!(/^>/, ">#{o[:pergenome] ? "OG#{og}" : genome}-") if
+	       o[:prefix] and not hand.nil?
+	 end
+	 ofhs[hand].puts(ln) unless hand.nil?
+      end
+   end
+   $stderr.puts "  #{genome_i+1} genomes processed." unless o[:q]
+
+   # Close outputs
+   $stderr.puts "Closing output files." unless o[:q]
+   ofhs.each{|h| h.close}
+   $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+

data/utils/enveomics/Scripts/ogs.mcl.rb

@@ -0,0 +1,186 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @update: Sep-11-2015
+# @license: artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require 'enveomics_rb/og'
+require 'optparse'
+require 'tmpdir'
+
+o = {q:false, f:"(\\S+)-(\\S+)\\.rbm", mcl:"", inflation:1.5, blind:false,
+   evalue:false, thr:2, identity:false, bestmatch:false}
+ARGV << "-h" if ARGV.size==0
+OptionParser.new do |opts|
+   opts.banner = "
+Identifies Orthology Groups (OGs) in Reciprocal Best Matches (RBM)
+between all pairs in a collection of genomes, using the Markov Cluster
+Algorithm.
+
+Requires MCL (see http://www.micans.org/mcl).
+
+Usage: #{$0} [options]"
+   opts.separator ""
+   opts.separator "Mandatory"
+   opts.on("-o", "--out FILE",
+      "Output file containing the detected OGs."){ |v| o[:out]=v }
+   opts.on("-d", "--dir DIR",
+      "Directory containing the RBM files.",
+      "Becomes optional iff --abc is set to a non-empty file."){ |v| o[:dir]=v }
+   opts.separator ""
+   opts.separator "Other Options"
+   opts.on("-f", "--format STRING",
+      "Format of the filenames for the RBM files (within -d), using regex " +
+      "syntax.", "By default: '#{o[:f]}'."){ |v| o[:f]=v }
+   opts.on("-I", "--inflation FLOAT",
+      "Inflation parameter for MCL clustering. By default: #{o[:inflation]}."
+      ){ |v| o[:inflation]=v.to_f }
+   opts.on("-b", "--blind",
+      "If set, computes clusters without taking bitscore into account."
+      ){ |v| o[:blind]=v }
+   opts.on("-e", "--evalue",
+      "If set, uses the e-value to weight edges, instead of the default " +
+      "Bit-Score."){ |v| o[:evalue]=v }
+   opts.on("-i", "--identity",
+      "If set, uses the identity to weight edges, instead of the default " +
+      "Bit-Score."){ |v| o[:identity]=v }
+   opts.on("-B", "--best-match",
+      "If set, it assumes best-matches instead reciprocal best matches."
+      ){ |v| o[:bestmatch]=v }
+   opts.on("-m", "--mcl-bin DIR",
+      "Path to the directory containing the mcl binaries.",
+      "By default, assumed to be in the PATH."){ |v| o[:mcl]=v+"/" }
+   opts.on("--abc FILE",
+      "Use this abc file instead of a temporal file."){ |v| o[:abc] = v }
+   opts.on("-t", "--threads INT",
+      "Number of threads to use. By default: #{o[:thr]}."){ |v| o[:thr]=v.to_i }
+   opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+   opts.on("-h", "--help", "Display this screen.") do
+      puts opts
+      exit
+   end
+   opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:out].nil?
+o[:evalue] = false if o[:identity]
+o[:evalue] = false if o[:blind]
+o[:identity] = false if o[:blind]
+
+##### MAIN:
+begin
+   Dir.mktmpdir do |dir|
+      o[:abc] = "#{dir}/rbms.abc" if o[:abc].nil?
+      abort "-d must exist and be a directory" unless
+	 File.size?(o[:abc]) or
+	 (!o[:dir].nil? and File.exists?(o[:dir]) and File.directory?(o[:dir]))
+      # Traverse the whole directory
+      if File.size? o[:abc]
+	 $stderr.puts "Reusing existing abc file '#{o[:abc]}'." unless o[:q]
+      else
+	 file_i = 0
+	 ln_i = 0
+	 $stderr.puts "Reading RBM files within '#{o[:dir]}'." unless o[:q]
+	 abc = File.open(o[:abc] + ".tmp", "w")
+	 Dir.entries(o[:dir]).each do |rbm_file|
+	    next unless File.file?(o[:dir]+"/"+rbm_file)
+	    # Parse the filename to identify the genomes
+	    m = /#{o[:f]}/.match(rbm_file)
+	    if m.nil? or m[2].nil?
+	       warn "Ignoring #{rbm_file}: doesn't match /#{o[:f]}/."
+	       next
+	    end
+	    file_i += 1
+	    # Read the RBMs list
+	    f = File.open(o[:dir]+"/"+rbm_file, "r")
+	    while ln = f.gets
+	       # Add the RBM to the abc file
+	       row = ln.split(/\t/)
+	       abc.puts [m[1]+">"+row[0], m[2]+">"+row[1],
+		  (o[:blind] ? "1" :
+		  (o[:evalue] ? row[10] :
+		  (o[:identity] ? row[2] : row[11])))].join("\t")
+	       ln_i += 1
+	    end
+	    f.close
+	    $stderr.print " Scanned files: #{file_i}. " +
+	       "Found RBMs: #{ln_i}.   \r" unless o[:q]
+	 end
+	 abc.close
+	 File.rename(o[:abc] + ".tmp", o[:abc])
+	 $stderr.print "\n" unless o[:q]
+      end # if File.size? o[:abc] ... else
+
+      # Build .mci file (mcxload) and compute .mccl file (mcl)
+      $stderr.puts "Markov-Clustering" unless o[:q]
+      `'#{o[:mcl]}mcxload' #{"--stream-mirror" unless o[:bestmatch]} \
+	 -abc '#{o[:abc]}' -o '#{dir}/rbms.mci' --write-binary \
+	 -write-tab '#{dir}/genes.tab' #{"--stream-neg-log10" if o[:evalue]} \
+	 &>/dev/null`
+      `'#{o[:mcl]}mcl' '#{dir}/rbms.mci' -V all -I #{o[:inflation].to_s} \
+	 -o '#{dir}/ogs.mcl' -te #{o[:thr].to_s}`
+
+      # Load .tab as Gene objects
+      $stderr.puts "Loading gene table from '#{dir}/genes.tab'." unless o[:q]
+      genes = []
+      tab = File.open("#{dir}/genes.tab", "r")
+      while ln = tab.gets
+	 ln.chomp!
+	 r = ln.split /\t|>/
+	 genes[ r[0].to_i ] = Gene.new(r[1], r[2])
+      end
+      tab.close
+      $stderr.puts " Got " + genes.size.to_s + " genes in " +
+	 Gene.genomes.size.to_s + " genomes." unless o[:q]
+      
+      # Load .mcl file as OGCollection
+      $stderr.puts "Loading clusters from '#{dir}/ogs.mcl'." unless o[:q]
+      collection = OGCollection.new
+      mcl = File.open("#{dir}/ogs.mcl", "r")
+      in_matrix = false
+      my_genes = nil
+      while ln = mcl.gets
+         ln.chomp!
+	 if ln =~ /^\(mclmatrix$/
+	    in_matrix = true
+	    next
+	 end
+	 next if ln =~ /^begin$/
+	 if in_matrix
+	    break if ln =~ /^\)$/
+	    if ln =~ /^\d+\s+/
+	       ln.sub!(/^\d+\s+/, "")
+	       my_genes = []
+	    end
+	    ln.sub!(/^\s+/, "")
+	    raise "Incomplete mcl matrix, offending line: #{$.}: #{ln}" if
+	       my_genes.nil?
+	    my_genes += ln.split(/\s/)
+	    if my_genes.last == "$"
+	       my_genes.pop
+	       og = OG.new
+	       my_genes.each{|i| og << genes[ i.to_i ]}
+	       collection << og
+	       my_genes = nil
+	    end
+	 end
+      end
+      mcl.close
+      $stderr.puts " Got #{collection.ogs.size} clusters." unless o[:q]
+      
+      # Save the output matrix
+      $stderr.puts "Saving matrix into '#{o[:out]}'." unless o[:q]
+      f = File.open(o[:out], "w")
+      f.puts collection.to_s
+      f.close
+      $stderr.puts "Done.\n" unless o[:q] 
+   end
+rescue => err
+   $stderr.puts "Exception: #{err}\n\n"
+   err.backtrace.each { |l| $stderr.puts l + "\n" }
+   err
+end
+
+