miga-base 0.4.3.0 → 0.5.0.0

data/utils/enveomics/Scripts/HMM.haai.rb

@@ -22,6 +22,10 @@ Usage: #{$0} [options]"
   opt.separator 'Options'
   opt.on('-a', '--aln-out FILE',
     'Output file containing the aligned proteins'){ |v| o[:alnout] = v }
+  opt.on('-c', '--components FILE',
+    'Output file containing the components of the estimation.',
+    'Tab-delimited file with model name, matches, and columns.'
+    ){ |v| o[:compout] = v }
   opt.on('-q', '--quiet', 'Run quietly (no STDERR output).'){ o[:q] = true }
   opt.on('-h', '--help', 'Display this screen.') do
     puts opt
@@ -34,6 +38,7 @@ abort '-2 is mandatory.' if o[:b].nil?
 
 class HList
   attr_accessor :list
+  
   def initialize(file)
     @list = {}
     r = File.readlines(file)
@@ -63,6 +68,7 @@ end
 class HElement
   attr_accessor :defline, :model_id, :protein_id, :protein_coords
   attr_accessor :model_aln, :protein_aln
+
   def initialize(defline, model_aln, protein_aln)
     @defline = defline.chomp
     @model_aln = model_aln.chomp
@@ -81,32 +87,27 @@ class HElement
   ##
   # Returns an HAln object
   def align(other)
+    return nil unless model_width == other.model_width
     HAln.new(self, other)
   end
 
-  def mask
-    @mask ||= model_aln.chars.
-      each_with_index.map{ |v, k| v == '.' ? k : nil }.
-      compact.reverse
+  def masked_protein
+    @masked_protein ||= model_aln.chars.
+      each_with_index.map{ |c, pos| c == 'X' ? protein_aln[pos] : nil }.
+        compact.join('')
   end
 
-  def mask!(template)
-    (template - mask).each do |d|
-      @model_aln[d]   = '-' + @model_aln[d]
-      @protein_aln[d] = '-' + @protein_aln[d]
-    end
+  def model_width
+    masked_protein.size
   end
 end
 
 class HAln
   attr :protein_1, :protein_2, :model_id, :protein_1_id, :protein_2_id
+  
   def initialize(a, b)
-    a_masked = a.dup
-    a_masked.mask! b.mask.reverse
-    b_masked = b.dup
-    b_masked.mask! b_masked.mask
-    @protein_1 = a_masked.protein_aln
-    @protein_2 = b_masked.protein_aln
+    @protein_1 = a.masked_protein
+    @protein_2 = b.masked_protein
     @model_id = a.model_id
     @protein_1_id = a.protein_id + '/' + a.protein_coords
     @protein_2_id = b.protein_id + '/' + b.protein_coords
@@ -116,7 +117,9 @@ class HAln
     @stats = { len: 0, gaps: 0, matches: 0 }
     return @stats unless @stats[:id].nil?
     protein_1.chars.each_with_index do |v, k|
+      # Ignore gaps in both proteins
       next if v == '-' and protein_2[k] == '-'
+      # Count matches
       @stats[:len] += 1
       if v == protein_2[k]
         @stats[:matches] += 1
@@ -124,16 +127,16 @@ class HAln
         @stats[:gaps] += 1
       end
     end
-    @stats.tap { |i| i[:id] = 100.0 * @stats[:matches] / @stats[:len] }
+    @stats.tap { |i| i[:id] = 100.0 * i[:matches] / i[:len] }
   end
 
   def stats_to_s
-    stats.map{ |k,v| "#{k}:#{v}" }.join " "
+    stats.map{ |k,v| "#{k}:#{v}" }.join ' '
   end
 
   def to_s
-    "# #{model_id} | #{protein_1_id} | #{protein_2_id} | #{stats_to_s}\n" +
-      protein_1 + "\n" + protein_2 + "\n"
+    ["# #{model_id} | #{protein_1_id} | #{protein_2_id} | #{stats_to_s}",
+      protein_1, protein_2, ''].join("\n")
   end
 end
 
@@ -151,8 +154,14 @@ puts "SD identity: #{sd_identity.round(2)}"
 
 if o[:alnout]
   File.open(o[:alnout], 'w') do |fh|
+    haln_arr.each { |i| fh.puts i }
+  end
+end
+
+if o[:compout]
+  File.open(o[:compout], 'w') do |fh|
     haln_arr.each do |i|
-      fh.puts i
+      fh.puts "#{i.model_id}\t#{i.stats[:matches]}\t#{i.stats[:len]}"
     end
   end
 end

data/utils/enveomics/Scripts/SRA.download.bash

@@ -45,7 +45,7 @@ tail -n +2 "$DIR/srr_list.txt" | while read ln ; do
   for uri in $(echo "$ftp" | tr ";" " ") ; do
     file="$dir/$(basename $uri)"
     curl "$uri" -o "$file"
-    md5obs=$(md5value "$file")
+    md5obs=$(md5value "$file" 2> /dev/null)
     if [[ "$md5" == "$md5obs"* ]] ; then
       md5=$(echo "$md5" | perl -pe 's/^[^;]+;//')
     else

data/utils/enveomics/Scripts/aai.rb

@@ -3,144 +3,177 @@
 # @author  Luis M. Rodriguez-R
 # @license Artistic-2.0
 
-require "optparse"
-require "tmpdir"
+require 'optparse'
+require 'tmpdir'
+require 'zlib'
 has_rest_client = true
 has_sqlite3 = true
 begin
-  require "rubygems"
-  require "restclient"
+  require 'rubygems'
+  require 'restclient'
 rescue LoadError
   has_rest_client = false
 end
 begin
-  require "sqlite3"
+  require 'sqlite3'
 rescue LoadError
   has_sqlite3 = false
 end
 
-o = {bits:0, id:20, len:0, hits:50, q:false, bin:"", program:"blast+", thr:1,
-  dec:2, auto:false, lookupfirst:false, dbrbm: true, nucl: false,
-  len_fraction:0.0, max_actg:0.95}
-ARGV << "-h" if ARGV.size==0
+o = {
+  bits: 0, id: 20, len: 0, hits: 50, q: false, bin: '', program: 'blast+',
+  thr: 1, dec: 2, auto: false, lookupfirst: false, dbrbm: true, nucl: false,
+  len_fraction: 0.0, max_actg: 0.95
+}
+ARGV << '-h' if ARGV.size == 0
 OptionParser.new do |opts|
   opts.banner = "
-Calculates the Average Amino acid Identity between two genomes.
+Calculates the Average Amino Acid Identity between two genomes
 
 Usage: #{$0} [options]"
-  opts.separator ""
-  opts.separator "Mandatory"
-  opts.on("-1", "--seq1 FILE",
-    "Path to the FastA file containing the genome 1 (proteins)."
-    ){ |v| o[:seq1] = v }
-  opts.on("-2", "--seq2 FILE",
-    "Path to the FastA file containing the genome 2 (proteins)."
-    ){ |v| o[:seq2] = v }
+  opts.separator ''
+  opts.separator 'Mandatory'
+  opts.on(
+    '-1', '--seq1 FILE',
+    'Path to the FastA file (.gz allowed) containing the genome 1 (proteins)'
+  ) { |v| o[:seq1] = v }
+  opts.on(
+    '-2', '--seq2 FILE',
+    'Path to the FastA file (.gz allowed) containing the genome 2 (proteins)'
+  ) { |v| o[:seq2] = v }
   if has_rest_client
-    opts.separator "    Alternatively, you can supply the NCBI-acc of a " +
-      "genome (nucleotides) with the format ncbi:CP014272 instead of files."
+    opts.separator '    Alternatively, you can supply the NCBI-acc of a ' +
+      'genome (nucleotides) with the format ncbi:CP014272 instead of files'
   else
-    opts.separator "    Install rest-client to enable NCBI-acc support."
+    opts.separator '    Install rest-client to enable NCBI-acc support'
   end
-  opts.separator ""
-  opts.separator "Search Options"
-  opts.on("-l", "--len INT",
-    "Minimum alignment length (in residues).  By default: #{o[:len]}."
-    ){ |v| o[:len] = v.to_i }
-  opts.on("-L", "--len-fraction NUM",
-    "Minimum alignment length as a fraction of the shorter sequence",
-    "(range 0-1).  By default: #{o[:len_fraction]}."
-    ){ |v| o[:len_fraction] = v.to_f }
-  opts.on("-i", "--id NUM",
-    "Minimum alignment identity (in %).  By default: #{o[:id]}."
-    ){ |v| o[:id] = v.to_f }
-  opts.on("-s", "--bitscore NUM",
-    "Minimum bit score (in bits).  By default: #{o[:bits]}."
-    ){ |v| o[:bits] = v.to_f }
-  opts.on("-n", "--hits INT",
-    "Minimum number of hits.  By default: #{o[:hits]}."
-    ){ |v| o[:hits] = v.to_i }
-  opts.on("-N", "--nucl",
-    "The input sequences are nucleotides (genes), not proteins."
-    ){ |v| o[:nucl] = v }
-  opts.on("--max-actg FLOAT",
-    "Maximum fraction of ACTGN in the sequences before assuming nucleotides.",
-    "By default: #{o[:max_actg]}."
-    ){ |v| o[:max_actg] = v.to_f }
-  opts.separator ""
-  opts.separator "Software Options"
-  opts.on("-b", "--bin DIR",
-    "Path to the directory containing the binaries of the search program."
-    ){ |v| o[:bin] = v }
-  opts.on("-p", "--program STR",
-    "Search program to be used. One of: blast+ (default), blast, blat, diamond."
-    ){ |v| o[:program] = v }
-  opts.on("-t", "--threads INT",
-    "Number of parallel threads to be used.  By default: #{o[:thr]}."
-    ){ |v| o[:thr] = v.to_i }
-  opts.separator ""
-  opts.separator "SQLite3 Options"
-  opts.on("-S", "--sqlite3 FILE",
-    "Path to the SQLite3 database to create (or update) with the results."
-    ){ |v| o[:sqlite3] = v }
-  opts.separator "    Install sqlite3 gem to enable database support." unless
-    has_sqlite3
-  opts.on("--name1 STR",
-    "Name of --seq1 to use in --sqlite3.  By default determined by filename."
-    ){ |v| o[:seq1name] = v }
-  opts.on("--name2 STR",
-    "Name of --seq2 to use in --sqlite3.  By default determined by filename."
-    ){ |v| o[:seq2name] = v }
-  opts.on("--[no-]save-rbm",
-    "Save (or don't save) the reciprocal best matches in the --sqlite3 db.",
-    "By default: #{o[:dbrbm]}."){ |v| o[:dbrbm] = !!v }
-  opts.on("--lookup-first",
-    "Indicates if the AAI should be looked up first in the database.",
-    "Requires --sqlite3, --auto, --name1, and --name2.",
-    "Incompatible with --res, --tab, --out, and --rbm."
-    ){ |v| o[:lookupfirst] = v }
-  opts.separator ""
-  opts.separator "Other Output Options"
-  opts.on("-d", "--dec INT",
+  opts.separator ''
+  opts.separator 'Search Options'
+  opts.on(
+    '-l', '--len INT', Integer,
+    "Minimum alignment length (in residues).  By default: #{o[:len]}"
+  ) { |v| o[:len] = v }
+  opts.on(
+    '-L', '--len-fraction NUM', Float,
+    'Minimum alignment length as a fraction of the shorter sequence',
+    "(range 0-1).  By default: #{o[:len_fraction]}"
+  ) { |v| o[:len_fraction] = v }
+  opts.on(
+    '-i', '--id FLOAT', Float,
+    "Minimum alignment identity (in %).  By default: #{o[:id]}"
+  ) { |v| o[:id] = v }
+  opts.on(
+    '-s', '--bitscore FLOAT', Float,
+    "Minimum bit score (in bits).  By default: #{o[:bits]}"
+  ) { |v| o[:bits] = v }
+  opts.on(
+    '-n', '--hits INT', Integer,
+    "Minimum number of hits.  By default: #{o[:hits]}"
+  ) { |v| o[:hits] = v }
+  opts.on(
+    '-N', '--nucl',
+    'The input sequences are nucleotides (genes), not proteins'
+  ) { |v| o[:nucl] = v }
+  opts.on(
+    '--max-actg FLOAT', Float,
+    'Maximum fraction of ACTGN in the sequences before assuming nucleotides',
+    "By default: #{o[:max_actg]}"
+  ) { |v| o[:max_actg] = v }
+  opts.separator ''
+  opts.separator 'Software Options'
+  opts.on(
+    '-b', '--bin DIR',
+    'Path to the directory containing the binaries of the search program'
+  ) { |v| o[:bin] = v }
+  opts.on(
+    '-p', '--program STR',
+    'Search program to be used. One of: blast+ (default), blast, blat, diamond'
+  ) { |v| o[:program] = v }
+  opts.on(
+    '-t', '--threads INT', Integer,
+    "Number of parallel threads to be used.  By default: #{o[:thr]}"
+  ) { |v| o[:thr] = v }
+  opts.separator ''
+  opts.separator 'SQLite3 Options'
+  unless has_sqlite3
+    opts.separator '    Install sqlite3 gem to enable database support'
+  end
+  opts.on(
+    '-S', '--sqlite3 FILE',
+    'Path to the SQLite3 database to create (or update) with the results'
+  ) { |v| o[:sqlite3] = v }
+  opts.on(
+    '--name1 STR',
+    'Name of --seq1 to use in --sqlite3.  By default determined by filename'
+  ) { |v| o[:seq1name] = v }
+  opts.on(
+    '--name2 STR',
+    'Name of --seq2 to use in --sqlite3.  By default determined by filename'
+  ) { |v| o[:seq2name] = v }
+  opts.on(
+    '--[no-]save-rbm',
+    'Save (or don\'t save) the reciprocal best matches in the --sqlite3 db',
+    "By default: #{o[:dbrbm]}"
+  ) { |v| o[:dbrbm] = v }
+  opts.on(
+    '--lookup-first',
+    'Indicates if the AAI should be looked up first in the database',
+    'Requires --sqlite3, --auto, --name1, and --name2',
+    'Incompatible with --res, --tab, --out, and --rbm'
+  ) { |v| o[:lookupfirst] = v }
+  opts.separator ''
+  opts.separator 'Other Output Options'
+  opts.on(
+    '-d', '--dec INT', Integer,
     "Decimal positions to report. By default: #{o[:dec]}"
-    ){ |v| o[:dec] = v.to_i }
-  opts.on("-R", "--rbm FILE",
-    "Saves a file with the reciprocal best matches."){ |v| o[:rbm] = v }
-  opts.on("-o", "--out FILE",
-    "Saves a file describing the alignments used for two-way AAI."
-    ){ |v| o[:out] = v }
-  opts.on("-r", "--res FILE",
-    "Saves a file with the final results."){ |v| o[:res] = v }
-  opts.on("-T", "--tab FILE",
-    "Saves a file with the final two-way results in a tab-delimited form.",
-    "The columns are (in that order):",
-    "AAI, standard deviation, proteins used, proteins in the smallest genome."
-    ){ |v| o[:tab]=v }
-  opts.on("-a", "--auto",
-    "ONLY outputs the AAI value in STDOUT (or nothing, if calculation fails)."
-    ){ o[:auto] = true }
-  opts.on("-q", "--quiet", "Run quietly (no STDERR output)"){ o[:q] = true }
-  opts.on("-h", "--help", "Display this screen") do
+  ) { |v| o[:dec] = v }
+  opts.on(
+    '-R', '--rbm FILE',
+    'Saves a file with the reciprocal best matches'
+  ) { |v| o[:rbm] = v }
+  opts.on(
+    '-o', '--out FILE',
+    'Saves a file describing the alignments used for two-way AAI'
+  ) { |v| o[:out] = v }
+  opts.on(
+    '-r', '--res FILE', 'Saves a file with the final results'
+  ) { |v| o[:res] = v }
+  opts.on(
+    '-T', '--tab FILE',
+    'Saves a file with the final two-way results in a tab-delimited form',
+    'The columns are (in that order):',
+    'AAI, standard deviation, proteins used, proteins in the smallest genome'
+  ) { |v| o[:tab] = v }
+  opts.on(
+    '-a', '--auto',
+    'ONLY outputs the AAI value in STDOUT (or nothing, if calculation fails)'
+  ) { o[:auto] = true }
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') do
     puts opts
     exit
   end
-  opts.separator ""
+  opts.separator ''
 end.parse!
-abort "-1 is mandatory" if o[:seq1].nil?
-abort "-2 is mandatory" if o[:seq2].nil?
-abort "-p diamond is incompatible with -N" if o[:program]=="diamond" && o[:nucl]
-abort "SQLite3 requested (-S) but sqlite3 not supported.  First install gem " +
-  "sqlite3." unless o[:sqlite3].nil? or has_sqlite3
-o[:bin] = o[:bin]+"/" if o[:bin].size > 0
+
+# Check input
+abort '-1 is mandatory' if o[:seq1].nil?
+abort '-2 is mandatory' if o[:seq2].nil?
+if o[:program] == 'diamond' && o[:nucl]
+  abort '-p diamond is incompatible with -N'
+end
+unless o[:sqlite3].nil? or has_sqlite3
+  abort 'SQLite3 requested (-S) but sqlite3 not supported: gem install sqlite3'
+end
+o[:bin] = o[:bin] + '/' if o[:bin].size > 0
 if o[:lookupfirst]
-  abort "--lookup-first needs --sqlite3" if o[:sqlite3].nil?
-  abort "--lookup-first requires --auto" unless o[:auto]
-  abort "--lookup-first requires --name1" if o[:seq1name].nil?
-  abort "--lookup-first requires --name2" if o[:seq2name].nil?
-  abort "--lookup-first conflicts with --res" unless o[:res].nil?
-  abort "--lookup-first conflicts with --tab" unless o[:tab].nil?
-  abort "--lookup-first conflicts with --out" unless o[:out].nil?
-  abort "--lookup-first conflicts with --rbm" unless o[:rbm].nil?
+  abort '--lookup-first requires --name1' if o[:seq1name].nil?
+  abort '--lookup-first requires --name2' if o[:seq2name].nil?
+  abort '--lookup-first needs --sqlite3' if o[:sqlite3].nil?
+  abort '--lookup-first requires --auto' unless o[:auto]
+  %w[res tab out rbm].each do |k|
+    abort "--lookup-first conflicts with --#{k}" unless o[k.to_sym].nil?
+  end
 end
 
 # Create SQLite3 file
@@ -180,7 +213,7 @@ Dir.mktmpdir do |dir|
     abort "GIs are no longer supported by NCBI. Please use NCBI-acc instead." if
       /^gi:/.match(o[seq])
     acc = /^ncbi:(\S+)/.match(o[seq])
-    if not acc.nil?
+    unless acc.nil?
       abort "NCBI-acc requested, but rest-client not supported.  First " +
         "install gem rest-client." unless has_rest_client
       abort "NCBI-acc are currently not supported with --nucl. Please use " +
@@ -226,22 +259,24 @@ Dir.mktmpdir do |dir|
     seq_len[seq] = [0]
     actg_cnt[seq] = 0
     seqs = 0
-    fi = File.open(o[seq], "r")
-    fo = File.open("#{dir}/#{seq.to_s}.fa", "w")
-    fi.each_line do |ln|
-      if ln =~ /^>(\S+)/
-        seqs += 1
-        ori_ids[seq] << $1 unless o[:rbm].nil? and o[:sqlite3].nil?
-        seq_len[seq][seqs] = 0
-        fo.puts ">#{seqs}"
-      else
-        fo.puts ln
-        seq_len[seq][seqs] += ln.chomp.gsub(/[^A-Za-z]/,"").length
-        actg_cnt[seq] += ln.chomp.gsub(/[^ACTGNactgn]/,"").length
+    fi = File.extname(o[seq]) == '.gz' ?
+      Zlib::GzipReader.open(o[seq]) :
+      File.open(o[seq], 'r')
+    File.open("#{dir}/#{seq.to_s}.fa", 'w') do |fo|
+      fi.each_line do |ln|
+        if ln =~ /^>(\S+)/
+          seqs += 1
+          ori_ids[seq] << $1 unless o[:rbm].nil? and o[:sqlite3].nil?
+          seq_len[seq][seqs] = 0
+          fo.puts ">#{seqs}"
+        else
+          fo.puts ln
+          seq_len[seq][seqs] += ln.chomp.gsub(/[^A-Za-z]/,"").length
+          actg_cnt[seq] += ln.chomp.gsub(/[^ACTGNactgn]/,"").length
+        end
       end
     end
     fi.close
-    fo.close
     unless o[:nucl]
       actg_frx = actg_cnt[seq].to_f/seq_len[seq].inject(:+).to_f
       abort "Input sequences appear to be nucleotides " +