miga-base 1.2.17.0 → 1.2.17.1

data/utils/enveomics/Scripts/ogs.stats.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+
+#
+# @author: Luis M. Rodriguez-R
+# @license: Artistic-2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + '/lib')
+require 'enveomics_rb/og'
+require 'optparse'
+require 'json'
+
+o = {q:false, a:false}
+ARGV << '-h' if ARGV.size==0
+OptionParser.new do |opts|
+  opts.banner = "
+Estimates some descriptive statistics on a set of Orthology Groups (OGs).
+
+Usage: #{$0} [options]"
+  opts.separator ""
+  opts.separator "Mandatory"
+  opts.on("-o", "--ogs FILE",
+    "Input file containing the precomputed OGs."){ |v| o[:ogs]=v }
+  opts.separator ""
+  opts.separator "Other Options"
+  opts.on("-j", "--json FILE", "Output file in JSON format."){ |v| o[:json]=v }
+  opts.on("-t", "--tab FILE","Output file in tabular format."){ |v| o[:tab]=v }
+  opts.on("-T", "--transposed-tab FILE",
+    "Output file in transposed tabular format."){ |v| o[:ttab]=v }
+  opts.on("-a", "--auto", "Run completely quietly (no STDERR or STDOUT)") do
+    o[:q] = true
+    o[:a] = true
+  end
+  opts.on("-q", "--quiet", "Run quietly (no STDERR output)."){ o[:q] = true }
+  opts.on("-h", "--help", "Display this screen.") do
+    puts opts
+    exit
+  end
+  opts.separator ""
+end.parse!
+abort "-o is mandatory" if o[:ogs].nil?
+
+##### MAIN:
+begin
+  # Initialize the collection of OGs.
+  collection = OGCollection.new
+   
+  # Read the pre-computed OGs
+  $stderr.puts "Reading pre-computed OGs in '#{o[:ogs]}'." unless o[:q]
+  f = File.open(o[:ogs], "r")
+  h = f.gets.chomp.split /\t/
+  while ln = f.gets
+    collection << OG.new(h, ln.chomp.split(/\t/))
+  end
+  f.close
+  $stderr.puts " Loaded OGs: #{collection.ogs.length}." unless o[:q]
+   
+  # Estimate descriptive stats
+  stat_name = {
+    genomes: "Number of genomes",
+    pan: "Pangenome (OGs)",
+    core: "Core genome (OGs)",
+    core90pc: "OGs in 90% of the genomes",
+    core80pc: "OGs in 80% of the genomes",
+    unus: "Unus genome, core genome discarding paralogs (OGs)",
+    avg: "Average number of OGs in a genome",
+    avg_pan: "Average genome (OGs) / Pangenome (OGs)",
+    core_avg: "Core genome (OGs) / Average genome (OGs)",
+    core_pan: "Core genome (OGs) / Pangenome (OGs)",
+    ogs_shannon: "Entropy of the OG frequencies (bits)"
+  }
+  stats = {}
+  stats[:genomes] = Gene.genomes.length
+  stats[:pan] = collection.ogs.length
+  stats[:core] = collection.ogs.map do |og|
+    (og.genomes.length == Gene.genomes.length) ? 1 : 0
+  end.inject(0,:+)
+  stats[:core90pc] = collection.ogs.map do |og|
+    (og.genomes.length >= 0.9*Gene.genomes.length) ? 1 : 0
+  end.inject(0,:+)
+  stats[:core80pc] = collection.ogs.map do |og|
+    (og.genomes.length >= 0.8*Gene.genomes.length) ? 1 : 0
+  end.inject(0,:+)
+  stats[:unus] = collection.ogs.map do |og|
+    (og.genomes.length != Gene.genomes.length) ? 0 :
+      (og.genes.all?{ |i| i.size==1 }) ? 1 : 0
+  end.inject(0,:+)
+  og_genomes = collection.ogs.map{ |og| og.genomes.length }.inject(0,:+)
+  stats[:avg] = og_genomes.to_f/Gene.genomes.length
+  stats[:avg_pan] = stats[:avg]/stats[:pan]
+  stats[:core_avg] = stats[:core].to_f/stats[:avg]
+  stats[:core_pan] = stats[:core].to_f/stats[:pan]
+  stats[:ogs_shannon] = -1 * collection.ogs.map do |og|
+    pi = og.genomes.length.to_f/Gene.genomes.length
+    pi * Math.log(pi)
+  end.inject(0.0,:+)
+
+  # Show result
+  $stderr.puts "Generating reports." unless o[:q]
+  stats.each_pair{ |k,v| puts " #{stat_name[k]}: #{v}" } unless o[:a]
+
+  # Save results in JSON
+  unless o[:json].nil?
+    ohf = File.open(o[:json], "w")
+    ohf.puts JSON.pretty_generate(stats)
+    ohf.close
+  end
+
+  # Save results in tab
+  unless o[:tab].nil?
+    ohf = File.open(o[:tab], "w")
+    stats.each_pair{ |k,v| ohf.puts "#{k}\t#{v}" }
+    ohf.close
+  end
+
+  # Save results in T(tab)
+  unless o[:ttab].nil?
+    ohf = File.open(o[:ttab], "w")
+    ohf.puts stats.keys.join("\t")
+    ohf.puts stats.values.join("\t")
+    ohf.close
+  end
+
+  $stderr.puts "Done.\n" unless o[:q] 
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end
+
+

data/utils/enveomics/Scripts/rbm-legacy.rb

@@ -0,0 +1,172 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+$VERSION = 0.1
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+require 'tmpdir'
+
+o = {
+  q: false, thr: 1,
+  len: 0, id: 0.0, fract: 0.0, score: 0.0,
+  bin: '', program: :'blast+', nucl: false
+}
+
+OptionParser.new do |opts|
+  cmd = File.basename($0)
+  opts.banner = <<~BANNER
+
+    [Enveomics Collection: #{cmd} v#{$VERSION}]
+
+    [DEPRECATED: Please use rbm.rb instead]
+
+    Finds the reciprocal best matches between two sets of sequences
+
+    Usage: #{cmd} [options]
+
+  BANNER
+
+  opts.separator 'Mandatory'
+  opts.on(
+    '-1', '--seq1 FILE',
+    'Path to the FastA file containing the set 1'
+  ) { |v| o[:seq1] = v }
+  opts.on(
+    '-2', '--seq2 FILE',
+    'Path to the FastA file containing the set 2'
+  ) { |v| o[:seq2] = v }
+  opts.separator ''
+  opts.separator 'Search Options'
+  opts.on(
+    '-n', '--nucl',
+    'Sequences are assumed to be nucleotides (proteins by default)',
+    'Incompatible with -p diamond'
+  ) { |v| o[:nucl] = true }
+  opts.on(
+    '-l', '--len INT', Integer,
+    'Minimum alignment length (in residues)',
+    "By default: #{o[:len]}"
+  ) { |v| o[:len] = v }
+  opts.on(
+    '-f', '--fract FLOAT', Float,
+    'Minimum alignment length (as a fraction of the query)',
+    'If set, requires BLAST+ or Diamond (see -p)',
+    "By default: #{o[:fract]}"
+  ) { |v| o[:fract] = v }
+  opts.on(
+    '-i', '--id NUM', Float,
+    'Minimum alignment identity (in %)',
+    "By default: #{o[:id]}"
+  ){ |v| o[:id] = v }
+  opts.on(
+    '-s', '--score NUM', Float,
+    'Minimum alignment score (in bits)',
+    "By default: #{o[:score]}"
+  ) { |v| o[:score] = v }
+  opts.separator ''
+  opts.separator 'Software Options'
+  opts.on(
+    '-b', '--bin DIR',
+    'Path to the directory containing the binaries of the search program'
+  ) { |v| o[:bin] = v }
+  opts.on(
+    '-p', '--program STR',
+    'Search program to be used.  One of: blast+ (default), blast, diamond'
+  ) { |v| o[:program] = v.downcase.to_sym }
+  opts.on(
+    '-t', '--threads INT', Integer,
+    'Number of parallel threads to be used',
+    "By default: #{o[:thr]}"
+  ) { |v| o[:thr] = v }
+  opts.separator ''
+  opts.separator 'Other Options'
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opts.on('-h', '--help', 'Display this screen') { puts opts ; exit }
+  opts.separator ''
+end.parse!
+
+abort '-1 is mandatory' if o[:seq1].nil?
+abort '-2 is mandatory' if o[:seq2].nil?
+if o[:program] == :diamond && o[:nucl]
+  abort '-p diamond is incompatible with -n'
+end
+if o[:fract] > 0.0 && o[:program] == :blast
+  abort 'Argument -f/--fract requires -p blast+ or -p diamond'
+end
+o[:bin] = o[:bin] + '/' if o[:bin].size > 0
+$quiet = o[:q]
+
+Dir.mktmpdir do |dir|
+  say('Temporal directory: ', dir)
+
+  # Create databases
+  say 'Creating databases'
+  [:seq1, :seq2].each do |seq|
+    case o[:program]
+    when :blast
+      `"#{o[:bin]}formatdb" -i "#{o[seq]}" -n "#{dir}/#{seq}" \
+        -p #{o[:nucl] ? 'F' : 'T'}`
+    when :'blast+'
+      `"#{o[:bin]}makeblastdb" -in "#{o[seq]}" -out "#{dir}/#{seq}" \
+        -dbtype #{o[:nucl] ? 'nucl' : 'prot'}`
+    when :diamond
+      `"#{o[:bin]}diamond" makedb --in "#{o[seq]}" \
+        --db "#{dir}/#{seq}.dmnd" --threads "#{o[:thr]}"`
+    else
+      abort "Unsupported program: #{o[:program]}"
+    end
+  end
+
+  # Best-hits
+  rbh = {}
+  n2 = 0
+  say ' Running comparisons'
+  [2, 1].each do |i|
+    qry_seen = {}
+    q = o[:"seq#{i}"]
+    s = "#{dir}/seq#{i == 1 ? 2 : 1}"
+    say('  Query: ', q)
+    case o[:program]
+    when :blast
+      `"#{o[:bin]}blastall" -p #{o[:nucl] ? 'blastn' : 'blastp'} -d "#{s}" \
+        -i "#{q}" -v 1 -b 1 -a #{o[:thr]} -m 8 -o "#{dir}/#{i}.tab"`
+    when :'blast+'
+      `"#{o[:bin]}#{o[:nucl] ? 'blastn' : 'blastp'}" -db "#{s}" -query "#{q}" \
+        -max_target_seqs 1 -num_threads #{o[:thr]} -out "#{dir}/#{i}.tab" \
+        -outfmt "6 qseqid sseqid pident length mismatch gapopen qstart qend \
+        sstart send evalue bitscore qlen slen"`
+    when :diamond
+      `"#{o[:bin]}diamond" blastp --threads "#{o[:thr]}" --db "#{s}.dmnd" \
+        --query "#{q}" --sensitive --daa "#{dir}/#{i}.daa" --quiet \
+        && "#{o[:bin]}diamond" view --daa "#{dir}/#{i}.daa" --outfmt \
+        6 qseqid sseqid pident length mismatch gapopen qstart qend sstart \
+        send evalue bitscore qlen slen --out "#{dir}/#{i}.tab" --quiet`
+    else
+      abort "Unsupported program: #{o[:program]}"
+    end
+
+    n = 0
+    File.open("#{dir}/#{i}.tab", 'r') do |fh|
+      fh.each do |ln|
+        ln.chomp!
+        row = ln.split(/\t/)
+        row[12] = '1' unless [:'blast+', :diamond].include? o[:program]
+        next unless qry_seen[row[0]].nil? &&
+          row[3].to_i >= o[:len] && row[2].to_f >= o[:id] &&
+          row[11].to_f >= o[:score] && row[3].to_f / row[12].to_i >= o[:fract]
+
+        qry_seen[row[0]] = 1
+        n += 1
+        if i == 2
+          rbh[row[0]] = row[1]
+        elsif !rbh[row[1]].nil? && rbh[row[1]] == row[0]
+          puts ln
+          n2 += 1
+        end
+      end
+    end
+    say "    #{n} sequences with hit"
+  end
+  say "  #{n2} RBMs"
+end

data/utils/enveomics/Scripts/rbm.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+$VERSION = 1.01
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/rbm'
+require 'tmpdir'
+
+bms_dummy = Enveomics::RBM.new('1', '2').bms1
+o = { q: false, out: '-' }
+%i[thr len id fract score bin program nucl].each do |k|
+  o[k] = bms_dummy.opt(k)
+end
+
+OptionParser.new do |opts|
+  opts.version = $VERSION
+  cmd = File.basename($0)
+  opts.banner = <<~BANNER
+
+    [Enveomics Collection: #{cmd} v#{$VERSION}]
+
+    Finds the reciprocal best matches between two sets of sequences
+
+    Usage: #{cmd} [options]
+
+  BANNER
+
+  opts.separator 'Mandatory'
+  opts.on(
+    '-1', '--seq1 FILE',
+    'Path to the FastA file containing the set 1'
+  ) { |v| o[:seq1] = v }
+  opts.on(
+    '-2', '--seq2 FILE',
+    'Path to the FastA file containing the set 2'
+  ) { |v| o[:seq2] = v }
+  opts.on(
+    '-o', '--out FILE',
+    'Reciprocal Best Matches in BLAST tabular format.',
+    'Supports compression with .gz extension, use - for STDOUT (default)'
+  ) { |v| o[:out] = v }
+  opts.separator ''
+  opts.separator 'Search Options'
+  opts.on(
+    '-n', '--nucl',
+    'Sequences are assumed to be nucleotides (proteins by default)',
+    'Incompatible with -p diamond'
+  ) { |v| o[:nucl] = true }
+  opts.on(
+    '-l', '--len INT', Integer,
+    'Minimum alignment length (in residues)',
+    "By default: #{o[:len]}"
+  ) { |v| o[:len] = v }
+  opts.on(
+    '-f', '--fract FLOAT', Float,
+    'Minimum alignment length (as a fraction of the query)',
+    'If set, requires BLAST+ or Diamond (see -p)',
+    "By default: #{o[:fract]}"
+  ) { |v| o[:fract] = v }
+  opts.on(
+    '-i', '--id NUM', Float,
+    'Minimum alignment identity (in %)',
+    "By default: #{o[:id]}"
+  ){ |v| o[:id] = v }
+  opts.on(
+    '-s', '--score NUM', Float,
+    'Minimum alignment score (in bits)',
+    "By default: #{o[:score]}"
+  ) { |v| o[:score] = v }
+  opts.separator ''
+  opts.separator 'Software Options'
+  opts.on(
+    '-b', '--bin DIR',
+    'Path to the directory containing the binaries of the search program'
+  ) { |v| o[:bin] = v }
+  opts.on(
+    '-p', '--program STR',
+    'Search program to be used',
+    'One of: blast+ (default), blast, diamond, blat'
+  ) { |v| o[:program] = v.downcase.to_sym }
+  opts.on(
+    '-t', '--threads INT', Integer,
+    'Number of parallel threads to be used',
+    "By default: #{o[:thr]}"
+  ) { |v| o[:thr] = v }
+  opts.separator ''
+  opts.separator 'Other Options'
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { $QUIET = true }
+  opts.on('-h', '--help', 'Display this screen') { puts opts ; exit }
+  opts.separator ''
+end.parse!
+
+raise Enveomics::OptionError.new('-1 is mandatory') if o[:seq1].nil?
+raise Enveomics::OptionError.new('-2 is mandatory') if o[:seq2].nil?
+raise Enveomics::OptionError.new(
+  'Argument -f/--fract requires -p blast+ or -p diamond'
+) if o[:fract] > 0.0 && !%i[blast+ diamond].include?(o[:program])
+
+rbm = Enveomics::RBM.new(o[:seq1], o[:seq2], o)
+ofh = writer(o[:out])
+rbm.each { |bm| ofh.puts bm.to_s }
+ofh.close
+
+say('Forward Best Matches: ', rbm.bms1.count)
+say('Reverse Best Matches: ', rbm.bms2.count)
+say('Reciprocal Best Matches: ', rbm.count)
+

data/utils/enveomics/Scripts/sam.filter.rb

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+$VERSION = 1.0
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+use 'shellwords'
+
+o = {
+  q: false, threads: 2, m_format: :sam, g_format: :fasta, identity: 95.0,
+  o: '-', header: true
+}
+
+OptionParser.new do |opt|
+  Enveomics.opt_banner(
+    opt, 'Filters a SAM or BAM file by target sequences and/or identity',
+    "#{File.basename($0)} -m map.sam -o filtered_map.sam [options]"
+  )
+
+  opt.separator 'Input/Output'
+  opt.on(
+    '-g', '--genome PATH',
+    'Genome assembly',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:g] = v }
+  opt.on(
+    '-m', '--mapping PATH',
+    'Mapping file',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:m] = v }
+  opt.on(
+    '-o', '--out-sam PATH',
+    'Output filtered file in SAM format',
+    'Supports compression with .gz extension, use - for STDOUT (default)'
+  ) { |v| o[:o] = v }
+  opt.separator ''
+
+  opt.separator 'Formats'
+  opt.on(
+    '--g-format STRING',
+    'Genome assembly format: fasta (default) or list'
+  ) { |v| o[:g_format] = v.downcase.to_sym }
+  opt.on(
+    '--m-format STRING',
+    'Mapping file format: sam (default) or bam',
+    'sam supports compression with .gz file extension'
+  ) { |v| o[:m_format] = v.downcase.to_sym }
+  opt.separator ''
+
+  opt.separator 'General'
+  opt.on(
+    '-i', '--identity FLOAT', Float,
+    "Set a fixed threshold of percent identity (default: #{o[:identity]})"
+  ) { |v| o[:identity] = v }
+  opt.on('--no-header', 'Do not include the headers') { |v| o[:header] = v }
+  opt.separator ''
+  opt.on(
+    '-t', '--threads INT', Integer, "Threads to use (default: #{o[:threads]})"
+  ) { |v| o[:threads] = v }
+  opt.on('-l', '--log PATH', 'Log file to save output') { |v| o[:log] = v }
+  opt.on('-q', '--quiet', 'Run quietly') { |v| o[:q] = v }
+  opt.on('-h', '--help', 'Display this screen') do
+    puts opt
+    exit
+  end
+  opt.separator ''
+end.parse!
+
+$QUIET = o[:q]
+
+# Functions
+
+##
+# Parses one line +ln+ in SAM format and outputs filtered lines to +ofh+
+# Filters by minimum +identity+ and +target+ sequences, and prints
+# the headers if +header+
+def parse_sam_line(ln, identity, target, header, ofh)
+  if ln =~ /^@/ || ln =~ /^\s*$/
+    ofh.puts ln if header
+    return
+  end
+
+  # No match
+  row = ln.chomp.split("\t")
+  return if row[2] == '*'
+
+  # Filter by target
+  return if !target.nil? && !target.include?(row[2])
+
+  # Exclude unless concordant or unaligned
+  length = row[9].size
+  row.shift(11) # Discard non-flag columns
+  flags = Hash[row.map { |i| i.sub(/:.:/, ':').split(':', 2) }]
+  return if flags['YT'] && !%w[CP UU].include?(flags['YT'])
+
+  # Filter by identity
+  unless flags['MD']
+    raise Enveomics::ParseError.new(
+      "SAM line missing MD flag:\n#{ln}\nFlags: #{flags}"
+    )
+  end
+  mismatches = flags['MD'].scan(/[^\d]/).count
+  id = 100.0 * (length - mismatches) / length
+  ofh.puts ln if id >= identity
+end
+
+# Reading targets
+if o[:g]
+  say 'Loading target sequences to filter'
+  reader = reader(o[:g])
+  target =
+    case o[:g_format]
+    when :fasta
+      reader.each.map { |ln| $1 if ln =~ /^>(\S+)/ }.compact
+    when :list
+      reader.each.map(&:chomp)
+    else
+      raise Enveomics::OptionError.new(
+        "Unsupported target sequences format: #{o[:g_format]}"
+      )
+    end
+  reader.close
+else
+  target = nil
+end
+
+# Reading and filtering mapping
+say 'Reading mapping file'
+ofh = writer(o[:o])
+case o[:m_format]
+when :sam
+  reader = reader(o[:m])
+  reader.each { |ln| parse_sam_line(ln, o[:identity], target, o[:header], ofh) }
+  reader.close
+when :bam
+  cmd = ['samtools', 'view', o[:m], '-@', o[:threads]]
+  cmd << '-h' if o[:header]
+  IO.popen(cmd.shelljoin) do |fh|
+    fh.each { |ln| parse_sam_line(ln, o[:identity], target, o[:header], ofh) }
+  end
+else
+  raise Enveomics::OptionError.new(
+    "Unsupported mapping format: #{o[:m_format]}"
+  )
+end
+ofh.close
+

data/utils/enveomics/Tests/Makefile

@@ -0,0 +1,10 @@
+
+# Makefile for the Enve-omics collection
+# @update Oct 13 2013
+# @author Luis M. Rodriguez-R <lmrodriguez at gmail dot com>
+
+include ../globals.mk
+
+all:
+	@echo NOTHING TO DO YET
+