miga-base 1.2.17.0 → 1.2.17.2

data/utils/enveomics/Scripts/gi2tax.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+#
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+#
+
+$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
+require "enveomics_rb/remote_data"
+use "nokogiri"
+
+#================================[ Options parsing ]
+o = {
+  :q=>false, :gis=>[], :dbfrom=>"nuccore", :header=>true,
+  :exact_gi=>false, :no_nil=>false, :ret=>"ScientificName",
+  :ranks=>%w(superkingdom phylum class order family genus species)}
+OptionParser.new do |opt|
+  opt.banner = "
+  Maps a list of NCBI GIs to their corresponding taxonomy using the NCBI
+  EUtilities. Avoid using this script on millions of entries at a time, since
+  each entry elicits two requests to NCBI's servers.
+
+  *IMPORTANT NOTE*: NCBI is phasing out support for GIs. Please use acc.ver
+  instead with NCBIacc2tax.rb.
+
+  Usage: #{$0} [options]".gsub(/^ +/,"")
+  opt.separator ""
+  opt.on("-g", "--gis GI1,GI2,...", Array,
+    "Comma-separated list of GIs. Required unless -i is passed."
+    ){ |v| o[:gis]=v }
+  opt.on("-i", "--infile FILE",
+    "Raw text file containing the list of GIs, one per line.",
+    "Required unless -g is passed."){ |v| o[:infile]=v }
+  opt.on("-p", "--protein",
+    "Use if the GIs are proteins. Otherwise, GIs are assumed to be from " +
+    "the Nuccore Database."){ o[:dbfrom]="protein" }
+  opt.on("-r", "--ranks RANK1,RANK2,...", Array,
+    "Taxonomic ranks to report. By default: #{o[:ranks].join(",")}."
+    ){ |v| o[:ranks]=v }
+  opt.on("-n", "--noheader",
+    "Do not include a header in the output."){ o[:header]=false }
+  opt.on("-t", "--taxids",
+    "Return Taxonomy IDs instead of scientific names."){ o[:ret]="TaxId" }
+  opt.on("--exact-gi",
+    "Returns only taxonomy associated with the exact GI passed.",
+    "By default, it attempts to update accession versions if possible."
+    ){ |v| o[:exact_gi]=v }
+  opt.on("--ignore-missing",
+    "Does not report missing GIs in the output file.",
+    "By default, it reports GI and empty values for all other columns."
+    ){ |v| o[:no_nil]=v }
+  opt.on("-q", "--quiet", "Run quietly."){ |v| o[:q]=true }
+  opt.on("-h", "--help","Display this screen") do
+    puts opt
+    exit
+  end
+  opt.separator ""
+end.parse!
+
+#================================[ Functions ]
+def gi2taxid(db, gi)
+  doc = Nokogiri::XML( RemoteData.elink({:dbfrom=>db,
+    :db=>"taxonomy", :id=>gi}) )
+  doc.at_xpath("/eLinkResult/LinkSet/LinkSetDb/Link/Id")
+end
+#================================[ Main ]
+begin
+  o[:gis] += File.readlines(o[:infile]).map{ |l| l.chomp } unless
+    o[:infile].nil?
+  o[:ranks].map!{ |r| r.downcase }
+  puts (["GI", "TaxId"] + o[:ranks].map{ |r| r.capitalize }).join("\t") if
+    o[:header]
+  o[:gis].each do |gi|
+    taxid = gi2taxid(o[:dbfrom], gi)
+    status = ""
+    if taxid.nil? and not o[:exact_gi]
+      new_gi, status = RemoteData.update_gi(o[:dbfrom], gi)
+      taxid = gi2taxid(o[:dbfrom], new_gi) unless new_gi.nil?
+    end
+    if taxid.nil?
+      warn "Cannot find link to taxonomy: #{gi} #{status}"
+      puts ([gi, ""] + o[:ranks].map{ |i| "" }).join("\t") unless o[:no_nil]
+      next
+    end
+    taxonomy = {}
+    unless taxid.nil?
+      doc = Nokogiri::XML( RemoteData.efetch({:db=>"taxonomy",
+        :id=>taxid.content}) )
+      taxonomy[ doc.at_xpath("/TaxaSet/Taxon/Rank").content ] =
+        doc.at_xpath("/TaxaSet/Taxon/#{o[:ret]}").content
+      doc.xpath("/TaxaSet/Taxon/LineageEx/Taxon").each do |taxon|
+        taxonomy[ taxon.at_xpath("./Rank").content ] =
+          taxon.at_xpath("./#{o[:ret]}").content
+      end
+    end
+    puts ([gi, taxid.content] +
+      o[:ranks].map{ |rank| taxonomy[ rank ] ||= "" }).join("\t")
+  end
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.each { |l| $stderr.puts l + "\n" }
+  err
+end

data/utils/enveomics/Scripts/in_silico_GA_GI.pl

@@ -0,0 +1,96 @@
+# usage perl in_silico_GA.pl [options]
+
+use Getopt::Long;
+use Math::Random qw(:all);
+
+$argu=GetOptions('in=s'=>\$infile, # input fasta chr file
+                 'out=s'=>\$outfile, # output file name
+                 'coverage=s'=>\$cov, # desired output
+                 'seq_error=s'=>\$seq_error, # sequencing error
+                 'read_len=s'=>\$read_len, # simulated read length
+                 'ins_len=s'=>\$ins_len,  # insertion length
+                 'ins_var=s'=>\$ins_var);
+
+$chr='';
+open(IN,$infile);
+open(OUT,">$outfile");
+%code=();
+$code{'0'}='C';
+$code{'1'}='A';
+$code{'2'}='T';
+$code{'3'}='G';
+
+while(<IN>){
+  chomp;
+  if(!/^\>/){
+	$chr.=$_;
+  }
+  else{
+    $gi=$_;
+    if($gi= ~/^\>gi\|(\S+)\|\S+\|\S+/){
+      $gi=$1;}
+
+}
+}
+close(IN);
+  
+$chr_size=length $chr;
+print "chromosome size: $chr_size\n";
+$seg_size=2*$read_len+$ins_len;
+$reads_number=int($cov*$chr_size/($read_len*2));
+print "generated reads $reads_number x 2\n";
+
+for(1..$reads_number){
+  $index=$_;
+  $l=length $index;
+  $k=8-$l;
+  $kk='0' x $k;
+  $id= 'read'.$kk.$index.'_'.$gi;
+
+#make start site;
+  $start_site=int(rand($chr_size));
+#make short seg length;
+  $seg_length=int(random_normal(1,$seg_size,$ins_var));
+
+#extract the segment
+  $seg=substr($chr,$start_site,$seg_length);
+  $s_len=length $seg;
+  $gap=$seg_length-$s_len;
+  if($gap!=0){
+	$makeup=substr($chr,0,$gap);
+	$seg.=$makeup;
+  }
+
+  $id.='.start'.$start_site.'.seg_len'.$seg_length;
+
+#get the reads
+  $seq1=substr($seg,0,$read_len);
+  #$seg=~tr/ATCG/TAGC/ this line can change the orientation of the second read;
+  $seq2=substr($seg,-$read_len);
+# sequencing error introducing
+  @seq1=split(//,$seq1);
+  @seq2=split(//,$seq2);
+  @mut1=random_binomial($read_len,1,$seq_error);
+  @mut2=random_binomial($read_len,1,$seq_error);
+
+  for(0..$#mut1){
+	$i=$_;
+	if($mut1[$i]==1){
+	  $r=int(rand(4));
+	  $seq1[$i]=$code{$r};
+	}
+	if($mut2[$i]==1){
+	  $r=int(rand(4));
+	  $seq2[$i]=$code{$r};
+	}
+  }
+  $seq1=join('',@seq1);
+  $seq2=join('',@seq2);
+
+  $id1=$id.'#0/1';
+  $id2=$id.'#0/2';
+  
+  print OUT ">$id1\n$seq1\n>$id2\n$seq2\n";
+}
+	
+

data/utils/enveomics/Scripts/lib/enveomics.R

@@ -0,0 +1 @@
+../../enveomics.R

data/utils/enveomics/Scripts/lib/enveomics_rb/anir.rb

@@ -0,0 +1,293 @@
+# frozen_string_literal: true
+
+require 'enveomics_rb/stats'
+require 'fileutils'
+require 'shellwords'
+require 'tmpdir'
+require 'zlib'
+
+module Enveomics
+  # Wrapper class for ANIr estimation
+  #
+  # Use as: +ANIr.new(opts).go!+
+  class ANIr
+    # Options hash
+    attr :opts
+
+    # Identities list (unsorted)
+    attr :identities
+
+    def initialize(opts)
+      @opts = opts
+      @identities = []
+    end
+
+    # --------------------------------------------------[ High-level pipelines ]
+
+    # Perform all the analyses
+    def go!
+      read_input
+      detect_identity
+      estimate_ani_r
+    end
+
+    # Identify input/output mode and read mapping
+    def read_input
+      if opts[:m_format] != :list
+        @tmpdir = Dir.mktmpdir
+        @filter_contigs = !opts[:g].nil?
+        opts[:m] = File.join(@tmpdir, 'map.sam') if opts[:m].nil?
+        run_mapping unless File.exist? opts[:m]
+        load_contigs_to_filter if @filter_contigs
+      end
+      read_mapping = :"read_mapping_from_#{opts[:m_format]}"
+      raise Enveomics::OptionError.new(
+        "Unsupported mapping format: #{opts[:m_format]}"
+      ) unless respond_to? read_mapping
+      @identities = []
+      send(read_mapping)
+      say "- Unfiltered average identity: #{sample.mean}"
+      say "- Reads mapped: #{sample.n}"
+      save_identities
+      save_histogram
+    ensure
+      @tmpdir ||= nil
+      FileUtils.rm_rf @tmpdir if @tmpdir
+    end
+
+    # Identify the identity threshold
+    def detect_identity
+      say 'Detecting identity threshold'
+      if opts[:algorithm] == :auto
+        say "- Bimodality: #{bimodality}"
+        opts[:algorithm] = bimodality >= opts[:bimodality] ? :gmm : :fix
+      end
+      say "- Algorithm: #{opts[:algorithm]}"
+      if opts[:algorithm] == :gmm
+        detect_identity_by_gmm
+      end
+    end
+
+    # Estimate ANIr
+    def estimate_ani_r
+      say 'Estimating ANIr'
+      @sample = nil # Empty cached sample
+      @identities.delete_if { |i| i < opts[:identity] }
+      say "- ANIr: #{sample.mean}"
+    end
+
+    # -----------------------------------------------------------------[ Utils ]
+
+    # Show progress unless +opts[:q]+
+    def say(*msg)
+      o = '[%s] %s' % [Time.now, msg.join('')]
+      $stderr.puts(o) unless opts[:q]
+      File.open(opts[:log], 'a') { |fh| fh.puts o } if opts[:log]
+    end
+
+    # Execute command in the shell
+    def run(cmd)
+      say "  - Running: #{cmd.join(' ')}"
+      `#{cmd.shelljoin} 2>&1 | tee >> #{opts[:log] || '/dev/null'}`
+      unless $?.success?
+        raise Enveomics::CommandError.new("#{cmd.first} failed: #{$?}")
+      end
+    end
+
+    # Returns an open file handler for the file, supporting .gz
+    def reader(file)
+      file =~ /\.gz$/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
+    end
+
+    # Is the mapping in SAM format?
+    def sam?
+      opts[:m_format] == :sam
+    end
+
+    # ------------------------------------------------------------[ Map it out ]
+
+    # Execute Bowtie2 and generate SAM file
+    def run_mapping
+      say 'Running mapping using Bowtie2'
+      raise Enveomics::OptionError.new(
+        'Only SAM output is supported for mapping'
+      ) unless sam?
+
+      @filter_contigs = false
+      say '- Indexing input sequences'
+      raise Enveomics::OptionError.new(
+        'Only FastA genome input is supported for mapping'
+      ) unless opts[:g_format] == :fasta
+
+      idx = File.join(@tmpdir, 'genome.idx')
+      run(['bowtie2-build', opts[:g], idx])
+
+      say '- Mapping metagenomic reads to genome assembly'
+      cmd = [
+        'bowtie2', '-x', idx, '-p', opts[:threads], '-S', opts[:m], '--no-mixed'
+      ]
+      cmd << '-f' if opts[:r_format] == :fasta
+      cmd +=
+        case opts[:r_type]
+        when :single
+          ['-U', opts[:r]]
+        when :coupled
+          pairs = opts[:r].split(',', 2)
+          ['-1', pairs[0], '-2', pairs[1], '--no-discordant']
+        when :interleaved
+          ['--interleaved', opts[:r], '--no-discordant']
+        else
+          raise Enveomics::OptionError.new(
+            "Unsupported reads type: #{o[:r_type]}"
+          )
+        end
+      run(cmd)
+    end
+
+    # If +@filter_contigs+ is true, reads the genome assembly and saves contig
+    # names to filter the mapping
+    def load_contigs_to_filter
+      return unless @filter_contigs
+      say 'Loading contigs to filter'
+      reader = reader(opts[:g])
+      @contigs_to_filter =
+        case opts[:g_format]
+        when :fasta
+          reader.each.map { |ln| $1 if ln =~ /^>(\S+)/ }.compact
+        when :list
+          reader.each.map(&:chomp)
+        else
+          raise Enveomics::OptionError.new(
+            "Unsupported genome assembly format: #{opts[:g_format]}"
+          )
+        end
+      reader.close
+      say "- Got #{@contigs_to_filter.size} contigs"
+    end
+
+    # Reads the mapping file assuming SAM format
+    def read_mapping_from_sam
+      say 'Reading mapping from SAM file'
+      reader = reader(opts[:m])
+      reader.each { |ln| parse_sam_line(ln) }
+      reader.close
+    end
+
+    # Reads the mapping file assuming BAM format
+    def read_mapping_from_bam
+      say 'Reading mapping from BAM file'
+      IO.popen(['samtools', 'view', opts[:m]].shelljoin) do |fh|
+        fh.each { |ln| parse_sam_line(ln) }
+      end
+    end
+
+    # Reads the mapping file assuming a Tabular BLAST format
+    def read_mapping_from_tab
+      say 'Reading mapping from Tabular BLAST file'
+      reader = reader(opts[:m])
+      reader.each do |ln|
+        next if ln =~ /^\s*(#.*)?$/ # Comment or empty line
+        row = ln.chomp.split("\t")
+        next if @filter_contigs && !@contigs_to_filter.include?(row[1])
+        @identities << row[2].to_f
+      end
+      reader.close
+    end
+
+    # Reads the identities from a raw-text list
+    def read_mapping_from_list
+      say 'Reading identities from raw text list'
+      reader = reader(opts[:m])
+      @identities = reader.each.map(&:to_f)
+      reader.close
+    end
+
+    # Parses one line in SAM format
+    def parse_sam_line(ln)
+      return if ln =~ /^@/ || ln =~ /^\s*$/
+      row = ln.chomp.split("\t")
+      return if row[2] == '*'
+      return if @filter_contigs && !@contigs_to_filter.include?(row[2])
+      length = row[9].size
+      row.shift(11) # Discard non-flag columns
+      flags = Hash[row.map { |i| i.sub(/:.:/, ':').split(':', 2) }]
+      return if flags['YT'] && !%w[CP UU].include?(flags['YT'])
+      unless flags['MD']
+        raise Enveomics::ParseError.new(
+          "SAM line missing MD flag:\n#{ln}\nFlags: #{flags}"
+        )
+      end
+      mismatches = flags['MD'].scan(/[^\d]/).count
+      @identities << 100.0 * (length - mismatches) / length
+    end
+
+    # Save identites as raw text
+    def save_identities
+      return unless opts[:L]
+      say '- Saving identities'
+      File.open(opts[:L], 'w') do |fh|
+        identities.each { |i| fh.puts i }
+      end
+    end
+
+    # Save identity histogram as raw text
+    def save_histogram
+      return unless opts[:H]
+      say '- Saving histogram'
+      File.open(opts[:H], 'w') do |fh|
+        fh.puts "from\tto\tcount"
+        sample.histo_ranges.each_with_index do |r, k|
+          fh.puts (r + [sample.histo_counts[k]]).join("\t")
+        end
+      end
+    end
+
+    # -----------------------------------------------------------[ Peak finder ]
+
+    # Detect identity threshold by gaussian mixture model EM
+    def detect_identity_by_gmm
+      model_identities_by_gmm_em
+      detect_valley_by_gmm
+    end
+
+    # Model identities as a 2-gaussian mix by EM
+    def model_identities_by_gmm_em
+      say 'Modeling identities by gaussian mixture model using EM'
+      # TODO: Implement
+      raise Enveomics::UnimplementedError.new('Unimplemented operation')
+    end
+
+    # Detect valley by gaussian mix
+    def detect_valley_by_gmm
+      say 'Detecting valley by gaussian mixture model'
+      # TODO: Implement
+      raise Enveomics::UnimplementedError.new('Unimplemented operation')
+    end
+
+    # -----------------------------------------------------------[ Do the math ]
+
+    # Identities as a Enveomics::Stats::Sample object
+    def sample
+      @sample ||= Enveomics::Stats::Sample.new(
+        identities,
+        effective_range: [nil, 100.0],
+        histo_bin_size: opts[:bin_size]
+      )
+    end
+
+    # Returns the bimodality coefficient indicated by +opts[:coefficient]+
+    def bimodality
+      @bimodality ||=
+        case opts[:coefficient]
+        when :sarle
+          sample.sarle_bimodality
+        when :dma
+          sample.dma_bimodality
+        else
+          raise Enveomics::OptionError.new(
+            "Unsupported coefficient of bimodality: #{opts[:coefficient]}"
+          )
+        end
+    end
+  end
+end

data/utils/enveomics/Scripts/lib/enveomics_rb/bm_set.rb

@@ -0,0 +1,175 @@
+
+require 'enveomics_rb/enveomics'
+require 'enveomics_rb/match'
+use 'tmpdir'
+use 'shellwords'
+
+module Enveomics
+  class BMset
+    attr :qry, :sbj, :set, :opt
+
+    ##
+    # Initialize Enveomics::BMset object with sequence paths +qry+ and +sbj+,
+    # and options Hash +opts+ (see #opt for supported options) with Symbol keys
+    def initialize(qry, sbj, opts = {})
+      @qry = qry
+      @sbj = sbj
+      @set = nil
+      @opt = opts
+    end
+
+    ##
+    # Returns option with key +k+ as defined by #initialize or by default
+    # Supported options include [defaults in brackets]:
+    # - len   [0]:   Minimum alignment length in residues
+    # - id    [0.0]: Minimum alignment identity in percent
+    # - fract [0.0]: Minimum alignment length as fraction of the query
+    # - score [0.0]: Minimum alignment score in bits
+    # - nucl  [false]: The sequences are in nucleotides
+    # - thr   [1]:   Number of threads to use
+    # - bin   ['']:  Path to the directory containing binaries
+    # - program [:blast+]: Search engine to use
+    def opt(k)
+      @defaults ||= {
+        len: 0, id: 0.0, fract: 0.0, score: 0.0,
+        nucl: false, thr: 1, bin: '', program: :'blast+'
+      }
+      k = k.to_sym
+      @opt[k] = @defaults[k] if @opt[k].nil?
+      @opt[k]
+    end
+
+    ##
+    # Array of Enveomics::Match objects
+    def set
+      match_and_filter! if @set.nil?
+      @set
+    end
+
+    ##
+    # Returns the best match of query +qry+ as Enveomics::Match or nil if
+    # no qualifying match was found
+    def [](qry)
+      set[qry]
+    end
+
+    ##
+    # Number of matches found
+    def count
+      set.count
+    end
+
+    ##
+    # Execute search and filter matches 
+    def match_and_filter!
+      @set = {}
+      match!.each do |match|
+        # Already a better match?
+        next if self[match.qry] && self[match.qry].score >= match.score
+
+        # Is this a good enough match?
+        next unless %i[len id score fract].all? do |metric|
+          match.send(metric) >= opt(metric)
+        end
+
+        # Save match
+        @set[match.qry] = match
+      end
+    end
+
+    ##
+    # Find all matches and return as an array of Enveomics::Match objects
+    def match!
+      y = []
+      Dir.mktmpdir do |dir|
+        # Determine commands
+        say('Temporal directory: ', dir)
+        db_path = File.join(dir, 'sbj.db')
+        out_path = File.join(dir, 'out.tsv')
+        cmds = []
+        case opt(:program)
+        when :blast
+          cmds << [
+            'formatdb', '-i', sbj, '-n', db_path, '-l', File.join(dir, 'log'),
+            '-p', opt(:nucl) ? 'F' : 'T'
+          ]
+          cmd << [
+            'blastall', '-p', opt(:nucl) ? 'blastn' : 'blastp', '-d', db_path,
+            '-i', qry, '-v', '1', '-b', '1', '-a', opt(:thr).to_s, '-m', '8',
+            '-o', out_path
+          ]
+        when :'blast+'
+          cmds << [
+            'makeblastdb', '-in', sbj, '-out', db_path,
+            '-dbtype', opt(:nucl) ? 'nucl' : 'prot'
+          ]
+          cmds << [
+            opt(:nucl) ? 'blastn' : 'blastp', '-db', db_path, '-query', qry,
+            '-num_threads', opt(:thr).to_s, '-out', out_path, '-outfmt',
+            '6 qseqid sseqid pident length mismatch gapopen qstart qend ' \
+              'sstart send evalue bitscore qlen slen'
+          ]
+        when :diamond
+          raise Enveomics::OptionError.new(
+            'Unsupported search engine diamond for nucleotides'
+          ) if opt(:nucl)
+          cmds << [
+            'diamond', 'makedb', '--in', sbj, '--db', db_path,
+            '--threads', opt(:thr).to_s
+          ]
+          cmds << [
+            'diamond', 'blastp', '--threads', opt(:thr).to_s,
+            '--db', db_path, '--query', qry, '--daa', "#{out_path}.daa",
+            '--quiet', '--sensitive'
+          ]
+          cmds << [
+            'diamond', 'view', '--daa', "#{out_path}.daa", '--out', out_path,
+            '--quiet', '--outfmt'
+          ] + %w[6 qseqid sseqid pident length mismatch gapopen qstart] +
+              %w[qend sstart send evalue bitscore qlen slen]
+        when :blat
+          cmds << ['blat', sbj, qry, '-out=blast8', out_path]
+          cmds[0] << '-prot' unless opt(:nucl)
+        else
+          raise Enveomics::OptionError.new(
+            "Unsupported search engine: #{opt(:program)}"
+          )
+        end
+
+        # Run commands
+        say('Running comparison')
+        say('Query: ', qry)
+        say('Subject: ', sbj)
+        cmd_err = File.join(dir, 'err')
+        begin
+          cmds.each do |cmd|
+            cmd[0] = File.join(opt(:bin), cmd[0]) unless opt(:bin) == ''
+            run_cmd(cmd, stderr: cmd_err)
+          end
+        rescue Enveomics::CommandError => e
+          $stderr.puts e
+          $stderr.puts ''
+          $stderr.puts '[ Error log ]'
+          $stderr.puts File.read(cmd_err)
+          exit
+        end
+
+        # Parse output
+        File.open(out_path, 'r') do |fh|
+          fh.each { |ln| y << Enveomics::Match.new(ln) }
+        end
+      end
+      y
+    end
+
+    ##
+    # Enumerate RBMs and yield +blk+
+    def each(&blk)
+      if block_given?
+        set.each { |_, bm| blk.call(bm) }
+      else
+        to_enum(:each)
+      end
+    end
+  end
+end

data/utils/enveomics/Scripts/lib/enveomics_rb/enveomics.rb

@@ -0,0 +1,24 @@
+
+require 'enveomics_rb/utils'
+use 'optparse'
+ARGV << '-h' if ARGV.empty?
+
+module Enveomics
+  class << self
+    def opt_banner(opt, banner, usage = nil)
+      opt.version ||= $VERSION
+      usage ||= "#{opt.program_name}.rb [options]"
+      opt.banner = <<~BANNER
+
+        [Enveomics Collection: #{opt.program_name} #{opt.version}]
+
+        #{banner}
+
+        Usage
+          #{usage}
+
+      BANNER
+    end
+  end
+end
+