miga-base 1.2.15.1 → 1.2.15.3

data/utils/enveomics/Scripts/gi2tax.rb

@@ -1,103 +0,0 @@
-#!/usr/bin/env ruby
-
-#
-# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @license artistic license 2.0
-#
-
-$:.push File.expand_path(File.dirname(__FILE__) + "/lib")
-require "enveomics_rb/remote_data"
-use "nokogiri"
-
-#================================[ Options parsing ]
-o = {
-  :q=>false, :gis=>[], :dbfrom=>"nuccore", :header=>true,
-  :exact_gi=>false, :no_nil=>false, :ret=>"ScientificName",
-  :ranks=>%w(superkingdom phylum class order family genus species)}
-OptionParser.new do |opt|
-  opt.banner = "
-  Maps a list of NCBI GIs to their corresponding taxonomy using the NCBI
-  EUtilities. Avoid using this script on millions of entries at a time, since
-  each entry elicits two requests to NCBI's servers.
-
-  *IMPORTANT NOTE*: NCBI is phasing out support for GIs. Please use acc.ver
-  instead with NCBIacc2tax.rb.
-
-  Usage: #{$0} [options]".gsub(/^ +/,"")
-  opt.separator ""
-  opt.on("-g", "--gis GI1,GI2,...", Array,
-    "Comma-separated list of GIs. Required unless -i is passed."
-    ){ |v| o[:gis]=v }
-  opt.on("-i", "--infile FILE",
-    "Raw text file containing the list of GIs, one per line.",
-    "Required unless -g is passed."){ |v| o[:infile]=v }
-  opt.on("-p", "--protein",
-    "Use if the GIs are proteins. Otherwise, GIs are assumed to be from " +
-    "the Nuccore Database."){ o[:dbfrom]="protein" }
-  opt.on("-r", "--ranks RANK1,RANK2,...", Array,
-    "Taxonomic ranks to report. By default: #{o[:ranks].join(",")}."
-    ){ |v| o[:ranks]=v }
-  opt.on("-n", "--noheader",
-    "Do not include a header in the output."){ o[:header]=false }
-  opt.on("-t", "--taxids",
-    "Return Taxonomy IDs instead of scientific names."){ o[:ret]="TaxId" }
-  opt.on("--exact-gi",
-    "Returns only taxonomy associated with the exact GI passed.",
-    "By default, it attempts to update accession versions if possible."
-    ){ |v| o[:exact_gi]=v }
-  opt.on("--ignore-missing",
-    "Does not report missing GIs in the output file.",
-    "By default, it reports GI and empty values for all other columns."
-    ){ |v| o[:no_nil]=v }
-  opt.on("-q", "--quiet", "Run quietly."){ |v| o[:q]=true }
-  opt.on("-h", "--help","Display this screen") do
-    puts opt
-    exit
-  end
-  opt.separator ""
-end.parse!
-
-#================================[ Functions ]
-def gi2taxid(db, gi)
-  doc = Nokogiri::XML( RemoteData.elink({:dbfrom=>db,
-    :db=>"taxonomy", :id=>gi}) )
-  doc.at_xpath("/eLinkResult/LinkSet/LinkSetDb/Link/Id")
-end
-#================================[ Main ]
-begin
-  o[:gis] += File.readlines(o[:infile]).map{ |l| l.chomp } unless
-    o[:infile].nil?
-  o[:ranks].map!{ |r| r.downcase }
-  puts (["GI", "TaxId"] + o[:ranks].map{ |r| r.capitalize }).join("\t") if
-    o[:header]
-  o[:gis].each do |gi|
-    taxid = gi2taxid(o[:dbfrom], gi)
-    status = ""
-    if taxid.nil? and not o[:exact_gi]
-      new_gi, status = RemoteData.update_gi(o[:dbfrom], gi)
-      taxid = gi2taxid(o[:dbfrom], new_gi) unless new_gi.nil?
-    end
-    if taxid.nil?
-      warn "Cannot find link to taxonomy: #{gi} #{status}"
-      puts ([gi, ""] + o[:ranks].map{ |i| "" }).join("\t") unless o[:no_nil]
-      next
-    end
-    taxonomy = {}
-    unless taxid.nil?
-      doc = Nokogiri::XML( RemoteData.efetch({:db=>"taxonomy",
-        :id=>taxid.content}) )
-      taxonomy[ doc.at_xpath("/TaxaSet/Taxon/Rank").content ] =
-        doc.at_xpath("/TaxaSet/Taxon/#{o[:ret]}").content
-      doc.xpath("/TaxaSet/Taxon/LineageEx/Taxon").each do |taxon|
-        taxonomy[ taxon.at_xpath("./Rank").content ] =
-          taxon.at_xpath("./#{o[:ret]}").content
-      end
-    end
-    puts ([gi, taxid.content] +
-      o[:ranks].map{ |rank| taxonomy[ rank ] ||= "" }).join("\t")
-  end
-rescue => err
-  $stderr.puts "Exception: #{err}\n\n"
-  err.backtrace.each { |l| $stderr.puts l + "\n" }
-  err
-end

data/utils/enveomics/Scripts/in_silico_GA_GI.pl

@@ -1,96 +0,0 @@
-# usage perl in_silico_GA.pl [options]
-
-use Getopt::Long;
-use Math::Random qw(:all);
-
-$argu=GetOptions('in=s'=>\$infile, # input fasta chr file
-                 'out=s'=>\$outfile, # output file name
-                 'coverage=s'=>\$cov, # desired output
-                 'seq_error=s'=>\$seq_error, # sequencing error
-                 'read_len=s'=>\$read_len, # simulated read length
-                 'ins_len=s'=>\$ins_len,  # insertion length
-                 'ins_var=s'=>\$ins_var);
-
-$chr='';
-open(IN,$infile);
-open(OUT,">$outfile");
-%code=();
-$code{'0'}='C';
-$code{'1'}='A';
-$code{'2'}='T';
-$code{'3'}='G';
-
-while(<IN>){
-  chomp;
-  if(!/^\>/){
-	$chr.=$_;
-  }
-  else{
-    $gi=$_;
-    if($gi= ~/^\>gi\|(\S+)\|\S+\|\S+/){
-      $gi=$1;}
-
-}
-}
-close(IN);
-  
-$chr_size=length $chr;
-print "chromosome size: $chr_size\n";
-$seg_size=2*$read_len+$ins_len;
-$reads_number=int($cov*$chr_size/($read_len*2));
-print "generated reads $reads_number x 2\n";
-
-for(1..$reads_number){
-  $index=$_;
-  $l=length $index;
-  $k=8-$l;
-  $kk='0' x $k;
-  $id= 'read'.$kk.$index.'_'.$gi;
-
-#make start site;
-  $start_site=int(rand($chr_size));
-#make short seg length;
-  $seg_length=int(random_normal(1,$seg_size,$ins_var));
-
-#extract the segment
-  $seg=substr($chr,$start_site,$seg_length);
-  $s_len=length $seg;
-  $gap=$seg_length-$s_len;
-  if($gap!=0){
-	$makeup=substr($chr,0,$gap);
-	$seg.=$makeup;
-  }
-
-  $id.='.start'.$start_site.'.seg_len'.$seg_length;
-
-#get the reads
-  $seq1=substr($seg,0,$read_len);
-  #$seg=~tr/ATCG/TAGC/ this line can change the orientation of the second read;
-  $seq2=substr($seg,-$read_len);
-# sequencing error introducing
-  @seq1=split(//,$seq1);
-  @seq2=split(//,$seq2);
-  @mut1=random_binomial($read_len,1,$seq_error);
-  @mut2=random_binomial($read_len,1,$seq_error);
-
-  for(0..$#mut1){
-	$i=$_;
-	if($mut1[$i]==1){
-	  $r=int(rand(4));
-	  $seq1[$i]=$code{$r};
-	}
-	if($mut2[$i]==1){
-	  $r=int(rand(4));
-	  $seq2[$i]=$code{$r};
-	}
-  }
-  $seq1=join('',@seq1);
-  $seq2=join('',@seq2);
-
-  $id1=$id.'#0/1';
-  $id2=$id.'#0/2';
-  
-  print OUT ">$id1\n$seq1\n>$id2\n$seq2\n";
-}
-	
-

data/utils/enveomics/Scripts/lib/enveomics.R

@@ -1 +0,0 @@
-../../enveomics.R

data/utils/enveomics/Scripts/lib/enveomics_rb/anir.rb

@@ -1,293 +0,0 @@
-# frozen_string_literal: true
-
-require 'enveomics_rb/stats'
-require 'fileutils'
-require 'shellwords'
-require 'tmpdir'
-require 'zlib'
-
-module Enveomics
-  # Wrapper class for ANIr estimation
-  #
-  # Use as: +ANIr.new(opts).go!+
-  class ANIr
-    # Options hash
-    attr :opts
-
-    # Identities list (unsorted)
-    attr :identities
-
-    def initialize(opts)
-      @opts = opts
-      @identities = []
-    end
-
-    # --------------------------------------------------[ High-level pipelines ]
-
-    # Perform all the analyses
-    def go!
-      read_input
-      detect_identity
-      estimate_ani_r
-    end
-
-    # Identify input/output mode and read mapping
-    def read_input
-      if opts[:m_format] != :list
-        @tmpdir = Dir.mktmpdir
-        @filter_contigs = !opts[:g].nil?
-        opts[:m] = File.join(@tmpdir, 'map.sam') if opts[:m].nil?
-        run_mapping unless File.exist? opts[:m]
-        load_contigs_to_filter if @filter_contigs
-      end
-      read_mapping = :"read_mapping_from_#{opts[:m_format]}"
-      raise Enveomics::OptionError.new(
-        "Unsupported mapping format: #{opts[:m_format]}"
-      ) unless respond_to? read_mapping
-      @identities = []
-      send(read_mapping)
-      say "- Unfiltered average identity: #{sample.mean}"
-      say "- Reads mapped: #{sample.n}"
-      save_identities
-      save_histogram
-    ensure
-      @tmpdir ||= nil
-      FileUtils.rm_rf @tmpdir if @tmpdir
-    end
-
-    # Identify the identity threshold
-    def detect_identity
-      say 'Detecting identity threshold'
-      if opts[:algorithm] == :auto
-        say "- Bimodality: #{bimodality}"
-        opts[:algorithm] = bimodality >= opts[:bimodality] ? :gmm : :fix
-      end
-      say "- Algorithm: #{opts[:algorithm]}"
-      if opts[:algorithm] == :gmm
-        detect_identity_by_gmm
-      end
-    end
-
-    # Estimate ANIr
-    def estimate_ani_r
-      say 'Estimating ANIr'
-      @sample = nil # Empty cached sample
-      @identities.delete_if { |i| i < opts[:identity] }
-      say "- ANIr: #{sample.mean}"
-    end
-
-    # -----------------------------------------------------------------[ Utils ]
-
-    # Show progress unless +opts[:q]+
-    def say(*msg)
-      o = '[%s] %s' % [Time.now, msg.join('')]
-      $stderr.puts(o) unless opts[:q]
-      File.open(opts[:log], 'a') { |fh| fh.puts o } if opts[:log]
-    end
-
-    # Execute command in the shell
-    def run(cmd)
-      say "  - Running: #{cmd.join(' ')}"
-      `#{cmd.shelljoin} 2>&1 | tee >> #{opts[:log] || '/dev/null'}`
-      unless $?.success?
-        raise Enveomics::CommandError.new("#{cmd.first} failed: #{$?}")
-      end
-    end
-
-    # Returns an open file handler for the file, supporting .gz
-    def reader(file)
-      file =~ /\.gz$/ ? Zlib::GzipReader.open(file) : File.open(file, 'r')
-    end
-
-    # Is the mapping in SAM format?
-    def sam?
-      opts[:m_format] == :sam
-    end
-
-    # ------------------------------------------------------------[ Map it out ]
-
-    # Execute Bowtie2 and generate SAM file
-    def run_mapping
-      say 'Running mapping using Bowtie2'
-      raise Enveomics::OptionError.new(
-        'Only SAM output is supported for mapping'
-      ) unless sam?
-
-      @filter_contigs = false
-      say '- Indexing input sequences'
-      raise Enveomics::OptionError.new(
-        'Only FastA genome input is supported for mapping'
-      ) unless opts[:g_format] == :fasta
-
-      idx = File.join(@tmpdir, 'genome.idx')
-      run(['bowtie2-build', opts[:g], idx])
-
-      say '- Mapping metagenomic reads to genome assembly'
-      cmd = [
-        'bowtie2', '-x', idx, '-p', opts[:threads], '-S', opts[:m], '--no-mixed'
-      ]
-      cmd << '-f' if opts[:r_format] == :fasta
-      cmd +=
-        case opts[:r_type]
-        when :single
-          ['-U', opts[:r]]
-        when :coupled
-          pairs = opts[:r].split(',', 2)
-          ['-1', pairs[0], '-2', pairs[1], '--no-discordant']
-        when :interleaved
-          ['--interleaved', opts[:r], '--no-discordant']
-        else
-          raise Enveomics::OptionError.new(
-            "Unsupported reads type: #{o[:r_type]}"
-          )
-        end
-      run(cmd)
-    end
-
-    # If +@filter_contigs+ is true, reads the genome assembly and saves contig
-    # names to filter the mapping
-    def load_contigs_to_filter
-      return unless @filter_contigs
-      say 'Loading contigs to filter'
-      reader = reader(opts[:g])
-      @contigs_to_filter =
-        case opts[:g_format]
-        when :fasta
-          reader.each.map { |ln| $1 if ln =~ /^>(\S+)/ }.compact
-        when :list
-          reader.each.map(&:chomp)
-        else
-          raise Enveomics::OptionError.new(
-            "Unsupported genome assembly format: #{opts[:g_format]}"
-          )
-        end
-      reader.close
-      say "- Got #{@contigs_to_filter.size} contigs"
-    end
-
-    # Reads the mapping file assuming SAM format
-    def read_mapping_from_sam
-      say 'Reading mapping from SAM file'
-      reader = reader(opts[:m])
-      reader.each { |ln| parse_sam_line(ln) }
-      reader.close
-    end
-
-    # Reads the mapping file assuming BAM format
-    def read_mapping_from_bam
-      say 'Reading mapping from BAM file'
-      IO.popen(['samtools', 'view', opts[:m]].shelljoin) do |fh|
-        fh.each { |ln| parse_sam_line(ln) }
-      end
-    end
-
-    # Reads the mapping file assuming a Tabular BLAST format
-    def read_mapping_from_tab
-      say 'Reading mapping from Tabular BLAST file'
-      reader = reader(opts[:m])
-      reader.each do |ln|
-        next if ln =~ /^\s*(#.*)?$/ # Comment or empty line
-        row = ln.chomp.split("\t")
-        next if @filter_contigs && !@contigs_to_filter.include?(row[1])
-        @identities << row[2].to_f
-      end
-      reader.close
-    end
-
-    # Reads the identities from a raw-text list
-    def read_mapping_from_list
-      say 'Reading identities from raw text list'
-      reader = reader(opts[:m])
-      @identities = reader.each.map(&:to_f)
-      reader.close
-    end
-
-    # Parses one line in SAM format
-    def parse_sam_line(ln)
-      return if ln =~ /^@/ || ln =~ /^\s*$/
-      row = ln.chomp.split("\t")
-      return if row[2] == '*'
-      return if @filter_contigs && !@contigs_to_filter.include?(row[2])
-      length = row[9].size
-      row.shift(11) # Discard non-flag columns
-      flags = Hash[row.map { |i| i.sub(/:.:/, ':').split(':', 2) }]
-      return if flags['YT'] && !%w[CP UU].include?(flags['YT'])
-      unless flags['MD']
-        raise Enveomics::ParseError.new(
-          "SAM line missing MD flag:\n#{ln}\nFlags: #{flags}"
-        )
-      end
-      mismatches = flags['MD'].scan(/[^\d]/).count
-      @identities << 100.0 * (length - mismatches) / length
-    end
-
-    # Save identites as raw text
-    def save_identities
-      return unless opts[:L]
-      say '- Saving identities'
-      File.open(opts[:L], 'w') do |fh|
-        identities.each { |i| fh.puts i }
-      end
-    end
-
-    # Save identity histogram as raw text
-    def save_histogram
-      return unless opts[:H]
-      say '- Saving histogram'
-      File.open(opts[:H], 'w') do |fh|
-        fh.puts "from\tto\tcount"
-        sample.histo_ranges.each_with_index do |r, k|
-          fh.puts (r + [sample.histo_counts[k]]).join("\t")
-        end
-      end
-    end
-
-    # -----------------------------------------------------------[ Peak finder ]
-
-    # Detect identity threshold by gaussian mixture model EM
-    def detect_identity_by_gmm
-      model_identities_by_gmm_em
-      detect_valley_by_gmm
-    end
-
-    # Model identities as a 2-gaussian mix by EM
-    def model_identities_by_gmm_em
-      say 'Modeling identities by gaussian mixture model using EM'
-      # TODO: Implement
-      raise Enveomics::UnimplementedError.new('Unimplemented operation')
-    end
-
-    # Detect valley by gaussian mix
-    def detect_valley_by_gmm
-      say 'Detecting valley by gaussian mixture model'
-      # TODO: Implement
-      raise Enveomics::UnimplementedError.new('Unimplemented operation')
-    end
-
-    # -----------------------------------------------------------[ Do the math ]
-
-    # Identities as a Enveomics::Stats::Sample object
-    def sample
-      @sample ||= Enveomics::Stats::Sample.new(
-        identities,
-        effective_range: [nil, 100.0],
-        histo_bin_size: opts[:bin_size]
-      )
-    end
-
-    # Returns the bimodality coefficient indicated by +opts[:coefficient]+
-    def bimodality
-      @bimodality ||=
-        case opts[:coefficient]
-        when :sarle
-          sample.sarle_bimodality
-        when :dma
-          sample.dma_bimodality
-        else
-          raise Enveomics::OptionError.new(
-            "Unsupported coefficient of bimodality: #{opts[:coefficient]}"
-          )
-        end
-    end
-  end
-end

data/utils/enveomics/Scripts/lib/enveomics_rb/bm_set.rb

@@ -1,175 +0,0 @@
-
-require 'enveomics_rb/enveomics'
-require 'enveomics_rb/match'
-use 'tmpdir'
-use 'shellwords'
-
-module Enveomics
-  class BMset
-    attr :qry, :sbj, :set, :opt
-
-    ##
-    # Initialize Enveomics::BMset object with sequence paths +qry+ and +sbj+,
-    # and options Hash +opts+ (see #opt for supported options) with Symbol keys
-    def initialize(qry, sbj, opts = {})
-      @qry = qry
-      @sbj = sbj
-      @set = nil
-      @opt = opts
-    end
-
-    ##
-    # Returns option with key +k+ as defined by #initialize or by default
-    # Supported options include [defaults in brackets]:
-    # - len   [0]:   Minimum alignment length in residues
-    # - id    [0.0]: Minimum alignment identity in percent
-    # - fract [0.0]: Minimum alignment length as fraction of the query
-    # - score [0.0]: Minimum alignment score in bits
-    # - nucl  [false]: The sequences are in nucleotides
-    # - thr   [1]:   Number of threads to use
-    # - bin   ['']:  Path to the directory containing binaries
-    # - program [:blast+]: Search engine to use
-    def opt(k)
-      @defaults ||= {
-        len: 0, id: 0.0, fract: 0.0, score: 0.0,
-        nucl: false, thr: 1, bin: '', program: :'blast+'
-      }
-      k = k.to_sym
-      @opt[k] = @defaults[k] if @opt[k].nil?
-      @opt[k]
-    end
-
-    ##
-    # Array of Enveomics::Match objects
-    def set
-      match_and_filter! if @set.nil?
-      @set
-    end
-
-    ##
-    # Returns the best match of query +qry+ as Enveomics::Match or nil if
-    # no qualifying match was found
-    def [](qry)
-      set[qry]
-    end
-
-    ##
-    # Number of matches found
-    def count
-      set.count
-    end
-
-    ##
-    # Execute search and filter matches 
-    def match_and_filter!
-      @set = {}
-      match!.each do |match|
-        # Already a better match?
-        next if self[match.qry] && self[match.qry].score >= match.score
-
-        # Is this a good enough match?
-        next unless %i[len id score fract].all? do |metric|
-          match.send(metric) >= opt(metric)
-        end
-
-        # Save match
-        @set[match.qry] = match
-      end
-    end
-
-    ##
-    # Find all matches and return as an array of Enveomics::Match objects
-    def match!
-      y = []
-      Dir.mktmpdir do |dir|
-        # Determine commands
-        say('Temporal directory: ', dir)
-        db_path = File.join(dir, 'sbj.db')
-        out_path = File.join(dir, 'out.tsv')
-        cmds = []
-        case opt(:program)
-        when :blast
-          cmds << [
-            'formatdb', '-i', sbj, '-n', db_path, '-l', File.join(dir, 'log'),
-            '-p', opt(:nucl) ? 'F' : 'T'
-          ]
-          cmd << [
-            'blastall', '-p', opt(:nucl) ? 'blastn' : 'blastp', '-d', db_path,
-            '-i', qry, '-v', '1', '-b', '1', '-a', opt(:thr).to_s, '-m', '8',
-            '-o', out_path
-          ]
-        when :'blast+'
-          cmds << [
-            'makeblastdb', '-in', sbj, '-out', db_path,
-            '-dbtype', opt(:nucl) ? 'nucl' : 'prot'
-          ]
-          cmds << [
-            opt(:nucl) ? 'blastn' : 'blastp', '-db', db_path, '-query', qry,
-            '-num_threads', opt(:thr).to_s, '-out', out_path, '-outfmt',
-            '6 qseqid sseqid pident length mismatch gapopen qstart qend ' \
-              'sstart send evalue bitscore qlen slen'
-          ]
-        when :diamond
-          raise Enveomics::OptionError.new(
-            'Unsupported search engine diamond for nucleotides'
-          ) if opt(:nucl)
-          cmds << [
-            'diamond', 'makedb', '--in', sbj, '--db', db_path,
-            '--threads', opt(:thr).to_s
-          ]
-          cmds << [
-            'diamond', 'blastp', '--threads', opt(:thr).to_s,
-            '--db', db_path, '--query', qry, '--daa', "#{out_path}.daa",
-            '--quiet', '--sensitive'
-          ]
-          cmds << [
-            'diamond', 'view', '--daa', "#{out_path}.daa", '--out', out_path,
-            '--quiet', '--outfmt'
-          ] + %w[6 qseqid sseqid pident length mismatch gapopen qstart] +
-              %w[qend sstart send evalue bitscore qlen slen]
-        when :blat
-          cmds << ['blat', sbj, qry, '-out=blast8', out_path]
-          cmds[0] << '-prot' unless opt(:nucl)
-        else
-          raise Enveomics::OptionError.new(
-            "Unsupported search engine: #{opt(:program)}"
-          )
-        end
-
-        # Run commands
-        say('Running comparison')
-        say('Query: ', qry)
-        say('Subject: ', sbj)
-        cmd_err = File.join(dir, 'err')
-        begin
-          cmds.each do |cmd|
-            cmd[0] = File.join(opt(:bin), cmd[0]) unless opt(:bin) == ''
-            run_cmd(cmd, stderr: cmd_err)
-          end
-        rescue Enveomics::CommandError => e
-          $stderr.puts e
-          $stderr.puts ''
-          $stderr.puts '[ Error log ]'
-          $stderr.puts File.read(cmd_err)
-          exit
-        end
-
-        # Parse output
-        File.open(out_path, 'r') do |fh|
-          fh.each { |ln| y << Enveomics::Match.new(ln) }
-        end
-      end
-      y
-    end
-
-    ##
-    # Enumerate RBMs and yield +blk+
-    def each(&blk)
-      if block_given?
-        set.each { |_, bm| blk.call(bm) }
-      else
-        to_enum(:each)
-      end
-    end
-  end
-end

data/utils/enveomics/Scripts/lib/enveomics_rb/enveomics.rb

@@ -1,24 +0,0 @@
-
-require 'enveomics_rb/utils'
-use 'optparse'
-ARGV << '-h' if ARGV.empty?
-
-module Enveomics
-  class << self
-    def opt_banner(opt, banner, usage = nil)
-      opt.version ||= $VERSION
-      usage ||= "#{opt.program_name}.rb [options]"
-      opt.banner = <<~BANNER
-
-        [Enveomics Collection: #{opt.program_name} #{opt.version}]
-
-        #{banner}
-
-        Usage
-          #{usage}
-
-      BANNER
-    end
-  end
-end
-