BioDSL 1.0.0

data/lib/BioDSL/commands/classify_seq.rb

@@ -0,0 +1,217 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Classify sequences in the stream.
+  #
+  # +classify_seq+ searches sequences in the stream against a pre-indexed
+  # (using +index_taxonomy+) database. The database consists a taxonomic tree
+  # index and indices for each taxonomic level saved in the following files
+  # (here using the prefix "taxonomy"):
+  #
+  #  * taxonomy_tax_index.dat  - return node for a given node id.
+  #  * taxonomy_kmer_index.dat - return list of node ids for a given level
+  #                              and kmer.
+  #
+  # Each sequence is broken down into unique kmers of a given kmer_size
+  # overlapping with a given step_size - see +index_taxonomy+. Now, for each
+  # taxonomic level, starting from species all nodes for each kmer is looked
+  # up in the database. The nodes containing most kmers are considered hits.
+  # If there are no hits at a taxonomic level, we move to the next level. Hits
+  # are sorted according to how many kmers matched this particular node and a
+  # consensus taxonomy string is determined. Hits are also filtered with the
+  # following options:
+  #
+  #  * hits_max  - Include maximally this number of hits in the consensus.
+  #  * best_only - Include only the best scoring hits in the consensus.
+  #                That is if a hit consists of 344 kmers out of 345
+  #                possible, only hits with 344 kmers are included.
+  #  * coverage  - Filter hits based on kmer coverage. If a hit contains
+  #                fewer kmers than the total amount of kmers x coverage
+  #                it will be filtered.
+  #  * consensus - For a number of hits accept consensus at a given level
+  #                if within this percentage.
+  #
+  # The output of +classify_seq+ are sequence type records with the
+  # additional keys:
+  #
+  #  * TAXONOMY_HITS - The number of hits used in the consensus.
+  #  * TAXONOMY      - The taxonomy string.
+  #
+  # The consensus is determined from a list of taxonomic strings, i.e. the
+  # TAXONOMIC_HITS, and is composed of a consensus for each taxonomic level.
+  # E.g. for the kingdom level if 60% of the taxonomic strings indicate
+  # 'Bacteria' and the consensus is 50% then the consensus for the kingdom
+  # level will be reported as 'Bacteria(60)'. If the name at any level
+  # consists of multiple words they are treated independently. E.g if we have
+  # three taxonomic strings at the species level with the names:
+  #
+  #   *  Escherichia coli K-12
+  #   *  Escherichia coli sp. AC3432
+  #   *  Escherichia coli sp. AC1232
+  #
+  # The corresponding consensus for that level will be reported as
+  # 'Escherichia coli sp.(100/100/66)'. The forth word in the last two
+  # taxonomy strings (AC3432 and AC1232) have a consensus below 50% and are
+  # ignored.
+  #
+  # == Usage
+  #
+  #    classify_seq(<dir: <dir>>[, prefix: <string>[, kmer_size: <uint>
+  #                 [, step_size: <uint>[, hits_max: <uint>[, consensus:
+  #                 <float>[, coverage: <float>[, best_only: <bool>]]]]]]])
+  #
+  # === Options
+  #
+  # * dir:       <dir>    - Directory containing taxonomy files.
+  # * prefix:    <string> - Taxonomy files prefix (default="taxonomy").
+  # * kmer_size: <uint>   - Kmer size (default=8).
+  # * step_size: <uint>   - Step size (default=1).
+  # * hits_max:  <uint>   - Maximum hits to include in consensus (default=50).
+  # * consensus: <float>  - Consensus cutoff (default=0.51).
+  # * coverage:  <float>  - Coverate cutoff (default=0.9).
+  # * best_only: <bool>   - Only use best hits for consensus (default=true).
+  #
+  # == Examples
+  #
+  # To classify a bunch of OTU sequences in the file +otus.fna+ we do:
+  #
+  #    BP.new.
+  #    read_fasta(input: "otus.fna").
+  #    classify_seq(dir: "RDP11_3").
+  #    write_table(keys: [:SEQ_NAME, :TAXONOMY_HITS, :TAXONOMY]).
+  #    run
+  #
+  #    OTU_0  1 K#Bacteria(100);P#Proteobacteria(100);C#Gammaproteobacteria...
+  #    OTU_1  1 K#Bacteria(100);P#Proteobacteria(100);C#Gammaproteobacteria...
+  #    OTU_2  1 K#Bacteria(100);P#Proteobacteria(100);C#Gammaproteobacteria...
+  #    OTU_3  1 K#Bacteria(100);P#Proteobacteria(100);C#Gammaproteobacteria...
+  #    OTU_4  2 K#Bacteria(100);P#Fusobacteria(100);C#Fusobacteriia(100);O#...
+  class ClassifySeq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for the ClassifySeq class.
+    #
+    # @param [Hash] options Options hash.
+    # @option options [String]  :dir       Directory path with indexes.
+    # @option options [String]  :prefix    Index prefix.
+    # @option options [Integer] :kmer_size Kmer size.
+    # @option options [Integer] :step_size Step size.
+    # @option options [Integer] :hits_max  Max hits to report per sequence.
+    # @option options [Float]   :consensus Taxonomy string consensus percent.
+    # @option options [Float]   :coverage  Kmer coverage filter percent.
+    # @option options [Boolean] :best_only Flag to report best hit only.
+    #
+    # @return [ClassifySeq] Returns an instance of the class.
+    def initialize(options)
+      @options = options
+
+      check_options
+      defaults
+    end
+
+    # Return a lambda for the ClassifySeq command.
+    #
+    # @return [Proc] Returns the command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        @status[:sequences_in] = 0
+
+        search = BioDSL::Taxonomy::Search.new(@options)
+
+        input.each_with_index do |record, i|
+          @status[:records_in] += 1
+
+          classify_seq(record, i, search) if record.key? :SEQ
+
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :dir, :prefix, :kmer_size, :step_size,
+                      :hits_max, :consensus, :coverage, :best_only)
+      options_required(@options, :dir)
+      options_dirs_exist(@options, :dir)
+      options_allowed_values(@options, best_only: [nil, true, false])
+
+      run_assertions
+    end
+
+    # Run assertions.
+    def run_assertions
+      options_assert(@options, ':kmer_size > 0')
+      options_assert(@options, ':kmer_size <= 12')
+      options_assert(@options, ':step_size > 0')
+      options_assert(@options, ':step_size <= 12')
+      options_assert(@options, ':hits_max > 0')
+      options_assert(@options, ':consensus > 0')
+      options_assert(@options, ':consensus <= 1')
+      options_assert(@options, ':coverage > 0')
+      options_assert(@options, ':coverage <= 1')
+    end
+
+    # Set default options.
+    def defaults
+      @options[:prefix]    ||= 'taxonomy'
+      @options[:kmer_size] ||= 8
+      @options[:step_size] ||= 1
+      @options[:hits_max]  ||= 50
+      @options[:consensus] ||= 0.51
+      @options[:coverage]  ||= 0.9
+      @options[:best_only] = true if @options[:best_only].nil?
+    end
+
+    # Execute classfication of a sequence containing record.
+    #
+    # @param record [Hash]                        BioDSL record.
+    # @param i      [Fixnum]                      Record number,
+    # @param search [BioDSL::Taxonomy::Search] Search object.
+    def classify_seq(record, i, search)
+      @status[:sequences_in]  += 1
+      @status[:sequences_out] += 1
+      @status[:residues_in]   += record[:SEQ].length
+      @status[:residues_out]  += record[:SEQ].length
+      seq_name = record[:SEQ_NAME] || i.to_s
+
+      result = search.execute(BioDSL::Seq.new(seq_name: seq_name,
+                                                 seq: record[:SEQ]))
+
+      record[:TAXONOMY]      = result.taxonomy
+      record[:TAXONOMY_HITS] = result.hits
+      record[:RECORD_TYPE]   = 'taxonomy'
+    end
+  end
+end

data/lib/BioDSL/commands/classify_seq_mothur.rb

@@ -0,0 +1,226 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Run classify_seq_mothur on sequences in the stream.
+  #
+  # This is a wrapper for the +mothur+ command +classify.seqs()+. Basically,
+  # it classifies sequences in the stream given a database file and a taxonomy
+  # file which can be downloaded here:
+  #
+  # http://www.mothur.org/w/images/5/59/Trainset9_032012.pds.zip
+  #
+  # Please refer to the manual:
+  #
+  # http://www.mothur.org/wiki/Classify.seqs
+  #
+  # Mothur must be installed for +classify_seq_mothurs+ to work. Read more here:
+  #
+  # http://www.mothur.org/
+  #
+  # == Usage
+  #
+  #    classify_seq_mothur(<database: <file>>, <taxonomy: <file>>
+  #                        [, confidence: <uint>[, cpus: <uint>]])
+  #
+  # === Options
+  #
+  # * database:   <file> - Database to search.
+  # * taxonomy:   <file> - Taxonomy file for mapping names.
+  # * confidence: <uint> - Confidence threshold (defualt=80).
+  # * cpus:       <uint> - Number of CPU cores to use (default=1).
+  #
+  # == Examples
+  #
+  # To classify a bunch of OTU sequences in the file +otus.fna+ we do:
+  #
+  #    database = "trainset9_032012.pds.fasta"
+  #    taxonomy = "trainset9_032012.pds.tax"
+  #
+  #    BP.new.
+  #    read_fasta(input: "otus.fna").
+  #    classify_seq_mothur(database: database, taxonomy: taxonomy).
+  #    grab(exact: true, keys: :RECORD_TYPE, select: "taxonomy").
+  #    write_table(output: "classified.tab", header: true, force: true,
+  #                skip: [:RECORD_TYPE]).
+  #    run
+  class ClassifySeqMothur
+    require 'English'
+    require 'BioDSL/helpers/aux_helper'
+
+    include AuxHelper
+
+    STATS = %i(records_in records_out sequences_in sequences_out
+               residues_in residues_out)
+
+    # Constructor for ClassifySeqMothur.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [String] :database Path to database file.
+    # @option options [String] :taxonomy Path to taxonomy file.
+    # @option options [Integer] :confidence Confidence cutoff.
+    # @option options [Integer] :cpus Number of CPUs to use.
+    #
+    # @return [ClassifySeqMothur] Instance of class.
+    def initialize(options)
+      @options = options
+
+      aux_exist('mothur')
+      check_options
+      defaults
+    end
+
+    # Command lambda for ClassifySeqMothur.
+    #
+    # @return [Proc] Lambda for the command.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        TmpDir.create('input.fasta') do |tmp_in, tmp_dir|
+          process_input(input, output, tmp_in)
+          run_mothur(tmp_dir, tmp_in)
+          tmp_out = Dir.glob("#{tmp_dir}/input.*.taxonomy").first
+          process_output(output, tmp_out)
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :database, :taxonomy, :confidence, :cpus)
+      options_required(@options, :database, :taxonomy)
+      options_files_exist(@options, :database, :taxonomy)
+      options_assert(@options, ':confidence > 0')
+      options_assert(@options, ':confidence <= 100')
+      options_assert(@options, ':cpus >= 1')
+      options_assert(@options, ":cpus <= #{BioDSL::Config::CORES_MAX}")
+
+      defaults
+    end
+
+    # Set default options.
+    def defaults
+      @options[:confidence] ||= 80
+      @options[:cpus]       ||= 1
+    end
+
+    # Process input data and save sequences to a temporary file for
+    # classifcation.
+    #
+    # @param input  [Enumerator] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_in [String] Path to temporary FASTA file.
+    def process_input(input, output, tmp_in)
+      BioDSL::Fasta.open(tmp_in, 'w') do |ios|
+        input.each_with_index do |record, i|
+          @status[:records_in] += 1
+
+          if record[:SEQ]
+            @status[:sequences_in]  += 1
+            @status[:sequences_out] += 1
+            @status[:residues_in]   += record[:SEQ].length
+            @status[:records_out]   += record[:SEQ].length
+            seq_name = record[:SEQ_NAME] || i.to_s
+
+            entry = BioDSL::Seq.new(seq_name: seq_name, seq: record[:SEQ])
+
+            ios.puts entry.to_fasta
+          end
+
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    # Run Mothur using a system call.
+    #
+    # @param tmp_dir [String] Path to temporary dir.
+    # @param tmp_in  [String] Path to input file.
+    #
+    # @raise [RunTimeError] If system call fails.
+    def run_mothur(tmp_dir, tmp_in)
+      cmd = <<-CMD.gsub(/^\s+\|/, '').delete("\n")
+        |mothur "#set.dir(input=#{tmp_dir});
+        |set.dir(output=#{tmp_dir});
+        |classify.seqs(fasta=#{tmp_in},
+        |reference=#{@options[:database]},
+        |taxonomy=#{@options[:taxonomy]},
+        |method=wang,
+        |processors=#{@options[:cpus]})"
+      CMD
+
+      BioDSL.verbose ? system(cmd) : system("#{cmd} > /dev/null 2>&1")
+
+      fail 'Mothur failed' unless $CHILD_STATUS.success?
+    end
+
+    # Parse mothur classfication output and emit to stream.
+    #
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_out [String] Path to file with classfication result.
+    def process_output(output, tmp_out)
+      BioDSL::CSV.open(tmp_out) do |ios|
+        ios.each_hash do |new_record|
+          new_record[:SEQ_NAME] = new_record[:V0]
+          new_record[:TAXONOMY] = new_record[:V1]
+          new_record[:TAXONOMY].tr!('"', '')
+          new_record.delete(:V0)
+          new_record.delete(:V1)
+          new_record[:TAXONOMY]    = confidence_filter(new_record)
+          new_record[:RECORD_TYPE] = 'taxonomy'
+          output << new_record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    # Filter taxonomic leveles based on the confidence.
+    #
+    # @param record [Hash] BioDSL record with taxonomy.
+    #
+    # @return [String] Return taxonomic string.
+    def confidence_filter(record)
+      new_levels = []
+
+      record[:TAXONOMY].split(';').each do |level|
+        next unless level =~ /^([^(]+)\((\d+)\)$/
+        name       = Regexp.last_match(1)
+        confidence = Regexp.last_match(2).to_i
+
+        if confidence >= @options[:confidence]
+          new_levels << "#{name}(#{confidence})"
+        end
+      end
+
+      new_levels.empty? ? 'Unclassified' : new_levels.join(';')
+    end
+  end
+end