BioDSL 1.0.0

data/lib/BioDSL/commands/index_taxonomy.rb

@@ -0,0 +1,226 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Create taxonomy index from sequences in the stream.
+  #
+  # +index_taxonomy+ is used to create a taxonomy index to allow subsequent
+  # taxonomic classification with +classify_seq+. The records with taxnomic
+  # information must contain :SEQ_NAME and :SEQ keys where the :SEQ_NAME value
+  # must be formatted with an initial ID number followed by a space and then the
+  # taxonomy string progressing from kingdom to species level. Only the
+  # following leves are accepted:
+  #
+  #  * K - kingdom
+  #  * P - phylum
+  #  * C - class
+  #  * O - order
+  #  * F - family
+  #  * G - genus
+  #  * S - species
+  #
+  # Truncated taxonomic strings are allowed, e.g. a string that stops at family
+  # level. Below is an example of a full taxonomic string:
+  #
+  #     32 K#Bacteria;P#Actinobacteria;C#Actinobacteria;O#Acidimicrobiales; \
+  #        F#Acidimicrobiaceae;G#Ferrimicrobium;S#Ferrimicrobium acidiphilum
+  #
+  # The resulting index consists of the following files (here using the default
+  # "taxonomy" as prefix) which are saved to a specified +output_dir+:
+  #
+  #  * taxonomy_tax_index.dat  - return node for a given node id.
+  #  * taxonomy_kmer_index.dat - return list of node ids for a given level and
+  #                              kmer.
+  #
+  # The index is constructed by breaking the sequences into kmers of a given
+  # kmer_size and using a given step_size:
+  #
+  # Example FASTA entry:
+  #
+  #    >2 K#Bacteria;P#Proteobacteria;C#Gammaproteobacteria;O#Vibrionales; \
+  #       F#Vibrionaceae;G#Vibrio;S#Vibrio
+  #    UCCUACGGGAGGCAGCAGUGGGGAAUAUUGCACAAUGGGCGCAAGCCUGAUGCAGCCAUGCCGCGUGUAUGA
+  #
+  # This sequence is broken down to a list of oligos using the default kmer_size
+  # and step_size of 8 and 1, respectively:
+  #
+  #    UCCUACGG
+  #     CCUACGGG
+  #      CUACGGGA
+  #       UACGGGAG
+  #        ACGGGAGG
+  #         ...
+  #
+  # Oligos containing ambiguity codes are skipped. Each oligo is encoded as an
+  # kmer (integer) by encoding two bits per nucletoide:
+  #
+  #  * A = 00
+  #  * U = 01
+  #  * C = 10
+  #  * G = 11
+  #
+  # E.g. UCCUACGG = 0110100100101111 = 26927
+  #
+  # For each node in the tree a vector is kept containing information of all
+  # observed oligos for that particular node. Thus all child nodes contain a
+  # subset of oligos compared to the parent node. Finally, the tree is saved to
+  # files.
+  #
+  # It should be noted that the speed and accuarcy of the classification is
+  # strongly dependent on the size and quality of the taxonomic database used
+  # (RDP, GreenGenes or Silva) and for a particular amplicon it is strongly
+  # recommended to create a slice from the database aligment matching the
+  # amplicon.
+  #
+  # == Usage
+  #
+  #    index_taxonomy(<output_dir: <dir>>[, kmer_size: <uint>
+  #                   [, step_size: <uint>[, prefix: <string>
+  #                   [, force: <bool>]]]])
+  #
+  # === Options
+  #
+  #  * output_dir: <dir> - Output directory to contain index files.
+  #  * kmer_size: <uint> - Size of kmer to use (default=8).
+  #  * step_size: <uint> - Size of steps (default=1).
+  #  * prefix: <string>  - Prefix to use with index file names
+  #                        (default="taxonomy").
+  #  * force: <bool>     - Force overwrite existing index files.
+  #
+  # == Examples
+  #
+  #    BP.new.
+  #    read_fasta(input: "RDP_11_Bacteria.fna").
+  #    index_taxonomy(output_dir: "RDP_11").
+  #    run
+  class IndexTaxonomy
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for IndexTaxonomy.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [String] :output_dir Path to output directory.
+    # @option options [String]  :prefix Database file name prefix.
+    # @option options [Integer] :kmer_size Kmer size to use for indexing.
+    # @option options [Integer] :step_size Step size to use for indexing.
+    # @option options [Boolean] :force Flag for force-overwriting output files.
+    #
+    # @return [IndexTaxonomy] Instance of class.
+    def initialize(options)
+      @options = options
+
+      defaults
+      check_options
+      create_output_dir
+      check_output_files
+
+      @index   = BioDSL::Taxonomy::Index.new(options)
+    end
+
+    # Return command lambda for index_taxonomy.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          add_to_index(record) if record[:SEQ_NAME] && record[:SEQ]
+
+          output << record
+          @status[:records_out] += 1
+        end
+
+        @index.save
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :output_dir, :kmer_size, :step_size, :prefix,
+                      :force)
+      options_required(@options, :output_dir)
+      options_allowed_values(@options, force: [nil, true, false])
+      options_files_exist_force(@options, :report)
+      options_assert(@options, ':kmer_size > 0')
+      options_assert(@options, ':kmer_size <= 12')
+      options_assert(@options, ':step_size > 0')
+      options_assert(@options, ':step_size <= 12')
+    end
+
+    # Set the default options hash values.
+    def defaults
+      @options[:prefix]    ||= 'taxonomy'
+      @options[:kmer_size] ||= 8
+      @options[:step_size] ||= 1
+    end
+
+    # Create the output directory specified in the options hash if this does not
+    # already exist.
+    def create_output_dir
+      return if File.exist?(@options[:output_dir])
+
+      FileUtils.mkdir_p(@options[:output_dir])
+    end
+
+    # Check if the output files already exist and throw an exception if so and
+    # the no force options is used.
+    #
+    # @raise [BioDSL::OptionsError] If file exists and force option not used.
+    def check_output_files
+      files = [
+        File.join(@options[:output_dir], "#{@options[:prefix]}_tax_index.dat"),
+        File.join(@options[:output_dir], "#{@options[:prefix]}_kmer_index.dat")
+      ]
+
+      files.each do |file|
+        next unless File.exist? file
+
+        unless @options[:force]
+          msg = "File exists: #{file} - use 'force: true' to overwrite"
+          fail BioDSL::OptionError, msg
+        end
+      end
+    end
+
+    # Add to the taxonomy index the sequence information from a given record.
+    #
+    # @param record [Hash] BioDSL record with sequence info.
+    def add_to_index(record)
+      @status[:sequences_in] += 1
+
+      _, seq_name = record[:SEQ_NAME].split(' ', 2)
+
+      @index.add(BioDSL::Seq.new(seq_name: seq_name, seq: record[:SEQ]))
+    end
+  end
+end

data/lib/BioDSL/commands/mask_seq.rb

@@ -0,0 +1,175 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Mask sequences in the stream based on quality scores.
+  #
+  # +mask_seq+ masks sequences in the stream using either hard masking or
+  # soft masking (default). Hard masking is replacing residues with
+  # corresponding quality score below a specified +quality_min+ with an N,
+  # while soft is replacing such residues with lower case. The sequences are
+  # values to SEQ keys and the quality scores are values to SCORES keys. The
+  # SCORES are encoded as ranges of ASCII characters from '!' to 'I'
+  # indicating scores from 0 to 40.
+  #
+  # == Usage
+  #
+  #    mask_seq([quality_min: <uint>[, mask: <:soft|:hard>]])
+  #
+  # === Options
+  #
+  # * quality_min: <uint> - Minimum quality (default=20).
+  # * mask: <string>      - Soft or Hard mask (default=soft).
+  #
+  # == Examples
+  #
+  # Consider the following FASTQ entry in the file test.fq:
+  #
+  #    @HWI-EAS157_20FFGAAXX:2:1:888:434
+  #    TTGGTCGCTCGCTCCGCGACCTCAGATCAGACGTGGGCGAT
+  #    +HWI-EAS157_20FFGAAXX:2:1:888:434
+  #    !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHI
+  #
+  # We can read in these sequence using +read_fastq+ and then soft mask the
+  # sequence with mask_seq like this:
+  #
+  #    BP.new.read_fastq(input: "test.fq").mask_seq.dump.run
+  #
+  #    {:SEQ_NAME=>"HWI-EAS157_20FFGAAXX:2:1:888:434",
+  #     :SEQ=>"ttggtcgctcgctccgcgacCTCAGATCAGACGTGGGCGAT",
+  #     :SEQ_LEN=>41,
+  #     :SCORES=>"!\"\#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHI"}
+  #
+  # Using the +quality_min+ option we can change the cutoff:
+  #
+  #    BP.new.read_fastq(input: "test.fq").mask_seq(quality_min: 25).dump.run
+  #
+  #    {:SEQ_NAME=>"HWI-EAS157_20FFGAAXX:2:1:888:434",
+  #     :SEQ=>"ttggtcgctcgctccgcgacctcagATCAGACGTGGGCGAT",
+  #     :SEQ_LEN=>41,
+  #     :SCORES=>"!\"\#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHI"}
+  #
+  # Using the +mask+ option for hard masking:
+  #
+  #    BP.new.read_fastq(input: "test.fq").mask_seq(mask: :hard).dump.run
+  #
+  #    {:SEQ_NAME=>"HWI-EAS157_20FFGAAXX:2:1:888:434",
+  #     :SEQ=>"NNNNNNNNNNNNNNNNNNNNCTCAGATCAGACGTGGGCGAT",
+  #     :SEQ_LEN=>41,
+  #     :SCORES=>"!\"\#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHI"}
+  class MaskSeq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out masked)
+
+    # Constructor for MaskSeq.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [Integer] Minimum quality score.
+    # @option options [Symbol,String] Mask scheme.
+    #
+    # @return [MaskSeq] Instance of MaskSeq.
+    def initialize(options)
+      @options = options
+
+      check_options
+      defaults
+
+      @mask = options[:mask].to_sym
+    end
+
+    # Return command lambda for mask_seq.
+    #
+    # @return [Proc] command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          mask_seq(record) if record[:SEQ] && record[:SCORES]
+
+          output << record
+
+          @status[:records_out] += 1
+        end
+
+        @status[:masked_percent] =
+          (100 * @status[:masked].to_f / @status[:residues_in]).round(2)
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :quality_min, :mask)
+      options_allowed_values(@options, mask: [:soft, :hard, 'soft', 'hard'])
+      options_assert(@options, ':quality_min >= 0')
+      options_assert(@options, ':quality_min <= 40')
+    end
+
+    # Set default options.
+    def defaults
+      @options[:quality_min] ||= 20
+      @options[:mask]        ||= :soft
+    end
+
+    # Mask sequence in given record.
+    #
+    # @param record [Hash] BioDSL record.
+    def mask_seq(record)
+      entry = BioDSL::Seq.new_bp(record)
+
+      @status[:sequences_in] += 1
+      @status[:residues_in]  += entry.length
+
+      @mask == :soft ? mask_seq_soft(entry) : mask_seq_hard(entry)
+
+      @status[:sequences_out] += 1
+      @status[:residues_out]  += entry.length
+
+      record.merge! entry.to_bp
+    end
+
+    # Soft mask sequences in given entry.
+    #
+    # @param entry [BioDSL::seq] sequences entry.
+    def mask_seq_soft(entry)
+      entry.mask_seq_soft!(@options[:quality_min])
+      @status[:masked] += entry.seq.count('a-z')
+    end
+
+    # Hard mask sequences in given entry.
+    #
+    # @param entry [BioDSL::seq] sequences entry.
+    def mask_seq_hard(entry)
+      entry.mask_seq_hard!(@options[:quality_min])
+      @status[:masked] += entry.seq.count('N')
+    end
+  end
+end

data/lib/BioDSL/commands/mean_scores.rb

@@ -0,0 +1,168 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Calculate the mean or local mean of quality SCORES in the stream.
+  #
+  # +mean_scores+ calculates either the global or local mean value or quality
+  # SCORES in the stream. The quality SCORES are encoded Phred style in
+  # character string.
+  #
+  # The global (default) behaviour calculates the SCORES_MEAN as the sum of all
+  # the scores over the length of the SCORES string.
+  #
+  # The local means SCORES_MEAN_LOCAL are calculated using means from a sliding
+  # window, where the smallest mean is returned.
+  #
+  # Thus, subquality records, with either an overall low mean quality or with
+  # local dip in quality, can be filtered using +grab+.
+  #
+  # == Usage
+  #
+  #    mean_scores([local: <bool>[, window_size: <uint>]])
+  #
+  # === Options
+  #
+  # * local:       <bool> - Calculate local mean score (default=false).
+  # * window_size: <uint> - Size of sliding window (defaul=5).
+  #
+  # == Examples
+  #
+  # Consider the following FASTQ entry in the file test.fq:
+  #
+  #    @HWI-EAS157_20FFGAAXX:2:1:888:434
+  #    TTGGTCGCTCGCTCGACCTCAGATCAGACGTGG
+  #    +
+  #    BCDEFGHIIIIIII,,,,,IFFIIIIIIIIIII
+  #
+  # The values of the scores in decimal are:
+  #
+  #    SCORES: 33;34;35;36;37;38;39;40;40;40;40;40;40;40;11;11;11;11;11;40;37;
+  #            37;40;40;40;40;40;40;40;40;40;40;40;
+  #
+  # To calculate the mean score do:
+  #
+  #    BP.new.read_fastq(input: "test.fq").mean_scores.dump.run
+  #
+  #    {:SEQ_NAME=>"HWI-EAS157_20FFGAAXX:2:1:888:434",
+  #     :SEQ=>"TTGGTCGCTCGCTCGACCTCAGATCAGACGTGG",
+  #     :SEQ_LEN=>33,
+  #     :SCORES=>"BCDEFGHIIIIIII,,,,,IFFIIIIIIIIIII",
+  #     :SCORES_MEAN=>34.58}
+  #
+  # To calculate local means for a sliding window, do:
+  #
+  #    BP.new.read_fastq(input: "test.fq").mean_scores(local: true).dump.run
+  #
+  #    {:SEQ_NAME=>"HWI-EAS157_20FFGAAXX:2:1:888:434",
+  #     :SEQ=>"TTGGTCGCTCGCTCGACCTCAGATCAGACGTGG",
+  #     :SEQ_LEN=>33,
+  #     :SCORES=>"BCDEFGHIIIIIII,,,,,IFFIIIIIIIIIII",
+  #     :SCORES_MEAN_LOCAL=>11.0}
+  #
+  # Which indicates a local minimum was located at the stretch of ,,,,, =
+  # 11+11+11+11+11 / 5 = 11.0
+  class MeanScores
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out min_mean max_mean mean_mean)
+
+    # Constructor for MeanScores.
+    #
+    # @param options  [Hash]    Options hash.
+    # @option options [Boolean] :local
+    # @option options [Fixnum]  :window_size
+    #
+    # @return [MeanScores] Class instance.
+    def initialize(options)
+      @options = options
+      @min     = Float::INFINITY
+      @max     = 0
+      @sum     = 0
+      @count   = 0
+
+      check_options
+      defaults
+    end
+
+    # Return command lambda for mean_scores.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          calc_mean(record) if record[:SCORES] && record[:SCORES].length > 0
+
+          output << record
+
+          @status[:records_out] += 1
+        end
+
+        @status[:mean_mean] = (@sum.to_f / @count).round(2)
+      end
+    end
+
+    private
+
+    # Check options
+    def check_options
+      options_allowed(@options, :local, :window_size)
+      options_tie(@options, window_size: :local)
+      options_allowed_values(@options, local: [true, false])
+      options_assert(@options, ':window_size > 1')
+    end
+
+    # Set default options.
+    def defaults
+      @options[:window_size] ||= 5
+    end
+
+    # Calculate the mean score for a given record and record
+    # count, sum, min and max.
+    #
+    # @param record [Hash] BioDSL record.
+    def calc_mean(record)
+      entry = BioDSL::Seq.new_bp(record)
+
+      if @options[:local]
+        mean = entry.scores_mean_local(@options[:window_size]).round(2)
+        record[:SCORES_MEAN_LOCAL] = mean
+      else
+        mean = entry.scores_mean.round(2)
+        record[:SCORES_MEAN] = mean
+      end
+
+      @sum   += mean
+      @status[:min_mean] = mean if mean < @status[:min_mean]
+      @status[:max_mean] = mean if mean > @status[:max_mean]
+      @count += 1
+    end
+  end
+end