BioDSL 1.0.0

data/lib/BioDSL/commands/plot_scores.rb

@@ -0,0 +1,285 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+# rubocop: disable LineLength
+module BioDSL
+  # == Create a histogram with mean sequence quality scores.
+  #
+  # +plot_scores+ creates a histogram of the mean values per base of the quality
+  # scores from sequence data.
+  #
+  # Plotting is done using GNUplot which allows for different types of output
+  # the default one being crufty ASCII graphics.
+  #
+  # If plotting scores from sequences of variable length you can use the +count+
+  # option to co-plot the relative count at each base position. This allow you
+  # to detect areas with a low relative count showing a high mean score.
+  #
+  # GNUplot must be installed for plot_scores to work. Read more here:
+  #
+  # http://www.gnuplot.info/
+  #
+  # == Usage
+  #
+  #    plot_scores([count: <bool>[, output: <file>[, force: <bool>
+  #                [, terminal: <string>[, title: <string>
+  #                [, xlabel: <string>[, ylabel: <string>
+  #                [, test: <bool>]]]]]]]])
+  #
+  # === Options
+  #
+  # * count: <bool>      - Add line plot of relative counts.
+  # * output: <file>     - Output file.
+  # * force: <bool>      - Force overwrite existing output file.
+  # * terminal: <string> - Terminal for output: dumb|post|svg|x11|aqua|png|pdf
+  #                        (default=dumb).
+  # * title: <string>    - Plot title (default="Histogram").
+  # * xlabel: <string>   - X-axis label (default=<key>).
+  # * ylabel: <string>   - Y-axis label (default="n").
+  # * test: <bool>       - Output Gnuplot script instread of plot.
+  #
+  # == Examples
+  #
+  # Here we plot the mean quality scores from a FASTQ file:
+  #
+  #    read_fastq(input: "test.fq").plot_scores.run
+  #
+  #                                 Mean Quality Scores
+  #        +             +            +             +             +            +
+  #    40 ++-------------+------------+-------------+-------------+------------+++
+  #        |  *****************                               mean score ****** |
+  #    35 ++ ***********************                                            ++
+  #        ****************************** **                                    |
+  #    30 +*********************************   *                                ++
+  #        ************************************* *                              |
+  #    25 +*************************************** *                            ++
+  #        ****************************************** *****                     |
+  #    20 +****************************************************  ** * *         ++
+  #        ******************************************************************** *
+  #    15 +**********************************************************************+
+  #        **********************************************************************
+  #    10 +**********************************************************************+
+  #        **********************************************************************
+  #     5 +**********************************************************************+
+  #        **********************************************************************
+  #     0 +**********************************************************************+
+  #        +             +            +             +             +            +
+  #        0             50          100           150           200          250
+  #                                  Sequence position
+  #
+  # To render X11 output (i.e. instant view) use the +terminal+ option:
+  #
+  #    read_fastq(input: "test.fq").
+  #    plot_scores(terminal: :x11).run
+  #
+  # To generate a PNG image and save to file:
+  #
+  #    read_fastq(input: "test.fq").
+  #    plot_scores(terminal: :png, output: "plot.png").run
+  #
+  # rubocop: enable LineLength
+  # rubocop: disable ClassLength
+  class PlotScores
+    require 'gnuplotter'
+    require 'narray'
+    require 'BioDSL/helpers/aux_helper'
+
+    include AuxHelper
+
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    SCORES_MAX = 100_000   # Maximum score string length.
+
+    # Constructor for PlotScores.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [Boolean] :count
+    # @option options [String]  :output
+    # @option options [Boolean] :force
+    # @option options [Symbol]  :terminal
+    # @option options [String]  :title
+    # @option options [String]  :xlabel
+    # @option options [String]  :ylabel
+    # @option options [Boolean] :ylogscale
+    # @option options [Boolean] :test
+    #
+    # @return [PlotScores] Class instance.
+    def initialize(options)
+      @options    = options
+      @scores_vec = NArray.int(SCORES_MAX)
+      @count_vec  = NArray.int(SCORES_MAX)
+      @max        = 0
+
+      aux_exist('gnuplot')
+      check_options
+      default
+    end
+
+    # Return command lambda for plot_scores.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          collect_plot_data(record)
+
+          write_output(output, record)
+        end
+
+        prepare_plot_data
+
+        plot_defaults
+        plot_scores
+        plot_count
+        plot_output
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :count, :output, :force, :terminal, :title,
+                      :xlabel, :ylabel, :ylogscale, :test)
+      options_allowed_values(@options, count: [true, false])
+      options_allowed_values(@options, test: [true, false])
+      options_allowed_values(@options, terminal: [:dumb, :post, :svg, :x11,
+                                                  :aqua, :png, :pdf])
+      options_files_exist_force(@options, :output)
+    end
+
+    # Set default options.
+    def default
+      @options[:terminal] ||= :dumb
+      @options[:title]    ||= 'Mean Quality Scores'
+      @options[:xlabel]   ||= 'Sequence Position'
+      @options[:ylabel]   ||= 'Mean Score'
+    end
+
+    # Collect plot data from a given record.
+    #
+    # @param record [Hash] BioDSL record.
+    def collect_plot_data(record)
+      scores = record[:SCORES]
+      return unless scores && scores.length > 0
+
+      check_length(scores)
+
+      score_vec = NArray.to_na(scores, 'byte') - Seq::SCORE_BASE
+      @scores_vec[0...scores.length] += score_vec
+      @count_vec[0...scores.length]  += 1
+
+      @max = scores.length if scores.length > @max
+    end
+
+    # Check if the scores string is longer than SCORES_MAX.
+    #
+    # @raise [BioDSLError] if too long.
+    def check_length(scores)
+      return unless scores.length > SCORES_MAX
+      msg = "score string too long: #{scores.length} > #{SCORES_MAX}"
+      fail BioDSLError, msg
+    end
+
+    # Prepare data to plot.
+    def prepare_plot_data
+      @max = 1 if @max == 0   # ugly fix to avaid index error
+
+      count_vec  = @count_vec[0...@max].to_f
+      count_vec *= (Seq::SCORE_MAX / @count_vec.max(0).to_f)
+
+      @x  = (1..@max).to_a
+      @y1 = mean_vec.to_a
+      @y2 = count_vec.to_a
+    end
+
+    # Calculate the mean scores vector.
+    #
+    # @return [NArray] NArray with mean scores.
+    def mean_vec
+      @scores_vec[0...@max].to_f / @count_vec[0...@max]
+    end
+
+    # Set plot defaults
+    def plot_defaults
+      @gp = GnuPlotter.new
+      @gp.set terminal: @options[:terminal]
+      @gp.set title:    @options[:title]
+      @gp.set xlabel:   @options[:xlabel]
+      @gp.set ylabel:   @options[:ylabel]
+      @gp.set output:   @options[:output] if @options[:output]
+      @gp.set xrange:   "[#{@x.min - 1}:#{@x.max + 1}]"
+      @gp.set yrange:   "[#{Seq::SCORE_MIN}:#{Seq::SCORE_MAX}]"
+      @gp.set style:    'fill solid 0.5 border'
+      @gp.set xtics:    'out'
+      @gp.set ytics:    'out'
+    end
+
+    # Plot scores data.
+    def plot_scores
+      style = {with: 'boxes lc rgb "red"', title: '"mean score"'}
+
+      @gp.add_dataset(style) do |plotter|
+        @x.zip(@y1).each { |e| plotter << e }
+      end
+    end
+
+    # Plot count data.
+    def plot_count
+      return unless @options[:count]
+
+      style = {with: 'lines lt rgb "black"', title: '"relative count"'}
+
+      @gp.add_dataset(style) do |plotter|
+        @x.zip(@y2).each { |e| plotter << e }
+      end
+    end
+
+    # Output plot
+    def plot_output
+      if @options[:test]
+        $stderr.puts @gp.to_gp
+      elsif @options[:terminal] == :dumb
+        puts @gp.plot
+      else
+        @gp.plot
+      end
+    end
+
+    # Write record to output.
+    def write_output(output, record)
+      return unless output
+      output << record
+      @status[:records_out] += 1
+    end
+  end
+end

data/lib/BioDSL/commands/random.rb

@@ -0,0 +1,153 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Pick number of rand om records from the stream.
+  #
+  # +random+ can be used to pick a random number of records from the stream.
+  # Note that the order of records is preserved.
+  #
+  # Using the `pair: true` option allows random picking of interleaved
+  # paired-end sequence records.
+  #
+  # == Usage
+  #
+  #    random(<number: <uint>[, pairs: <bool>])
+  #
+  # === Options
+  #
+  # * number: <uint>  - Number of records to pick.
+  # * pairs: <bool>   - Preserve interleaved pair order.
+  #
+  # == Examples
+  #
+  # To pick some random records from the stream do:
+  #
+  #    BP.new.
+  #    read_fasta(input: "in.fna").
+  #    random(number: 10_000).
+  #    write_fasta(output: "out.fna").
+  #    run
+  class Random
+    STATS = %i(records_in records_out)
+
+    # Constructor for Randowm.
+    #
+    # @param options [Hash] Options hash.
+    #
+    # @option options [Fixnum]  :number
+    # @option options [Boolean] :pairs
+    #
+    # @return [Random] Class instance.
+    def initialize(options)
+      @options = options
+      @wanted  = nil
+
+      check_options
+    end
+
+    # Return command lambda for random.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        TmpDir.create('random') do |file, _|
+          process_input(input, file)
+          decide_wanted
+          process_output(output, file)
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :number, :pairs)
+      options_required(@options, :number)
+      options_allowed_values(@options, pairs: [nil, true, false])
+      options_assert(@options, ':number > 0')
+    end
+
+    # Serialize records from input
+    #
+    # @param input [Enumerator] Input stream.
+    # @param file [String] Path to temporary file.
+    def process_input(input, file)
+      File.open(file, 'wb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          input.each do |record|
+            @status[:records_in] += 1
+
+            s << record
+          end
+        end
+      end
+    end
+
+    # Compile a random set of numbers.
+    def decide_wanted
+      if @options[:pairs]
+        decide_wanted_pairs
+      else
+        @wanted =
+          (0...@status[:records_in]).to_a.shuffle[0...@options[:number]].to_set
+      end
+    end
+
+    # Compile a random set of number pairs.
+    def decide_wanted_pairs
+      @wanted = Set.new
+      range   = (0...@status[:records_in])
+      num     = @options[:number] / 2
+
+      range.to_a.shuffle.select(&:even?)[0...num].each do |i|
+        @wanted.merge([i, i + 1])
+      end
+    end
+
+    # Read records from temporary file and emit wanted records to the output
+    # stream.
+    #
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param file [String] Path to termorary file with records.
+    def process_output(output, file)
+      File.open(file, 'rb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          s.each_with_index do |record, i|
+            if @wanted.include? i
+              output << record
+              @status[:records_out] += 1
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/read_fasta.rb

@@ -0,0 +1,222 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Read FASTA entries from one or more files.
+  #
+  # +read_fasta+ read in sequence entries from FASTA files. Each sequence
+  # entry consists of a sequence name prefixed by a '>' followed by the sequence
+  # name on a line of its own, followed by one or my lines of sequence until the
+  # next entry or the end of the file. The resulting Biopiece record consists of
+  # the following record type:
+  #
+  #     {:SEQ_NAME=>"test",
+  #      :SEQ=>"AGCATCGACTAGCAGCATTT",
+  #      :SEQ_LEN=>20}
+  #
+  # Input files may be compressed with gzip og bzip2.
+  #
+  # For more about the FASTA format:
+  #
+  # http://en.wikipedia.org/wiki/Fasta_format
+  #
+  # == Usage
+  #    read_fasta(input: <glob>[, first: <uint>|last: <uint>])
+  #
+  # === Options
+  # * input <glob> - Input file or file glob expression.
+  # * first <uint> - Only read in the _first_ number of entries.
+  # * last <uint>  - Only read in the _last_ number of entries.
+  #
+  # == Examples
+  #
+  # To read all FASTA entries from a file:
+  #
+  #    read_fasta(input: "test.fna")
+  #
+  # To read all FASTA entries from a gzipped file:
+  #
+  #    read_fasta(input: "test.fna.gz")
+  #
+  # To read in only 10 records from a FASTA file:
+  #
+  #    read_fasta(input: "test.fna", first: 10)
+  #
+  # To read in the last 10 records from a FASTA file:
+  #
+  #    read_fasta(input: "test.fna", last: 10)
+  #
+  # To read all FASTA entries from multiple files:
+  #
+  #    read_fasta(input: "test1.fna,test2.fna")
+  #
+  # To read FASTA entries from multiple files using a glob expression:
+  #
+  #    read_fasta(input: "*.fna")
+  class ReadFasta
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for the ReadFasta class.
+    #
+    # @param [Hash] options Options hash.
+    # @option options [String, Array] :input String or Array with glob
+    #   expressions.
+    # @option options [Integer] :first Dump first number of records.
+    # @option options [Integer] :last  Dump last number of records.
+    #
+    # @return [ReadFasta] Returns an instance of the class.
+    def initialize(options)
+      @options = options
+      @count   = 0
+      @buffer  = []
+
+      check_options
+    end
+
+    # Return a lambda for the read_fasta command.
+    #
+    # @return [Proc] Returns the read_fasta command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        read_input(input, output)
+
+        options_glob(@options[:input]).each do |file|
+          BioDSL::Fasta.open(file) do |ios|
+            if @options[:first] && read_first(ios, output)
+            elsif @options[:last] && read_last(ios)
+            else
+              read_all(ios, output)
+            end
+          end
+        end
+
+        write_buffer(output) if @options[:last]
+      end
+    end
+
+    private
+
+    # Check the options.
+    def check_options
+      options_allowed(@options, :input, :first, :last)
+      options_required(@options, :input)
+      options_files_exist(@options, :input)
+      options_unique(@options, :first, :last)
+      options_assert(@options, ':first >= 0')
+      options_assert(@options, ':last >= 0')
+    end
+
+    # Read and emit records from the input to the output stream.
+    #
+    # @param input  [Enumerable::Yielder] Input stream.
+    # @param output [Enumerable::Yielder] Output stream.
+    def read_input(input, output)
+      return unless input
+
+      input.each do |record|
+        output << record
+        @status[:records_in]  += 1
+
+        if record[:SEQ]
+          @status[:sequences_in] += 1
+          @status[:residues_in]  += record[:SEQ].length
+        end
+      end
+    end
+
+    # Read in a specified number of entries from the input and emit to the
+    # output.
+    #
+    # @param input [BioDSL::Fasta] FASTA file input stream.
+    # @param output [Enumerable::Yielder] Output stream.
+    #
+    # @return [Fixnum] Number of read entries.
+    def read_first(input, output)
+      first = @options[:first]
+
+      input.each do |entry|
+        break if @count == first
+        output << entry.to_bp
+
+        @status[:records_out]   += 1
+        @status[:sequences_out] += 1
+        @status[:residues_out]  += entry.length
+
+        @count += 1
+      end
+
+      @count
+    end
+
+    # Read in entries from input and cache the specified last number in a
+    # buffer.
+    #
+    # @param input [BioDSL::Fasta] FASTA file input stream.
+    #
+    # @return [Fixnum] Number of read entries.
+    def read_last(input)
+      last = @options[:last]
+
+      input.each do |entry|
+        @buffer << entry
+        @buffer.shift if @buffer.size > last
+      end
+
+      @buffer.size
+    end
+
+    # Read in all entries from input and emit to output.
+    #
+    # @param input [BioDSL::Fasta] FASTA file input stream.
+    # @param output [Enumerable::Yielder] Output stream.
+    def read_all(input, output)
+      input.each do |entry|
+        output << entry.to_bp
+
+        @status[:records_out]   += 1
+        @status[:sequences_out] += 1
+        @status[:residues_out]  += entry.length
+      end
+    end
+
+    # Emit all entries in buffer to output.
+    #
+    # @param output [Enumerable::Yielder] Output stream.
+    def write_buffer(output)
+      @buffer.each do |entry|
+        output << entry.to_bp
+
+        @status[:records_out]   += 1
+        @status[:sequences_out] += 1
+        @status[:residues_out]  += entry.length
+      end
+    end
+  end
+end