BioDSL 1.0.0

data/lib/BioDSL/commands/usearch_local.rb

@@ -0,0 +1,171 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Run usearch_local on sequences in the stream.
+  #
+  # This is a wrapper for the +usearch+ tool to run the program usearch_local.
+  # Basically sequence type records are searched against a reference database
+  # and records with hit information are output.
+  #
+  # Please refer to the manual:
+  #
+  # http://drive5.com/usearch/manual/cmd_usearch_local.html
+  #
+  # Usearch 7.0 must be installed for +usearch+ to work. Read more here:
+  #
+  # http://www.drive5.com/usearch/
+  #
+  # == Usage
+  #
+  #    usearch_local(<database: <file>, <identity: float>,
+  #                  <strand: "plus|both">[, cpus: <uint>])
+  #
+  # === Options
+  #
+  # * database: <file>   - Database to search (in FASTA format).
+  # * identity: <float>  - Similarity for matching in percent between 0.0 and
+  #                        1.0.
+  # * strand:   <string> - For nucleotide search report hits from plus or both
+  #                        strands.
+  # * cpus:     <uint>   - Number of CPU cores to use (default=1).
+  #
+  # == Examples
+  #
+  class UsearchLocal
+    require 'BioDSL/helpers/aux_helper'
+
+    include AuxHelper
+
+    STATS = %i(records_in records_out sequences_in hits_out)
+
+    # Constructor for UsearchLocal.
+    #
+    # @param  options [Hash] Options hash.
+    # @option options [String]        :database
+    # @option options [Float]         :identity
+    # @option options [String,Symbol] :strand
+    # @option options [Integer]       :cpus
+    #
+    # @return [UsearchLocal] Class instance.
+    def initialize(options)
+      @options          = options
+      @options[:cpus] ||= 1
+
+      aux_exist('usearch')
+      check_options
+    end
+
+    # Return command lambda for usearch_local.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        TmpDir.create('in', 'out') do |tmp_in, tmp_out|
+          process_input(input, output, tmp_in)
+          run_usearch_local(tmp_in, tmp_out)
+          process_output(output, tmp_out)
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :database, :identity, :strand, :cpus)
+      options_required(@options, :database, :identity)
+      options_allowed_values(@options, strand: ['plus', 'both', :plus, :both])
+      options_files_exist(@options, :database)
+      options_assert(@options, ':identity >  0.0')
+      options_assert(@options, ':identity <= 1.0')
+      options_assert(@options, ':cpus >= 1')
+      options_assert(@options, ":cpus <= #{BioDSL::Config::CORES_MAX}")
+    end
+
+    # Process input and emit to the output stream while saving all records
+    # containing sequences to a temporary FASTA file.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_in [String] Path to temporary file.
+    def process_input(input, output, tmp_in)
+      BioDSL::Fasta.open(tmp_in, 'w') do |ios|
+        input.each_with_index do |record, i|
+          @status[:records_in] += 1
+
+          output << record
+
+          @status[:records_out] += 1
+
+          next unless record[:SEQ]
+
+          @status[:sequences_in] += 1
+          seq_name = record[:SEQ_NAME] || i.to_s
+
+          entry = BioDSL::Seq.new(seq_name: seq_name, seq: record[:SEQ])
+
+          ios.puts entry.to_fasta
+        end
+      end
+    end
+
+    # Run usearch local on the input file and save results in the output file.
+    def run_usearch_local(tmp_in, tmp_out)
+      run_opts = {
+        input: tmp_in,
+        output: tmp_out,
+        database: @options[:database],
+        strand: @options[:strand],
+        identity: @options[:identity],
+        cpus: @options[:cpus],
+        verbose: @options[:verbose]
+      }
+
+      BioDSL::Usearch.usearch_local(run_opts)
+    rescue BioDSL::UsearchError => e
+      raise unless e.message =~ /Empty input file/
+    end
+
+    # Parse usearch output file and emit records to the output stream.
+    #
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_out [String] Path to output file.
+    def process_output(output, tmp_out)
+      BioDSL::Usearch.open(tmp_out) do |ios|
+        ios.each(:uc) do |record|
+          record[:RECORD_TYPE] = 'usearch'
+          output << record
+          @status[:hits_out]    += 1
+          @status[:records_out] += 1
+        end
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/write_fasta.rb

@@ -0,0 +1,207 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Write sequences from stream in FASTA format.
+  #
+  # Description
+  #
+  # +write_fasta+ writes sequence from the data stream in FASTA format.
+  # However, a FASTA entry will only be written if a SEQ key and a SEQ_NAME key
+  # is present. An example FASTA entry:
+  #
+  #     >test1
+  #     TATGACGCGCATCGACAGCAGCACGAGCATGCATCGACTG
+  #     TGCACTGACTACGAGCATCACTATATCATCATCATAATCT
+  #     TACGACATCTAGGGACTAC
+  #
+  # For more about the FASTA format:
+  #
+  # http://en.wikipedia.org/wiki/FASTA_format
+  #
+  # == Usage
+  #    write_fasta([wrap: <uin>[, output: <file>[, force: <bool>
+  #                [, gzip: <bool> | bzip2: <bool>]]]])
+  #
+  # === Options
+  # * output <file> - Output file.
+  # * force <bool>  - Force overwrite existing output file.
+  # * wrap <uint>   - Wrap sequence into lines of wrap length.
+  # * gzip <bool>   - Write gzipped output file.
+  # * bzip2 <bool>  - Write bzipped output file.
+  #
+  # == Examples
+  #
+  # To write FASTA entries to STDOUT.
+  #
+  #    write_fasta
+  #
+  # To write FASTA entries wrapped in lines of length of 80 to STDOUT.
+  #
+  #    write_fasta(wrap: 80)
+  #
+  # To write FASTA entries to a file 'test.fna'.
+  #
+  #    write_fasta(output: "test.fna")
+  #
+  # To overwrite output file if this exists use the force option:
+  #
+  #    write_fasta(output: "test.fna", force: true)
+  #
+  # To write gzipped FASTA entries to file 'test.fna.gz'.
+  #
+  #    write_fasta(output: "test.fna.gz", gzip: true)
+  #
+  # To write bzipped FASTA entries to file 'test.fna.bz2'.
+  #
+  #    write_fasta(output: "test.fna.bz2", bzip2: true)
+  class WriteFasta
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for the WriteFasta class.
+    #
+    # @param [Hash] options Options hash.
+    # @option options [Bool] :force Flag allowing overwriting files.
+    # @option options [String] :output Output file path.
+    # @option options [Integer] :wrap Wrap sequences at this length (default no
+    #   wrap)
+    # @option options [Bool] :gzip Output will be gzip'ed.
+    # @option options [Bool] :bzip2 Output will be bzip2'ed.
+    #
+    # @return [WriteFasta] Returns an instance of the class.
+    def initialize(options)
+      @options = options
+      check_options
+      @options[:output] ||= $stdout
+    end
+
+    # Return a lambda for the write_fasta command.
+    #
+    # @return [Proc] Returns the write_fasta command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        if @options[:output] == $stdout
+          write_stdout(input, output)
+        else
+          write_file(input, output)
+        end
+      end
+    end
+
+    private
+
+    # Check the options.
+    def check_options
+      options_allowed(@options, :force, :output, :wrap, :gzip, :bzip2)
+      options_unique(@options, :gzip, :bzip2)
+      options_tie(@options, gzip: :output, bzip2: :output)
+      options_files_exist_force(@options, :output)
+    end
+
+    # Write all sequence entries to stdout.
+    #
+    # @param input  [Enumerator]          The input stream.
+    # @param output [Enumerator::Yielder] The output stream.
+    def write_stdout(input, output)
+      wrap = @options[:wrap]
+
+      input.each do |record|
+        @status[:records_in] += 1
+
+        if (entry = record2entry(record))
+          $stdout.puts entry.to_fasta(wrap)
+          @status[:sequences_in]  += 1
+          @status[:sequences_out] += 1
+          @status[:residues_in]   += entry.length
+          @status[:residues_out]  += entry.length
+        end
+
+        write_output(output, record)
+      end
+    end
+
+    # rubocop: disable Metrics/AbcSize
+
+    # Write all sequence entries to a specified file.
+    #
+    # @param input  [Enumerator]          The input stream.
+    # @param output [Enumerator::Yielder] The output stream.
+    def write_file(input, output)
+      Fasta.open(@options[:output], 'w', compress: compress) do |ios|
+        input.each do |record|
+          @status[:records_in] += 1
+
+          if (entry = record2entry(record))
+            ios.puts entry.to_fasta(@options[:wrap])
+            @status[:sequences_in]  += 1
+            @status[:sequences_out] += 1
+            @status[:residues_in]   += entry.length
+            @status[:residues_out]  += entry.length
+          end
+
+          write_output(output, record)
+        end
+      end
+    end
+
+    # rubocop: enable Metrics/AbcSize
+
+    # Write a given record to the output stream if this exist.
+    #
+    # @param output [Enumerator::Yielder, nil] Output stream.
+    # @param record [Hash] Biopices record to write.
+    def write_output(output, record)
+      return unless output
+
+      output << record
+      @status[:records_out] += 1
+    end
+
+    # Creates a Seq object from a given record if SEQ_NAME and SEQ is present.
+    #
+    # @param record [Hash] Biopices record to convert.
+    #
+    # @return [BioDSL::Seq] Sequence entry.
+    def record2entry(record)
+      return unless record.key? :SEQ_NAME
+      return unless record.key? :SEQ
+
+      BioDSL::Seq.new_bp(record)
+    end
+
+    # Determine what compression should be used for output.
+    #
+    # @return [Symbol, nil] Compression flag or nil if no compression.
+    def compress
+      return :gzip  if @options[:gzip]
+      return :bzip2 if @options[:bzip2]
+    end
+  end
+end

data/lib/BioDSL/commands/write_fastq.rb

@@ -0,0 +1,191 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Write sequences from stream in FASTQ format.
+  #
+  # Description
+  #
+  # +write_fastq+ writes sequence from the data stream in FASTQ format. However,
+  # a FASTQ entry will only be written if a SEQ key and a SEQ_NAME key is
+  # present. An example FASTQ entry:
+  #
+  #     >test1
+  #     TATGACGCGCATCGACAGCAGCACGAGCATGCATCGACTG
+  #     TGCACTGACTACGAGCATCACTATATCATCATCATAATCT
+  #     TACGACATCTAGGGACTAC
+  #
+  # For more about the FASTQ format:
+  #
+  # http://en.wikipedia.org/wiki/FASTQ_format
+  #
+  # == Usage
+  #    write_fastq([encoding: <:base_33|:base_64>[, output: <file>
+  #                [, force: <bool>[, gzip: <bool> | bzip2: <bool>]]])
+  #
+  # === Options
+  # * encoding <base> - Encoding quality scores using :base_33 (default) or
+  #                     :base_64.
+  # * output <file>   - Output file.
+  # * force <bool>    - Force overwrite existing output file.
+  # * gzip <bool>     - Write gzipped output file.
+  # * bzip2 <bool>    - Write bzipped output file.
+  #
+  # == Examples
+  #
+  # To write FASTQ entries to STDOUT.
+  #
+  #    write_fastq
+  #
+  # To write FASTQ entries to a file 'test.fq'.
+  #
+  #    write_fastq(output: "test.fq")
+  #
+  # To overwrite output file if this exists use the force option:
+  #
+  #    write_fastq(output: "test.fq", force: true)
+  #
+  # To write gzipped FASTQ entries to file 'test.fq.gz'.
+  #
+  #    write_fastq(output: "test.fq.gz", gzip: true)
+  #
+  # To write bzipped FASTQ entries to file 'test.fq.bz2'.
+  #
+  #    write_fastq(output: "test.fq.bz2", bzip2: true)
+  class WriteFastq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for WriteFastq.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [String,Symbol] :encoding
+    # @option options [Boolean] :force
+    # @option options [String] :output
+    # @option options [Boolean] :gzip
+    # @option options [Boolean] :bzip2
+    #
+    # @return [WriteFastq] Class instance.
+    def initialize(options)
+      @options            = options
+      check_options
+      @options[:output] ||= $stdout
+      @compress           = choose_compression
+      @encoding           = choose_encoding
+    end
+
+    # Return command lambda for write_fastq.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        if @options[:output] == $stdout
+          process_input(input, output, $stdout)
+        else
+          Fastq.open(@options[:output], 'w', compress: @compress) do |ios|
+            process_input(input, output, ios)
+          end
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :encoding, :force, :output, :gzip, :bzip2)
+      options_allowed_values(@options, encoding: [:base_33, :base_64, 'base_33',
+                                                  'base_64'])
+      options_unique(@options, :gzip, :bzip2)
+      options_tie(@options, gzip: :output, bzip2: :output)
+      options_files_exist_force(@options, :output)
+    end
+
+    # Process all records in the input stream and output FASTQ data to the given
+    # ios, and finally emit all records to the output stream if specified.
+    #
+    # @param input  [Enumerable]                  Input stream.
+    # @param output [Enumerable::Yielder]         Output stream.
+    # @param ios    [BioDSL::Fastq::IO,STDOUT] Output IO.
+    def process_input(input, output, ios)
+      input.each do |record|
+        @status[:records_in] += 1
+
+        if record[:SEQ]
+          @status[:sequences_in] += 1
+          @status[:residues_in]  += record[:SEQ].length
+
+          write_fastq(record, ios) if record[:SEQ_NAME] && record[:SCORES]
+        end
+
+        if output
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    # Given a BioPeices record convert this to a sequence entry and output in
+    # FASTQ format to the speficied IO.
+    #
+    # @param record [Hash] BioDSL record.
+    # @param ios    [BioDSL::Fastq::IO,STDOUT] Output IO.
+    def write_fastq(record, ios)
+      entry = BioDSL::Seq.new_bp(record)
+      entry.qual_convert!(:base_33, @encoding)
+
+      ios.puts entry.to_fastq
+      @status[:sequences_out] += 1
+      @status[:residues_out]  += entry.length
+    end
+
+    # Choose compression to use which can either be gzip or bzip2 or no
+    # compression.
+    #
+    # @return [Symbol,nil] Compression.
+    def choose_compression
+      if @options[:gzip]
+        :gzip
+      elsif @options[:bzip2]
+        :bzip2
+      end
+    end
+
+    # Chose the quality score encoding.
+    #
+    # @return [Symbol,nil] Encoding.
+    def choose_encoding
+      if @options[:encoding]
+        @options[:encoding].to_sym
+      else
+        :base_33
+      end
+    end
+  end
+end