BioDSL 1.0.0

data/lib/BioDSL/commands/degap_seq.rb

@@ -0,0 +1,253 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Remove gaps from sequences or gap only columns in alignments.
+  #
+  # +degap_seq+ remove gaps from sequences (the letters ~-_.). If the option
+  # +columns_only+ is used then gaps from aligned sequences will be removed, if
+  # and only if the the entire columns consists of gaps.
+  #
+  # == Usage
+  #
+  #    degap_seq([columns_only: <bool>])
+  #
+  # === Options
+  #
+  # * columns_only: <bool> - Remove gap columns only (default=false).
+  #
+  # == Examples
+  #
+  # Consider the following FASTA entries in the file `test.fna`:
+  #
+  #    >test1
+  #    A-G~T.C_
+  #    >test2
+  #    AGG_T-C~
+  #
+  # To remove all gaps from all sequences do:
+  #
+  #    BP.new.read_fasta(input: "test.fna").degap_seq.dump.run
+  #
+  #    {:SEQ_NAME=>"test1", :SEQ=>"AGTC", :SEQ_LEN=>4}
+  #    {:SEQ_NAME=>"test2", :SEQ=>"AGGTC", :SEQ_LEN=>5}
+  #
+  #
+  # To remove all gap-only columns use the +columns_only+ option:
+  #
+  #    BP.new.
+  #    read_fasta(input: "test.fna").
+  #    degap_seq(columns_only: true).
+  #    dump.
+  #    run
+  #
+  #    {:SEQ_NAME=>"test1", :SEQ=>"A-GTC", :SEQ_LEN=>5}
+  #    {:SEQ_NAME=>"test2", :SEQ=>"AGGTC", :SEQ_LEN=>5}
+  #
+  # rubocop:disable ClassLength
+  class DegapSeq
+    require 'narray'
+
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for DegapSeq.
+    #
+    # @param options [Hash] Options Hash.
+    #
+    # @option options [Boolean] :columns_only
+    #   Flag indicating that only gap-columns only shoule be removed.
+    #
+    # @return [DegapSeq] Instance of DegapSeq.
+    def initialize(options)
+      @options = options
+      @indels  = BioDSL::Seq::INDELS.sort.join('')
+      @na_mask = nil
+      @max_len = nil
+      @count   = 0
+
+      check_options
+    end
+
+    # Return the command lambda for DegapSeq.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        if @options[:columns_only]
+          degap_columns(input, output)
+          status[:columns_removed] = @na_mask.count_false
+        else
+          degap_all(input, output)
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :columns_only)
+      options_allowed_values(@options, columns_only: [true, false, nil])
+    end
+
+    # Remove all gap-only columns from all sequences in input stream and output
+    # to output stream.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param output [Enumerator::Yeilder] Output stream.
+    def degap_columns(input, output)
+      TmpDir.create('degap_seq') do |tmp_file, _|
+        process_input(input, tmp_file)
+        create_mask
+        process_output(output, tmp_file)
+      end
+    end
+
+    # Serialize all input record to a temporary file and at the same time add
+    # all sequence type records to the gap mask.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param tmp_file [String] Path to temporary file.
+    def process_input(input, tmp_file)
+      File.open(tmp_file, 'wb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          input.each do |record|
+            @status[:records_in] += 1
+
+            if (seq = record[:SEQ])
+              mask_add(seq)
+              @count += 1
+            end
+
+            s << record
+          end
+        end
+      end
+    end
+
+    # Add sequence gaps to mask.
+    #
+    # @param seq [String] Sequences.
+    def mask_add(seq)
+      @status[:sequences_in] += 1
+      @status[:residues_in]  += seq.length
+
+      @max_len ||= seq.length
+
+      check_length(seq)
+
+      @na_mask ||= NArray.int(seq.length)
+      na_seq  = NArray.to_na(seq, 'byte')
+      @indels.each_char { |c| @na_mask += na_seq.eq(c.ord) }
+    end
+
+    # Check if sequence length match max_len.
+    #
+    # @param seq [String] Sequences.
+    #
+    # @raise [BioDSL::SeqError] if sequence length and max_len don't match.
+    def check_length(seq)
+      return if @max_len == seq.length
+      fail BioDSL::SeqError,
+           "Uneven seq lengths: #{@max_len} != #{seq.length}"
+    end
+
+    # Create a mask for all-gap columns.
+    def create_mask
+      @na_mask = @na_mask.ne @count
+    end
+
+    # Read all serialized records from the temporary file and emit to the output
+    # stream records with degapped sequences.
+    #
+    # @param output [Enumerator::Yeilder] Output stream.
+    # @param tmp_file [String] Path to temporary file.
+    def process_output(output, tmp_file)
+      File.open(tmp_file, 'rb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          s.each do |record|
+            remove_residues(record) if record[:SEQ]
+
+            output << record
+            @status[:records_out] += 1
+          end
+        end
+      end
+    end
+
+    # Given a BioDSL record containing sequence information
+    # remove all residues based on the na_mask.
+    #
+    # @param record [Hash] BioDSL record.
+    def remove_residues(record)
+      na_seq           = NArray.to_na(record[:SEQ], 'byte')
+      record[:SEQ]     = na_seq[@na_mask].to_s
+      record[:SEQ_LEN] = record[:SEQ].length
+
+      @status[:sequences_out] += 1
+      @status[:residues_out]  += record[:SEQ].length
+    end
+
+    # Remove all gaps from all sequences in input stream and output to output
+    # stream.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param output [Enumerator::Yeilder] Output stream.
+    def degap_all(input, output)
+      input.each do |record|
+        @status[:records_in] += 1
+
+        degap_seq(record) if record.key? :SEQ
+
+        output << record
+
+        @status[:records_out] += 1
+      end
+    end
+
+    # Given a BioDSL record with sequence information, remove all gaps from
+    # the sequence.
+    #
+    # @param record [Hash] BioDSL record.
+    def degap_seq(record)
+      entry = BioDSL::Seq.new_bp(record)
+
+      @status[:sequences_in] += 1
+      @status[:residues_in]  += entry.length
+
+      entry.seq.delete!(@indels)
+
+      @status[:sequences_out] += 1
+      @status[:residues_out]  += entry.length
+
+      record.merge! entry.to_bp
+    end
+  end
+end

data/lib/BioDSL/commands/dereplicate_seq.rb

@@ -0,0 +1,168 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Dereplicate sequences in the stream.
+  #
+  # +dereplicate_seq+ removes all duplicate sequence records. Dereplicated
+  # sequences are output along with the count of replicates. Using the
+  # +ignore_case+ option disables the default case sensitive sequence matching.
+  #
+  # == Usage
+  #
+  #    dereplicate_seq([ignore_case: <bool>])
+  #
+  # === Options
+  #
+  # * ignore_case: <bool> - Ignore sequence case.
+  #
+  # == Examples
+  #
+  # Consider the following FASTA file test.fna:
+  #
+  #    >test1
+  #    ATGC
+  #    >test2
+  #    ATGC
+  #    >test3
+  #    GCAT
+  #
+  # To dereplicate all sequences we use +read_fasta+ and +dereplicate_seq+:
+  #
+  #    BP.new.read_fasta(input: "test.fna").dereplicate_seq.dump.run
+  #
+  #    {:SEQ_NAME=>"test1", :SEQ=>"ATGC", :SEQ_LEN=>4, :SEQ_COUNT=>2}
+  #    {:SEQ_NAME=>"test3", :SEQ=>"GCAT", :SEQ_LEN=>4, :SEQ_COUNT=>1}
+  class DereplicateSeq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for the DereplicateSeq class.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [Boolean] :ignore_case Ignore sequence case.
+    #
+    # @return [DereplicateSeq] Class intance.
+    def initialize(options)
+      @options = options
+      @lookup  = {}
+
+      check_options
+    end
+
+    # Return the command lambda for DereplicateSeq.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        TmpDir.create('dereplicate_seq') do |tmp_file, _|
+          process_input(input, output, tmp_file)
+          process_output(output, tmp_file)
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :ignore_case)
+      options_allowed_values(@options, ignore_case: [nil, true, false])
+    end
+
+    # Process input stream and serialize all records with sequence information.
+    # All other records are emitted to the output stream.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_file [String] Path to temporary file.
+    def process_input(input, output, tmp_file)
+      File.open(tmp_file, 'wb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          input.each do |record|
+            @status[:records_in] += 1
+
+            if record.key? :SEQ
+              serialize(record, s)
+            else
+              output << record
+
+              @status[:records_out] += 1
+            end
+          end
+        end
+      end
+    end
+
+    # Serialize records with unique sequences and keep a count of how many time
+    # each sequence was encountered.
+    #
+    # @param record [Hash] BioDSL record.
+    # @param s [BioDSL::Serializer] Serializer.
+    def serialize(record, s)
+      @status[:sequences_in] += 1
+
+      seq = record[:SEQ].dup
+      @status[:residues_in] += seq.length
+      seq.downcase! if @options[:ignore_case]
+      key = seq.to_sym
+
+      unless @lookup[key]
+        s << record
+
+        @lookup[key] = 0
+      end
+
+      @lookup[key] += 1
+    end
+
+    # Read all serialized records from tmp file and emit to the output stream
+    # along with the sequence count.
+    #
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_file [String] Path to tmp file.
+    def process_output(output, tmp_file)
+      File.open(tmp_file, 'rb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          s.each do |record|
+            seq = record[:SEQ].dup
+            @status[:residues_out] += seq.length
+            seq.downcase! if @options[:ignore_case]
+            record[:SEQ_COUNT] = @lookup[seq.to_sym]
+
+            output << record
+
+            @status[:records_out] += 1
+            @status[:sequences_out] += 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/dump.rb

@@ -0,0 +1,157 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Dump records in stream to STDOUT.
+  #
+  # +dump+ outputs records from the stream to STDOUT.
+  #
+  # == Usage
+  #
+  #    dump([first: <uint> |last: <uint>])
+  #
+  # === Options
+  #
+  # * first <uint> - Only dump the first number of records.
+  # * last <uint>  - Only dump the last number of records.
+  #
+  # == Examples
+  #
+  # To dump all records in the stream:
+  #
+  #    dump
+  #
+  # To dump only the _first_ 10 records:
+  #
+  #    dump(first: 10)
+  #
+  # To dump only the _last_ 10 records:
+  #
+  #    dump(last: 10)
+  class Dump
+    STATS = %i(records_in records_out)
+
+    # Constructor for the Dump class.
+    #
+    # @param [Hash] options Options hash.
+    # @option options [Integer] :first Dump first number of records.
+    # @option options [Integer] :last  Dump last number of records.
+    #
+    # @return [Dump] Returns an instance of the Dump class.
+    def initialize(options)
+      @options     = options
+
+      check_options
+    end
+
+    # Return a lambda for the dump command.
+    #
+    # @return [Proc] Returns the dump command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        if @options[:first]
+          dump_first(input, output)
+        elsif @options[:last]
+          dump_last(input, output)
+        else
+          dump_all(input, output)
+        end
+      end
+    end
+
+    private
+
+    # Check the options and return a lambda for the command.
+    def check_options
+      options_allowed(@options, :first, :last)
+      options_unique(@options, :first, :last)
+      options_assert(@options, ':first > 0')
+      options_assert(@options, ':last > 0')
+    end
+
+    # Dump the first number of records.
+    #
+    # @param input [Enumerator::Yielder] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    def dump_first(input, output)
+      input.first(@options[:first]).each do |record|
+        @status[:records_in] += 1
+
+        puts record
+
+        if output
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    # Dump the last number of records.
+    #
+    # @param input [Enumerator::Yielder] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    def dump_last(input, output)
+      buffer = []
+      last   = @options[:last]
+
+      input.each do |record|
+        @status[:records_in] += 1
+
+        buffer << record
+        buffer.shift if buffer.size > last
+      end
+
+      buffer.each do |record|
+        puts record
+
+        if output
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    # Dump all records.
+    #
+    # @param input [Enumerator::Yielder] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    def dump_all(input, output)
+      input.each do |record|
+        @status[:records_in] += 1
+
+        puts record
+
+        if output
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+  end
+end