BioDSL 1.0.0

data/lib/BioDSL/commands/merge_pair_seq.rb

@@ -0,0 +1,175 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Merge pair-end sequences in the stream.
+  #
+  # +merge_pair_seq+ merges paired sequences in the stream, if these are
+  # interleaved. Sequence names must be in either Illumina1.3/1.5 format
+  # trailing a /1 or /2 or Illumina1.8 containing  1: or 2:. Sequence names must
+  # match accordingly in order to merge sequences.
+  #
+  # == Usage
+  #
+  #    merge_pair_seq
+  #
+  # === Options
+  #
+  # == Examples
+  #
+  # Consider the following FASTQ entry in the file test.fq:
+  #
+  #    @M01168:16:000000000-A1R9L:1:1101:14862:1868 1:N:0:14
+  #    TGGGGAATATTGGACAATGG
+  #    +
+  #    <??????BDDDDDDDDGGGG
+  #    @M01168:16:000000000-A1R9L:1:1101:14862:1868 2:N:0:14
+  #    CCTGTTTGCTACCCACGCTT
+  #    +
+  #    ?????BB<-<BDDDDDFEEF
+  #    @M01168:16:000000000-A1R9L:1:1101:13906:2139 1:N:0:14
+  #    TAGGGAATCTTGCACAATGG
+  #    +
+  #    <???9?BBBDBDDBDDFFFF
+  #    @M01168:16:000000000-A1R9L:1:1101:13906:2139 2:N:0:14
+  #    ACTCTTCGCTACCCATGCTT
+  #    +
+  #    ,5<??BB?DDABDBDDFFFF
+  #    @M01168:16:000000000-A1R9L:1:1101:14865:2158 1:N:0:14
+  #    TAGGGAATCTTGCACAATGG
+  #    +
+  #    ?????BBBBBDDBDDBFFFF
+  #    @M01168:16:000000000-A1R9L:1:1101:14865:2158 2:N:0:14
+  #    CCTCTTCGCTACCCATGCTT
+  #    +
+  #    ??,<??B?BB?BBBBBFF?F
+  #
+  # To merge these interleaved pair-end sequences use merge_pair_seq:
+  #
+  #    BP.new.
+  #    read_fastq(input: "test.fq", encoding: :base_33).
+  #    merge_pair_seq.
+  #    dump.
+  #    run
+  #
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14862:1868 1:N:0:14",
+  #     :SEQ=>"TGGGGAATATTGGACAATGGCCTGTTTGCTACCCACGCTT",
+  #     :SEQ_LEN=>40,
+  #     :SCORES=>"<??????BDDDDDDDDGGGG?????BB<-<BDDDDDFEEF",
+  #     :SEQ_LEN_LEFT=>20,
+  #     :SEQ_LEN_RIGHT=>20}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:13906:2139 1:N:0:14",
+  #     :SEQ=>"TAGGGAATCTTGCACAATGGACTCTTCGCTACCCATGCTT",
+  #     :SEQ_LEN=>40,
+  #     :SCORES=>"<???9?BBBDBDDBDDFFFF,5<??BB?DDABDBDDFFFF",
+  #     :SEQ_LEN_LEFT=>20,
+  #     :SEQ_LEN_RIGHT=>20}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14865:2158 1:N:0:14",
+  #     :SEQ=>"TAGGGAATCTTGCACAATGGCCTCTTCGCTACCCATGCTT",
+  #     :SEQ_LEN=>40,
+  #     :SCORES=>"?????BBBBBDDBDDBFFFF??,<??B?BB?BBBBBFF?F",
+  #     :SEQ_LEN_LEFT=>20,
+  #     :SEQ_LEN_RIGHT=>20}
+  class MergePairSeq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for MergePairSeq.
+    #
+    # @param options [Hash] Options hash.
+    #
+    # @return [MergePairSeq] Instance of MergePairSeq.
+    def initialize(options)
+      @options = options
+
+      check_options
+    end
+
+    # Return the command lambda for merge_pair_seq.
+    #
+    # @return [Proc] Command lambda for.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each_slice(2) do |record1, record2|
+          @status[:records_in] += record2 ? 2 : 1
+
+          if record1[:SEQ] && record2[:SEQ]
+            output << merge_pair_seq(record1, record2)
+
+            @status[:sequences_in]  += 2
+            @status[:sequences_out] += 1
+            @status[:records_out]   += 1
+          else
+            output.puts record1, record2
+
+            @status[:records_out] += 2
+          end
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, nil)
+    end
+
+    # Merge entry pair and return a new BioDSL record with this.
+    #
+    # @param record1 [Hash] BioDSL record 1.
+    # @param record2 [Hash] BioDSL record 2.
+    #
+    # @return [Hash] BioDSL record.
+    def merge_pair_seq(record1, record2)
+      entry1 = BioDSL::Seq.new_bp(record1)
+      entry2 = BioDSL::Seq.new_bp(record2)
+
+      BioDSL::Seq.check_name_pair(entry1, entry2)
+
+      @status[:residues_in] += entry1.length + entry2.length
+
+      length1 = entry1.length
+      length2 = entry2.length
+
+      entry1 << entry2
+
+      @status[:residues_out] += entry1.length
+
+      new_record(entry1, length1, length2)
+    end
+
+    def new_record(entry1, length1, length2)
+      new_record = entry1.to_bp
+      new_record[:SEQ_LEN_LEFT]  = length1
+      new_record[:SEQ_LEN_RIGHT] = length2
+      new_record
+    end
+  end
+end

data/lib/BioDSL/commands/merge_table.rb

@@ -0,0 +1,225 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Merge records on a given key with tabular data from one or more files.
+  #
+  # +merge_table+ reads in one or more tabular files and merges any records in
+  # the stream with identical values for a given key. The values for the given
+  # key must be unique in the tabular files, but not necesarily in the stream.
+  #
+  # Consult *read_table* for details on how the tabular files are read.
+  #
+  # The stats for +merge_table+ includes the following values:
+  #
+  # *  rows_total     - total number of table rows.
+  # *  rows_matched   - number of table rows with the given key.
+  # *  rows_unmatched - number of table rows without the given key.
+  # *  merged         - number of records that was merged.
+  # *  non_merged     - number of records that was not merged.
+  #
+  # == Usage
+  #    merge_table(<input: <glob>>, <key: <string>>[, columns: <list>
+  #                [, keys: <list>[, skip: <uint>[, delimiter: <string>]]]])
+  #
+  # === Options
+  # * input <glob>       - Input file or file glob expression.
+  # * key <string>       - Key used to merge
+  # * columns <list>     - List of columns to read in that order.
+  # * keys <list>        - List of key identifiers to use for each column.
+  # * skip <uint>        - Number of initial lines to skip (default=0).
+  # * delimiter <string> - Delimter to use for separating columsn
+  #                        (default="\s+").
+  #
+  # == Examples
+  #
+  # Consider the following two files:
+  #
+  # test1.tab:
+  #     #ID ORGANISM
+  #     1   parrot
+  #     2   eel
+  #     3   platypus
+  #     4   beetle
+  #
+  # test2.tab:
+  #
+  #     #ID COUNT
+  #     1   5423
+  #     2   34
+  #     3   2423
+  #     4   234
+  #
+  # We can merge the data with +merge_table+ like this:
+  #
+  #    BP.new.
+  #    read_table(input: "test1.tab").
+  #    merge_table(input: "test2.tab", key: :ID).
+  #    dump.
+  #    run
+  #
+  #    {:ID=>1, :ORGANISM=>"parrot", :COUNT=>5423}
+  #    {:ID=>2, :ORGANISM=>"eel", :COUNT=>34}
+  #    {:ID=>3, :ORGANISM=>"platypus", :COUNT=>2423}
+  #    {:ID=>4, :ORGANISM=>"beetle", :COUNT=>234}
+  class MergeTable
+    STATS = %i(records_in records_out rows_total rows_matched rows_unmatched
+               merged non_merged)
+
+    # Constructor for MergeTable.
+    #
+    # @param options [Hash]
+    #   Options hash.
+    #
+    # @option options [String] :input
+    #   Input glob expression.
+    #
+    # @option options [String, Symbol] :key
+    #   Key used to merge.
+    #
+    # @option options [Array] :keys
+    #   List of key identifiers to use for each column.
+    #
+    # @option options [Array] :columns
+    #   List of columns to read in that order.
+    #
+    # @option options [Integer] :skip
+    #   Number of initial lines to skip.
+    #
+    # @option options [String] :delimiter
+    #   Delimter to use for separating columns.
+    #
+    # @return [MergeTable] Class instance.
+    def initialize(options)
+      @options = options
+
+      check_options
+      defaults
+
+      @table = {}
+      @key   = @options[:key].to_sym
+      @keys  = options[:keys] ? @options[:keys].map(&:to_sym) : nil
+    end
+
+    # Return command lambda for merge_table.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        parse_input_tables
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          if record[@key] && @table[record[@key]]
+            @status[:merged] += 1
+            record = record.merge(@table[record[@key]])
+          else
+            @status[:non_merged] += 1
+          end
+
+          output << record
+          @status[:records_out] += 1
+        end
+
+        @status[:rows_total] = @status[:rows_matched] + @status[:rows_unmatched]
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :input, :key, :keys, :columns, :skip,
+                      :delimiter)
+      options_required(@options, :input, :key)
+      options_files_exist(@options, :input)
+      options_list_unique(@options, :keys, :columns)
+      options_assert(@options, ':skip >= 0')
+    end
+
+    # Set default options.
+    def defaults
+      @options[:skip] ||= 0
+    end
+
+    # Parse input table files and add each row to a table hash.
+    def parse_input_tables
+      options_glob(@options[:input]).each do |file|
+        BioDSL::CSV.open(file) do |ios|
+          ios.skip(@options[:skip])
+
+          ios.each_hash(delimiter: @options[:delimiter],
+                        select: @options[:columns]) do |record|
+            trim_record(record) if @keys
+
+            add_row(record)
+          end
+        end
+      end
+    end
+
+    # Trim given record removing unwanted key/values.
+    #
+    # @param record [Hash] BioDSL record.
+    def trim_record(record)
+      record.first(@keys.size).each_with_index do |(k, v), i|
+        record.delete(k)
+        record[@keys[i]] = v
+      end
+    end
+
+    # Add a given record to the table hash.
+    #
+    # @param record [Hash] BioDSL record.
+    #
+    # @raise [RuntimeError] if duplicate values are found.
+    def add_row(record)
+      if record[@key]
+        check_duplicate(record)
+
+        @status[:rows_matched] += 1
+
+        @table[record[@key]] = record
+      else
+        @status[:rows_unmatched] += 1
+      end
+    end
+
+    # Check if a given record is already added to the table and raise if so.
+    #
+    # @param record [Hash] BioDSL record.
+    #
+    # @raise [RuntimeError] if duplicate values are found.
+    def check_duplicate(record)
+      return unless @table[record[@key]]
+      fail "Duplicate values found for key: #{@key} value: #{record[@key]}"
+    end
+  end
+end

data/lib/BioDSL/commands/merge_values.rb

@@ -0,0 +1,113 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Merge values of specified keys.
+  #
+  # +merge_values+ merges the values of a list of keys using a given delimiter
+  # and saves the new value as the value of the first key.
+  #
+  # == Usage
+  #
+  #    merge_values(<keys: <list>>[, delimiter: <string>])
+  #
+  # === Options
+  #
+  # * keys:      <list>   - List of keys to merge.
+  # * delimiter: <string> - Delimiter (default='_').
+  #
+  # == Examples
+  #
+  # Consider the following record:
+  #
+  #    {ID: "FOO", COUNT: 10, SEQ: "gataag"}
+  #
+  # To merge the values so that the COUNT and ID is merged in that order do:
+  #
+  #    merge_values(keys: [:COUNT, :ID])
+  #
+  #    {:ID=>"FOO", :COUNT=>"10_FOO", :SEQ=>"gataag"}
+  #
+  # Changing the +delimiter+ and order:
+  #
+  #    merge_values(keys: [:ID, :COUNT], delimiter: ':count=')
+  #
+  #    {:ID=>"FOO:count=10", :COUNT=>10, :SEQ=>"gataag"}
+  class MergeValues
+    STATS = %i(records_in records_out)
+
+    # Constructor for MergeValues.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [Array] :keys Keys whos values to merge.
+    # @option options [String] :delimiter Delimiter for joining.
+    #
+    # @return [MergeValues] Class instance of MergeValues.
+    def initialize(options)
+      @options = options
+      check_options
+      defaults
+
+      @keys      = options[:keys]
+      @delimiter = options[:delimiter]
+    end
+
+    # Return command lambda for merge_values.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          if @keys.all? { |key| record.key? key }
+            values = @keys.inject([]) { |a, e| a << record[e.to_sym] }
+            record[@keys.first] = values.join(@delimiter)
+          end
+
+          output << record
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :keys, :delimiter)
+      options_required(@options, :keys)
+    end
+
+    # Set default options.
+    def defaults
+      @options[:delimiter] ||= '_'
+    end
+  end
+end