BioDSL 1.0.0

data/lib/BioDSL/commands/sort.rb

@@ -0,0 +1,223 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Sort records in the stream.
+  #
+  # +sort+ records in the stream given a specific key. Sorting on multiple keys
+  # is currently not supported.
+  #
+  # == Usage
+  #
+  #    sort(key: <value>[, reverse: <bool>[, block_size: <uint>]])
+  #
+  # === Options
+  #
+  # * key: <value>       - Sort records on the value for key.
+  # * reverse: <bool>    - Reverse sort.
+  # * block_size: <uint> - Block size used for disk based sorting
+  #                        (default=250_000_000).
+  #
+  # == Examples
+  #
+  # Consider the following table in the file `test.tab`:
+  #
+  #    #COUNT  ORGANISM
+  #    4 Dog
+  #    3 Cat
+  #    1 Eel
+  #
+  # To sort this accoring to COUNT in descending order do:
+  #
+  #    BP.new.read_table(input: "test.tab").sort(key: :COUNT).dump.run
+  #
+  #    {:COUNT=>1, :ORGANISM=>"Eel"}
+  #    {:COUNT=>3, :ORGANISM=>"Cat"}
+  #    {:COUNT=>4, :ORGANISM=>"Dog"}
+  #
+  # And in ascending order:
+  #
+  #    BP.new.
+  #    read_table(input: "test.tab").
+  #    sort(key: :COUNT, reverse: true).
+  #    dump.
+  #    run
+  #
+  #    {:COUNT=>4, :ORGANISM=>"Dog"}
+  #    {:COUNT=>3, :ORGANISM=>"Cat"}
+  #    {:COUNT=>1, :ORGANISM=>"Eel"}
+  #
+  # The type of value determines the sorting, alphabetical order:
+  #
+  #    BP.new.read_table(input: "test.tab").sort(key: :ORGANISM).dump.run
+  #
+  #    {:COUNT=>3, :ORGANISM=>"Cat"}
+  #    {:COUNT=>4, :ORGANISM=>"Dog"}
+  #    {:COUNT=>1, :ORGANISM=>"Eel"}
+  #
+  # And reverse alphabetic order:
+  #
+  #    BP.new.
+  #    read_table(input: "test.tab").
+  #    sort(key: :ORGANISM, reverse: true).
+  #    dump.
+  #    run
+  #
+  #    {:COUNT=>1, :ORGANISM=>"Eel"}
+  #    {:COUNT=>4, :ORGANISM=>"Dog"}
+  #    {:COUNT=>3, :ORGANISM=>"Cat"}
+  class Sort
+    require 'pqueue'
+
+    STATS           = %i(records_in records_out)
+    SORT_BLOCK_SIZE = 250_000_000 # max bytes to hold in memory.
+
+    # Constructor for Sort.
+    #
+    # @param options [Hash] Options hash.
+    #
+    # @option options [String,Symbol] :key
+    # @option options [Boolean]       :reverse
+    # @option options [Integer]       :block_size
+    #
+    # @return [Sort] Class instance.
+    def initialize(options)
+      @options    = options
+      @block_size = options[:block_size] || SORT_BLOCK_SIZE
+      @key        = options[:key].to_sym
+      @files      = []
+      @records    = []
+      @size       = 0
+      @pqueue     = pqueue_init
+      @fds        = nil
+
+      check_options
+    end
+
+    # Return command lambda for Sort.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+          @records << record
+          @size += record.to_s.size
+          save_block if @size > @block_size
+        end
+
+        save_block
+        open_block_files
+        fill_pqueue
+        output_pqueue(output)
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :key, :reverse, :block_size)
+      options_required(@options, :key)
+      options_allowed_values(@options, reverse: [nil, true, false])
+      options_assert(@options, ':block_size >  0')
+    end
+
+    # Initialize pqueue
+    def pqueue_init
+      PQueue.new do |a, b|
+        if @options[:reverse]
+          a.first[@key] <=> b.first[@key]
+        else
+          b.first[@key] <=> a.first[@key]
+        end
+      end
+    end
+
+    # Save a block of records after sorting this.
+    def save_block
+      return if @records.empty?
+
+      @records.sort_by! { |r| r[@options[:key].to_sym] }
+      @records.reverse! if @options[:reverse]
+
+      serialize_records
+
+      @records = []
+      @size    = 0
+    end
+
+    # Save sorted records to file.
+    def serialize_records
+      file = Tempfile.new('sort')
+
+      File.open(file, 'wb') do |ios|
+        BioDSL::Serializer.new(ios) do |serializer|
+          @records.each { |record| serializer << record }
+        end
+      end
+
+      @files << file
+    end
+
+    # Open all sorted files.
+    def open_block_files
+      @fds = @files.inject([]) { |a, e| a << File.open(e, 'rb') }
+      at_exit { @fds.map(&:close) }
+    end
+
+    # Fill the pqueue with the first record from each of the file descriptors.
+    def fill_pqueue
+      @fds.each_with_index do |fd, i|
+        BioDSL::Serializer.new(fd) do |serializer|
+          @pqueue << [serializer.next_entry, i] unless fd.eof?
+        end
+      end
+    end
+
+    # Output all records from the pqueue while filling this with the next record
+    # from the list of file descriptors.
+    #
+    # @param output [Enumerator::Yeilder] Output stream.
+    def output_pqueue(output)
+      until @pqueue.empty?
+        record, i = @pqueue.pop
+
+        output << record
+        @status[:records_out] += 1
+
+        fd = @fds[i]
+
+        BioDSL::Serializer.new(fd) do |serializer|
+          @pqueue << [serializer.next_entry, i] unless fd.eof?
+        end
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/split_pair_seq.rb

@@ -0,0 +1,220 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Splite pair-end sequences in the stream.
+  #
+  # split_pair_seq splits sequences in the stream previously merged with
+  # merge_pair_seq. Sequence names must be in either Illumina1.3/1.5 format
+  # trailing a /1 or /2 or Illumina1.8 containing  1: or  2:. A sequence split
+  # into two will be output as two records where the first will be named with 1
+  # and the second with 2.
+  #
+  # == Usage
+  #
+  #    split_pair_seq
+  #
+  # === Options
+  #
+  # == Examples
+  #
+  # Consider the following records created with merge_pair_seq:
+  #
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14862:1868 1:N:0:14",
+  #     :SEQ=>"TGGGGAATATTGGACAATGGCCTGTTTGCTACCCACGCTT",
+  #     :SEQ_LEN=>40,
+  #     :SCORES=>"<??????BDDDDDDDDGGGG?????BB<-<BDDDDDFEEF",
+  #     :SEQ_LEN_LEFT=>20,
+  #     :SEQ_LEN_RIGHT=>20}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:13906:2139 1:N:0:14",
+  #     :SEQ=>"TAGGGAATCTTGCACAATGGACTCTTCGCTACCCATGCTT",
+  #     :SEQ_LEN=>40,
+  #     :SCORES=>"<???9?BBBDBDDBDDFFFF,5<??BB?DDABDBDDFFFF",
+  #     :SEQ_LEN_LEFT=>20,
+  #     :SEQ_LEN_RIGHT=>20}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14865:2158 1:N:0:14",
+  #     :SEQ=>"TAGGGAATCTTGCACAATGGCCTCTTCGCTACCCATGCTT",
+  #     :SEQ_LEN=>40,
+  #     :SCORES=>"?????BBBBBDDBDDBFFFF??,<??B?BB?BBBBBFF?F",
+  #     :SEQ_LEN_LEFT=>20,
+  #     :SEQ_LEN_RIGHT=>20}
+  #
+  # These can be split using split_pair_seq:
+  #
+  #    BP.new.
+  #    read_fastq(input: "test.fq", encoding: :base_33).
+  #    merge_pair_seq.
+  #    split_pair_seq.
+  #    dump.
+  #    run
+  #
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14862:1868 1:N:0:14",
+  #     :SEQ=>"TGGGGAATATTGGACAATGG",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>"<??????BDDDDDDDDGGGG"}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14862:1868 2:N:0:14",
+  #     :SEQ=>"CCTGTTTGCTACCCACGCTT",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>"?????BB<-<BDDDDDFEEF"}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:13906:2139 1:N:0:14",
+  #     :SEQ=>"TAGGGAATCTTGCACAATGG",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>"<???9?BBBDBDDBDDFFFF"}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:13906:2139 2:N:0:14",
+  #     :SEQ=>"ACTCTTCGCTACCCATGCTT",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>",5<??BB?DDABDBDDFFFF"}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14865:2158 1:N:0:14",
+  #     :SEQ=>"TAGGGAATCTTGCACAATGG",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>"?????BBBBBDDBDDBFFFF"}
+  #    {:SEQ_NAME=>"M01168:16:000000000-A1R9L:1:1101:14865:2158 2:N:0:14",
+  #     :SEQ=>"CCTCTTCGCTACCCATGCTT",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>"??,<??B?BB?BBBBBFF?F"}
+  class SplitPairSeq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for SplitPairSeq.
+    #
+    # @param options [Hash] Options hash.
+    #
+    # @return [SplitPairSeq] Class instance.
+    def initialize(options)
+      @options = options
+
+      check_options
+    end
+
+    # Return command lambda for split_pair_seq.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          if record[:SEQ_NAME] && record[:SEQ] && record[:SEQ_LEN_LEFT] &&
+             record[:SEQ_LEN_RIGHT]
+            split_pair_seq(output, record)
+          else
+            output << record
+
+            @status[:records_out] += 1
+          end
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, nil)
+    end
+
+    # Output two sequence entries from a sequence in the given record that has
+    # been split at a position defined by the SEQ_LEN_LEFT key in the record.
+    #
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param record [Hash]                BioDSL record.
+    #
+    # rubocop: disable Metrics/AbcSize
+    def split_pair_seq(output, record)
+      entry = BioDSL::Seq.new_bp(record)
+
+      @status[:sequences_in] += 1
+      @status[:residues_in]  += entry.length
+
+      pos = get_split_pos(record, entry)
+
+      entry1, entry2 = split_entry(entry, pos)
+
+      output << entry1.to_bp
+      output << entry2.to_bp
+
+      @status[:sequences_out] += 2
+      @status[:residues_out]  += entry1.length + entry2.length
+      @status[:records_out]   += 2
+    end
+
+    # Given a record locate the sequence split position.
+    #
+    # @param record [Hash]           BioDSL record.
+    # @param entry  [BioDSL::Seq] Sequence entry.
+    #
+    # @return [Integer] Sequence split position.
+    #
+    # @raise [BioDSL::SeqError]
+    #   If left and right lengths don't fit entry length.
+    def get_split_pos(record, entry)
+      len_left  = record[:SEQ_LEN_LEFT].to_i
+      len_right = record[:SEQ_LEN_RIGHT].to_i
+
+      unless len_left + len_right == entry.length
+        fail BioDSL::SeqError, 'SEQ_LEN_LEFT + SEQ_LEN_RIGHT != SEQ_LEN ' \
+          "#{len_left} + #{len_right} != #{entry.length}"
+      end
+
+      len_left
+    end
+
+    # Split the given entry at the given position and return two new entries.
+    #
+    # @param entry [BioDSL::Seq] Sequence entry.
+    # @param pos   [Integer]        Split position.
+    #
+    # @return [Array] Tuple with the two new entries.
+    def split_entry(entry, pos)
+      entry1 = entry[0...pos]
+      entry2 = entry[pos..-1]
+
+      fix_seq_names(entry, entry2)
+
+      [entry1, entry2]
+    end
+
+    # Fix sequence names.
+    #
+    # @param entry1 [BioDSL::Seq] Sequence entry1.
+    # @param entry2 [BioDSL::Seq] Sequence entry2.
+    #
+    # @raise [RuntimeError] If names wasn't fixed.
+    def fix_seq_names(entry1, entry2)
+      if entry1.seq_name =~ /^[^ ]+ \d:/
+        entry2.seq_name.sub!(/ \d:/, ' 2:')
+      elsif entry1.seq_name =~ /^.+\/\d$/
+        entry2.seq_name[-1] = '2'
+      else
+        fail "Could not match sequence name: #{entry1.seq_name}"
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/split_values.rb

@@ -0,0 +1,165 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Split the values of a key into new key/value pairs.
+  #
+  # +split_values+ splits the value of a given key into multiple values that are
+  # added to the record. The keys used for the values are per default based on
+  # the given key with an added index, but using the +keys+ option allows
+  # specifying a list of keys to use instead.
+  #
+  # == Usage
+  #
+  #    split_values(<key>: <string>>[, delimiter: <string>[, keys: <list>]])
+  #
+  # === Options
+  #
+  # # key:       <string> - Key who's value to split.
+  # * keys:      <list>   - List of keys to use with split values.
+  # * delimiter: <string> - Delimiter (default='_').
+  #
+  # == Examples
+  #
+  # Consider the following records:
+  #
+  #    {ID: "FOO:count=10", SEQ: "gataag"}
+  #    {ID: "FOO_10_20", SEQ: "gataag"}
+  #
+  # To split the value belinging to ID do:
+  #
+  #    split_values(key: :ID)
+  #
+  #    {:ID=>"FOO:count=10", :SEQ=>"gataag"}
+  #    {:ID=>"FOO_10_20", :SEQ=>"gataag", :ID_0=>"FOO", :ID_1=>10, :ID_2=>20}
+  #
+  # Using a different delimiter:
+  #
+  #    split_values(key: "ID", delimiter: ':count=')
+  #
+  #    {:ID=>"FOO:count=10", :SEQ=>"gataag", :ID_0=>"FOO", :ID_1=>10}
+  #    {:ID=>"FOO_10_20", :SEQ=>"gataag"}
+  #
+  # Using a different delimiter and a list of keys:
+  #
+  #    split_values(key: "ID", keys: ["ID", :COUNT], delimiter: ':count=')
+  #
+  #    {:ID=>"FOO", :SEQ=>"gataag", :COUNT=>10}
+  #    {:ID=>"FOO_10_20", :SEQ=>"gataag"}
+  class SplitValues
+    STATS = %i(records_in records_out)
+
+    # Constructor for SplitValues.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [String,Symbol] :key
+    # @option options [Array]         :keys
+    # @option options [String]        :delimiter
+    #
+    # @return [SplitValues] Class instance.
+    def initialize(options)
+      @options     = options
+
+      check_options
+
+      @first       = true
+      @convert     = []
+      @keys        = @options[:keys]
+      @key         = @options[:key].to_sym
+      @delimiter   = @options[:delimiter] || '_'
+    end
+
+    # Return command lambda for split_values.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          if (value = record[@key])
+            values = value.split(@delimiter)
+
+            if values.size > 1
+              determine_types(values) if @first
+
+              split_values(values, record)
+            end
+          end
+
+          output << record
+
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :key, :keys, :delimiter)
+      options_required(@options, :key)
+    end
+
+    # Given an array of values determine the types that must be converted to
+    # integers or floats and save the value index in a class variable.
+    #
+    # @param values [Array] List of values.
+    def determine_types(values)
+      values.each_with_index do |val, i|
+        val = val.to_num
+
+        if val.is_a? Fixnum
+          @convert[i] = :to_i
+        elsif val.is_a? Float
+          @convert[i] = :to_f
+        end
+      end
+
+      @first = false
+    end
+
+    # Convert values and add to record.
+    #
+    # @param values [Array] List of values.
+    # @param record [Hash]  BioDSL record.
+    def split_values(values, record)
+      values.each_with_index do |val, i|
+        val = val.send(@convert[i]) if @convert[i]
+
+        if @keys
+          record[@keys[i].to_sym] = val
+        else
+          record["#{@key}_#{i}".to_sym] = val
+        end
+      end
+    end
+  end
+end