BioDSL 1.0.0

data/lib/BioDSL/commands/collect_otus.rb

@@ -0,0 +1,150 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Collect OTU data from records in the stream.
+  #
+  # +collect_otus+ count the number of times each OTU is found in a set of
+  # samples. OTUs are given by the :S_ID key and samples by the :SAMPLE key.
+  # If a :SEQ_COUNT key is present it will be used to increment the OTU count,
+  # allowing for dereplicated sequences to be used.
+  #
+  # == Usage
+  #
+  #    collect_otus()
+  #
+  # === Options
+  #
+  # == Examples
+  #
+  class CollectOtus
+    require 'set'
+
+    STATS = %i(records_in records_out hits_in hits_out)
+
+    # Constructor for CollectOtus.
+    #
+    # @param options [Hash] Options hash.
+    def initialize(options)
+      @options = options
+
+      check_options
+    end
+
+    # Return lambda for CollectOtus command.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        count_hash = process_input(input, output)
+        samples    = collect_samples(count_hash)
+        process_output(count_hash, samples, output)
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, nil)
+    end
+
+    # Read input stream and for all hit records add these to the count hash.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param output [Enumerator::Yielder] Output stream.
+    #
+    # @return [Hash] Returns the count_hash.
+    def process_input(input, output)
+      count_hash = Hash.new { |h, k| h[k] = Hash.new(0) }
+
+      input.each do |record|
+        @status[:records_in] += 1
+
+        if record[:TYPE] && record[:TYPE] == 'H'
+          add_to_count_hash(count_hash, record)
+        end
+
+        output << record
+        @status[:records_out] += 1
+      end
+
+      count_hash
+    end
+
+    # Add to the count_hash a given record.
+    #
+    # @param count_hash [Hash] Hash with sample counts
+    # @param record [Hash] BioDSL record with sample and count.
+    def add_to_count_hash(count_hash, record)
+      id     = record[:S_ID].to_sym
+      sample = record[:SAMPLE].upcase.to_sym
+      count_hash[id][sample] += (record[:SEQ_COUNT] || 1)
+      @status[:hits_in] += 1
+    end
+
+    # Collect all samples in the count_hash into a sorted set.
+    #
+    # @param count_hash [Hash] Hash with sample counts.
+    #
+    # @return [SortedSet] Sample names.
+    def collect_samples(count_hash)
+      samples = SortedSet.new
+
+      count_hash.values.each do |value|
+        value.keys.map { |key| samples << key }
+      end
+
+      samples
+    end
+
+    # Output all samples and counts from the count_hash and samples to the
+    # output stream.
+    #
+    # @param count_hash [Hash] Hash with sample counts
+    # @param samples [SortedSet] Set with sample names.
+    # @param output [Enumerator::Yielder] Output stream.
+    def process_output(count_hash, samples, output)
+      count_hash.each do |key, value|
+        record = {}
+        record[:RECORD_TYPE] = 'OTU'
+        record[:OTU]         = key.to_s
+
+        samples.each do |sample|
+          record["#{sample}_COUNT".to_sym] = value[sample]
+        end
+
+        output << record
+
+        @status[:hits_out]    += 1
+        @status[:records_out] += 1
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/complement_seq.rb

@@ -0,0 +1,117 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Complment sequences in the stream.
+  #
+  # +complement_seq+ complements sequences in the stream. The sequence type -
+  # DNA or RNA - is guessed by inspected the first sequence in the stream.
+  #
+  # +complement_seq+ can be used together with +reverse_seq+ to reverse-
+  # complement sequences.
+  #
+  # == Usage
+  #
+  #    complement_seq()
+  #
+  # === Options
+  #
+  # == Examples
+  #
+  # Consider the following FASTQ entry in the file test.fq:
+  #
+  #    @M02529:88:000000000-AC0WY:1:1101:12879:1928 2:N:0:185
+  #    TTGTAAAACGACGGCCAGTG
+  #    +
+  #    >>>>>FFFFD@A?A0AE0FG
+  #
+  # To complement the sequence do:
+  #
+  #    BP.new.read_fastq(input:"test.fq").complement_seq.dump.run
+  #
+  #    {:SEQ_NAME=>"M02529:88:000000000-AC0WY:1:1101:12879:1928 2:N:0:185",
+  #     :SEQ=>"AACATTTTGCTGCCGGTCAC",
+  #     :SEQ_LEN=>20,
+  #     :SCORES=>">>>>>FFFFD@A?A0AE0FG"}
+  class ComplementSeq
+    STATS = %i(records_in records_out sequences_in sequences_out residues_in
+               residues_out)
+
+    # Constructor for ComplementSeq.
+    #
+    # @param options [Hash] Options hash.
+    def initialize(options)
+      @options = options
+      @type    = nil
+
+      check_options
+    end
+
+    # Return the command lambda for ComplementSeq.
+    #
+    # @return [Proc] Command lambda
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        input.each do |record|
+          @status[:records_in] += 1
+
+          complement(record) if record.key? :SEQ
+
+          output << record
+
+          @status[:records_out] += 1
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, nil)
+    end
+
+    # Complements sequence in record.
+    #
+    # @param record [Hash] BioDSL record with sequence.
+    def complement(record)
+      entry = BioDSL::Seq.new_bp(record)
+      @type = entry.type_guess unless @type
+      entry.type = @type
+      entry.complement!
+
+      @status[:sequences_in]  += 1
+      @status[:sequences_out] += 1
+      @status[:residues_in]   += entry.length
+      @status[:residues_out]  += entry.length
+
+      record.merge! entry.to_bp
+    end
+  end
+end

data/lib/BioDSL/commands/count.rb

@@ -0,0 +1,135 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+# Namespace for BioDSL.
+module BioDSL
+  # == Count the number of records in the stream.
+  #
+  # +count+ counts the number of records in the stream and outputs the
+  # count as a record who's count is _not_ included. Using the +output+
+  # option will output the count in a file as a table with header.
+  #
+  # == Usage
+  #
+  #    count([output: <file>[, force: <bool]])
+  #
+  # === Options
+  #
+  # * output: <file> - Output file.
+  # * force: <bool>  - Force overwrite existing output file.
+  #
+  # == Examples
+  #
+  # To count the number of records in the file `test.fq`:
+  #
+  #    BP.new.read_fastq(input: "test.fq").count(output: "count.txt").dump.run
+  #
+  #    {:SEQ_NAME=>"ILLUMINA-52179E_0004:2:1:1040:5263#TTAGGC/1",
+  #     :SEQ=>"TTCGGCATCGGCGGCGACGTTGGCGGCGGGGCCGGGCGGGTCGANNNCAT",
+  #     :SEQ_LEN=>50,
+  #     :SCORES=>"GGFBGGEADFAFFDDD,-5AC5?>C:)7?#####################"}
+  #    {:SEQ_NAME=>"ILLUMINA-52179E_0004:2:1:1041:14486#TTAGGC/1",
+  #     :SEQ=>"CATGGCGTATGCCAGACGGCCAGAACGATGGCCGCCGGGCTTCANNNAAG",
+  #     :SEQ_LEN=>50,
+  #     :SCORES=>"FFFFDBD?EEEEEEEFGGFAGAGEFDF=BFGFFGGDDDD=ABAA######"}
+  #    {:SEQ_NAME=>"ILLUMINA-52179E_0004:2:1:1043:19446#TTAGGC/1",
+  #     :SEQ=>"CGGTACTGATCGAGTGTCAGGCTGTTGATCGCCGCGGGCGGGGGTNNGAC",
+  #     :SEQ_LEN=>50,
+  #     :SCORES=>"ECAEBEEEEEFFFFFEFFFFDDEEEGGGGGDEBEECBDAE@#########"}
+  #    {:RECORD_TYPE=>"count", :COUNT=>3}
+  #
+  # And the count is also saved in the file `count.txt`:
+  #    #RECORD_TYPE COUNT
+  #    count  3
+  class Count
+    STATS = %i(records_in records_out)
+
+    # Constructor for the count command.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [String] :output Path to output file.
+    # @option options [Boolean] :force Force overwrite of output file.
+    #
+    # @return [Count] Instance of class Count.
+    def initialize(options)
+      @options = options
+
+      check_options
+    end
+
+    # Return the command lambda for count.
+    #
+    # @return [Proc] Command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        process_input(input, output)
+
+        new_record = {
+          RECORD_TYPE: 'count',
+          COUNT: @status[:records_in]
+        }
+
+        output << new_record
+        @status[:records_out] += 1
+
+        write_output if @options[:output]
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :output, :force)
+      options_allowed_values(@options, force: [true, false, nil])
+      options_files_exist_force(@options, :output)
+    end
+
+    # Process the input stream and emit all recors to the output stream.
+    #
+    # @param input [Enumerator] Input stream
+    # @param output [Enumerator::Yielder] Output stream
+    def process_input(input, output)
+      input.each do |record|
+        @status[:records_in] += 1
+
+        output << record
+        @status[:records_out] += 1
+      end
+    end
+
+    # Write output table to file.
+    def write_output
+      Filesys.open(@options[:output], 'w') do |ios|
+        ios.puts "#RECORD_TYPE\tCOUNT"
+        ios.puts "count\t#{@status[:records_in]}"
+      end
+    end
+  end
+end

data/lib/BioDSL/commands/count_values.rb

@@ -0,0 +1,149 @@
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# Copyright (C) 2007-2015 Martin Asser Hansen (mail@maasha.dk).                #
+#                                                                              #
+# This program is free software; you can redistribute it and/or                #
+# modify it under the terms of the GNU General Public License                  #
+# as published by the Free Software Foundation; either version 2               #
+# of the License, or (at your option) any later version.                       #
+#                                                                              #
+# This program is distributed in the hope that it will be useful,              #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of               #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the                #
+# GNU General Public License for more details.                                 #
+#                                                                              #
+# You should have received a copy of the GNU General Public License            #
+# along with this program; if not, write to the Free Software                  #
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,    #
+# USA.                                                                         #
+#                                                                              #
+# http://www.gnu.org/copyleft/gpl.html                                         #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+#                                                                              #
+# This software is part of the BioDSL framework (www.BioDSL.org).        #
+#                                                                              #
+# >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #
+
+module BioDSL
+  # == Count the number of times values of given keys exists in stream.
+  #
+  # +count_values+ count the values for a given comma seperated list of keys.
+  #
+  # == Usage
+  #
+  #    count_values(<keys: <list>)
+  #
+  # === Options
+  #
+  # * keys: <list>  - Keys whos values to count.
+  #
+  # == Examples
+  #
+  # Consider the following two column table in the file `test.tab`:
+  #
+  #    Human   H1
+  #    Human   H2
+  #    Human   H3
+  #    Dog     D1
+  #    Dog     D2
+  #    Mouse   M1
+  #
+  # To count the values of both columns we first read the table with
+  # +read_table+ and then pass the result to +count_values+:
+  #
+  #    BP.new.
+  #    read_table(input: "test.tab").
+  #    count_values(keys: [:V0, :V1]).
+  #    dump.
+  #    run
+  #
+  #    {:V0=>"Human", :V1=>"H1", :V0_COUNT=>3, :V1_COUNT=>1}
+  #    {:V0=>"Human", :V1=>"H2", :V0_COUNT=>3, :V1_COUNT=>1}
+  #    {:V0=>"Human", :V1=>"H3", :V0_COUNT=>3, :V1_COUNT=>1}
+  #    {:V0=>"Dog", :V1=>"D1", :V0_COUNT=>2, :V1_COUNT=>1}
+  #    {:V0=>"Dog", :V1=>"D2", :V0_COUNT=>2, :V1_COUNT=>1}
+  #    {:V0=>"Mouse", :V1=>"M1", :V0_COUNT=>1, :V1_COUNT=>1}
+  class CountValues
+    STATS = %i(records_in records_out)
+
+    # Constructor for CountValues.
+    #
+    # @param options [Hash] Options hash.
+    # @option options [Array] List of keys whos values to count.
+    #
+    # @return [CountValues] Instance of class.
+    def initialize(options)
+      @options    = options
+
+      check_options
+
+      @keys       = @options[:keys].map(&:to_sym)
+      @count_hash = Hash.new { |h, k| h[k] = Hash.new(0) }
+    end
+
+    # Return the command lambda for the count_values command.
+    #
+    # @return [Proc] Return command lambda.
+    def lmb
+      lambda do |input, output, status|
+        status_init(status, STATS)
+
+        TmpDir.create('count_values') do |tmp_file, _|
+          process_input(input, tmp_file)
+          process_output(output, tmp_file)
+        end
+      end
+    end
+
+    private
+
+    # Check options.
+    def check_options
+      options_allowed(@options, :keys)
+      options_required(@options, :keys)
+    end
+
+    # Save serialized stream to a temporary file and counting the requested
+    # values.
+    #
+    # @param input [Enumerator] Input stream.
+    # @param tmp_file [String] Path to temp file.
+    def process_input(input, tmp_file)
+      File.open(tmp_file, 'wb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          input.each do |record|
+            @keys.map do |key|
+              @count_hash[key][record[key]] += 1 if record.key? key
+            end
+
+            @status[:records_in] += 1
+
+            s << record
+          end
+        end
+      end
+    end
+
+    # Output serialized stream to the output stream including value counts.
+    #
+    # @param output [Enumerator::Yielder] Output stream.
+    # @param tmp_file [String] Path to temp file with serialized input stream.
+    def process_output(output, tmp_file)
+      File.open(tmp_file, 'rb') do |ios|
+        BioDSL::Serializer.new(ios) do |s|
+          s.each do |record|
+            @keys.map do |key|
+              if record.key? key
+                record["#{key}_COUNT".to_sym] = @count_hash[key][record[key]]
+              end
+            end
+
+            output << record
+            @status[:records_out] += 1
+          end
+        end
+      end
+    end
+  end
+end