viral_seq 1.0.6 → 1.0.11

data/lib/viral_seq/seq_hash_pair.rb

@@ -7,7 +7,7 @@ module ViralSeq
   # @example join the paired-end sequences with an overlap of 100 bp
   #   my_seqhashpair.join1(100)
   # @example join the paired-end sequences with unknown overlap, each pair of sequences has its own overlap size
-  #   my_seqhashpair.join1(:indiv)
+  #   my_seqhashpair.join2(model: :indiv)
 
   class SeqHashPair
 
@@ -80,6 +80,12 @@ module ViralSeq
       alias_method :fa, :new_from_fasta
     end
 
+    # the size of nt sequence hash of the SeqHashPair object
+    # @return [Integer] size of nt sequence hash of the SeqHash object
+    def size
+      self.dna_hash.size
+    end
+
     # Pair-end join function for KNOWN overlap size.
     # @param overlap [Integer] how many bases are overlapped. `0` means no overlap, R1 and R2 will be simply put together.
     # @param diff [Integer, Float] the maximum mismatch rate allowed for the overlapping region. default at 0.0, i.e. no mis-match allowed.
@@ -104,17 +110,21 @@ module ViralSeq
       raise ArgumentError.new(":overlap has to be Integer, input #{overlap} invalid.") unless overlap.is_a? Integer
       raise ArgumentError.new(":diff has to be float or integer, input #{diff} invalid.") unless (diff.is_a? Integer or diff.is_a? Float)
       joined_seq = {}
-      seq_pair_hash.each do |seq_name, seq_pair|
+      seq_pair_hash.uniq_hash.each do |seq_pair, seq_names|
         r1_seq = seq_pair[0]
         r2_seq = seq_pair[1]
         if overlap.zero?
-          joined_seq[seq_name] = r1_seq + r2_seq
+          joined_sequence = r1_seq + r2_seq
         elsif r1_seq[-overlap..-1].compare_with(r2_seq[0,overlap]) <= (overlap * diff)
-          joined_seq[seq_name] = r1_seq + r2_seq[overlap..-1]
+          joined_sequence= r1_seq + r2_seq[overlap..-1]
         else
           next
         end
+        seq_names.each do |seq_name|
+          joined_seq[seq_name] = joined_sequence
+        end
       end
+
       joined_seq_hash = ViralSeq::SeqHash.new
       joined_seq_hash.dna_hash = joined_seq
       joined_seq_hash.title = self.title + "_joined"
@@ -139,7 +149,7 @@ module ViralSeq
     #   my_seqhashpair = ViralSeq::SeqHashPair.new(paired_seq2)
     #   my_seqhashpair.join2.dna_hash
     #   => {">pair4"=>"AAAGGGGGGGGGGTT", ">pair5"=>"AAAAAAGGGGTTTTT", ">pair6"=>"AAACAAGGGGTTTTT"}
-    #   my_seqhashpair.join2(model :indiv).dna_hash
+    #   my_seqhashpair.join2(model: :indiv).dna_hash
     #   => {">pair4"=>"AAAGGGGGGGTT", ">pair5"=>"AAAAAAGGGGTTTTT", ">pair6"=>"AAACAAGGGGTTTTT"}
 
     def join2(model: :con, diff: 0.0)
@@ -207,7 +217,7 @@ module ViralSeq
     # {minimal overlap set to 4. }
     def overlap_matrix(sequence1, sequence2)
       min_overlap = 4
-      max_overlap = [sequence1.size, sequence2.size].max
+      max_overlap = [sequence1.size, sequence2.size].min
       matrix_hash = {}
       (min_overlap..max_overlap).each do |overlap|
         matrix_hash[overlap] = sequence1[-overlap..-1].compare_with(sequence2[0, overlap])

data/lib/viral_seq/tcs_core.rb

@@ -0,0 +1,305 @@
+module ViralSeq
+
+  # Core functions for `tcs` pipeline
+
+  class TcsCore
+    class << self
+
+      # methods to calculate TCS consensus cut-off based on the maximum numbers of PIDs and platform error rate.
+
+      def calculate_cut_off(m, error_rate = 0.02)
+        n = 0
+        case error_rate
+        when 0.005...0.015
+          if m <= 10
+            n = 2
+          else
+            n = 1.09*10**-26*m**6 + 7.82*10**-22*m**5 - 1.93*10**-16*m**4 + 1.01*10**-11*m**3 - 2.31*10**-7*m**2 + 0.00645*m + 2.872
+          end
+
+        when 0...0.005
+          if m <= 10
+            n = 2
+          else
+            n = -9.59*10**-27*m**6 + 3.27*10**-21*m**5 - 3.05*10**-16*m**4 + 1.2*10**-11*m**3 - 2.19*10**-7*m**2 + 0.004044*m + 2.273
+          end
+
+        else
+          if m <= 10
+            n = 2
+          elsif m <= 8500
+            n = -1.24*10**-21*m**6 + 3.53*10**-17*m**5 - 3.90*10**-13*m**4 + 2.12*10**-9*m**3 - 6.06*10**-6*m**2 + 1.80*10**-2*m + 3.15
+          else
+            n = 0.0079 * m + 9.4869
+          end
+        end
+
+        n = n.round
+        n = 2 if n < 3
+        return n
+      end
+
+      # identify which file in the directory is R1 file, and which is R2 file based on file names
+      # input as directory (Dir object or a string of path)
+      # by default, .gz files will be unzipped.
+      # return as an hash of {r1_file: file1, r1_file: file2}
+      def r1r2(directory, unzip = true)
+        files = []
+        Dir.chdir(directory) { files = Dir.glob "*" }
+        r1_file = ""
+        r2_file = ""
+        files.each do |f|
+          tag = parser_file_name(f)[:tag]
+
+          if tag.include? "R1"
+            unzip ? r1_file = unzip_r(directory, f) : r1_file = File.join(directory, f)
+          elsif tag.include? "R2"
+            unzip ? r2_file = unzip_r(directory, f) : r2_file = File.join(directory, f)
+          end
+        end
+        return { r1_file: r1_file, r2_file: r2_file }
+      end # end of ViralSeq:TcsCore.r1r2
+
+      # sort directories containing mulitple r1 and r2 files.
+      # use the library name (first string before "_") to seperate libraries
+      # out_dir is the Dir object or string of the output directory, by default named as directory + "_sorted"
+      # return a hash as { with_both_r1_r2: [lib1, lib2, ...], missing_r1: [lib1, lib2, ...], missing_r2: [lib1, lib2, ...], error: [lib1, lib2, ...]}
+
+      def sort_by_lib(directory, out_dir = directory + "_sorted")
+        Dir.mkdir(out_dir) unless File.directory?(out_dir)
+        files = []
+        Dir.chdir(directory) {files = Dir.glob("*")}
+
+        files.each do |file|
+          path = File.join(directory,file)
+          index = file.split("_")[0]
+          index_dir = File.join(out_dir, index)
+          Dir.mkdir(index_dir) unless File.directory?(index_dir)
+          File.rename(path, File.join(index_dir, file))
+        end
+
+        return_obj = { with_both_r1_r2: [],
+                       missing_r1: [],
+                       missing_r2: [],
+                       error: []
+                      }
+
+        libs = []
+        Dir.chdir(out_dir) { libs = Dir.glob('*') }
+        libs.each do |lib|
+          file_check = ViralSeq::TcsCore.r1r2(File.join(out_dir, lib))
+          if !file_check[:r1_file].empty? and !file_check[:r2_file].empty?
+            return_obj[:with_both_r1_r2] << lib
+          elsif file_check[:r1_file].empty? and !file_check[:r2_file].empty?
+            return_obj[:missing_r1] << lib
+          elsif file_check[:r2_file].empty? and !file_check[:r1_file].empty?
+            return_obj[:missing_r2] << lib
+          else
+            return_obj[:error] << lib
+          end
+        end
+        return return_obj
+      end
+
+      # sort array of file names to determine if there is potential errors
+      # input name_array array of file names
+      # output hash { }
+      # need to change for each file name have an error code. and a bool to show if all pass
+      def validate_file_name(name_array)
+        errors = {
+                   file_type_error: [] ,
+                   missing_r1_file: [] ,
+                   missing_r2_file: [] ,
+                   extra_r1_r2_file: [],
+                   no_region_tag: [] ,
+                   multiple_region_tag: []
+                 }
+
+        passed_libs = {}
+
+        name_with_r1_r2 = []
+
+        name_array.each do |name|
+          tag = parser_file_name(name)[:tag]
+          if name !~ /\.fastq\Z|\.fastq\.gz\Z/
+            errors[:file_type_error] << name
+          elsif tag.count("R1") == 0 and tag.count("R2") == 0
+            errors[:no_region_tag] << name
+          elsif tag.count("R1") > 0 and tag.count("R2") > 0
+            errors[:multiple_region_tag] << name
+          elsif tag.count("R1") > 1 or tag.count("R2") > 1
+            errors[:multiple_region_tag] << name
+          else
+            name_with_r1_r2 << name
+          end
+        end
+
+        libs = {}
+
+        name_with_r1_r2.map do |name|
+          libname = parser_file_name(name)[:libname]
+          libs[libname] ||= []
+          libs[libname] << name
+        end
+
+        libs.each do |libname, files|
+          count_r1_file = 0
+          count_r2_file = 0
+          files.each do |name|
+            tag = parser_file_name(name)[:tag]
+            if tag.include? "R1"
+              count_r1_file += 1
+            elsif tag.include? "R2"
+              count_r2_file += 1
+            end
+          end
+
+          if count_r1_file > 1 or count_r2_file > 1
+            errors[:extra_r1_r2_file] += files
+          elsif count_r1_file.zero?
+            errors[:missing_r1_file] += files
+          elsif count_r2_file.zero?
+            errors[:missing_r2_file] += files
+          else
+            passed_libs[libname] = files
+          end
+        end
+
+        passed_names = []
+
+        passed_libs.values.each { |names| passed_names += names}
+
+        if passed_names.size < name_array.size
+          pass = false
+        else
+          pass = true
+        end
+
+        return { errors: errors, all_pass: pass, passed_names: passed_names, passed_libs: passed_libs }
+      end
+
+      # filter r1 raw sequences for non-specific primers.
+      # input r1_sh, SeqHash obj.
+      # return filtered Hash of sequence name and seq pair, in the object { r1_filtered_seq: r1_filtered_seq_pair }
+
+      def filter_r1(r1_sh, forward_primer)
+        if forward_primer.match(/(N+)(\w+)$/)
+          forward_n = $1.size
+          forward_bio_primer = $2
+        else
+          forward_n = 0
+          forward_bio_primer = forward_primer
+        end
+        forward_bio_primer_size = forward_bio_primer.size
+        forward_starting_number = forward_n + forward_bio_primer_size
+        forward_primer_ref = forward_bio_primer.nt_parser
+
+        r1_passed_seq = {}
+        r1_raw = r1_sh.dna_hash
+
+        proc_filter = proc do |name|
+          seq = r1_raw[name]
+          next unless general_filter seq
+          primer_region_seq = seq[forward_n, forward_bio_primer_size]
+          if primer_region_seq =~ forward_primer_ref
+            new_name = remove_tag name
+            r1_passed_seq[new_name] = seq
+          end
+        end
+
+        r1_raw.keys.map do |name|
+          proc_filter.call name
+        end
+
+        return { r1_passed_seq: r1_passed_seq, forward_starting_number: forward_starting_number }
+      end # end of filter_r1
+
+      # filter r2 raw sequences for non-specific primers.
+      # input r2_sh, SeqHash obj.
+      # return filtered Hash of sequence name and seq pair, as well as the length of PID.
+      def filter_r2(r2_sh, cdna_primer)
+        r2_raw = r2_sh.dna_hash
+        cdna_primer.match(/(N+)(\w+)$/)
+        pid_length = $1.size
+        cdna_bio_primer = $2
+        cdna_bio_primer_size = cdna_bio_primer.size
+        reverse_starting_number = pid_length + cdna_bio_primer_size
+        cdna_primer_ref = cdna_bio_primer.nt_parser
+        r2_passed_seq = {}
+        proc_filter = proc do |name|
+          seq = r2_raw[name]
+          next unless general_filter seq
+          primer_region_seq = seq[pid_length, cdna_bio_primer_size]
+          if primer_region_seq =~ cdna_primer_ref
+            new_name = remove_tag name
+            r2_passed_seq[new_name] = seq
+          end
+        end
+
+        r2_raw.keys.map do |name|
+          proc_filter.call name
+        end
+
+        return { r2_passed_seq: r2_passed_seq, pid_length: pid_length, reverse_starting_number: reverse_starting_number }
+      end # end of filter_r2
+
+
+
+      # puts error message in the log file handler, and abort with the same infor
+
+      def log_and_abort(log, infor)
+        log.puts Time.now.to_s + "\t" + infor
+        log.close
+        abort infor.red.bold
+      end
+
+      private
+
+      def unzip_r(indir, f)
+        r_file = File.join(indir, f)
+        if f =~ /.gz/
+          `gzip -d #{r_file}`
+          new_f = f.sub ".gz", ""
+          r_file = File.join(indir, new_f)
+        end
+        return r_file
+      end
+
+      def parser_file_name(file_name)
+        t = file_name.split(".")[0].split("_")
+        if t.size == 1
+          libname = "lib"
+          tag = [ t[0].upcase ]
+        else
+          libname = t[0]
+          tag = t[1..-1].map(&:upcase)
+        end
+        return {libname: libname, tag: tag}
+      end
+
+      def general_filter(seq)
+        if seq[1..-2] =~ /N/ # sequences with ambiguities except the 1st and last position removed
+          return false
+        elsif seq =~ /A{11}/ # a string of poly-A indicates adaptor sequence
+          return false
+        elsif seq =~ /T{11}/ # a string of poly-T indicates adaptor sequence
+          return false
+        else
+          return true
+        end
+      end
+
+      # remove region info tags from the raw MiSeq sequences.
+      def remove_tag(seq_name)
+        if seq_name =~ /\s/
+          new_tag = $`
+        else
+          new_tag = seq_name[0..-3]
+        end
+      end
+
+    end # end of class << self
+
+  end # end of TcsCore module
+
+end # end of main module

data/lib/viral_seq/tcs_json.rb

@@ -0,0 +1,178 @@
+module ViralSeq
+  class TcsJson
+    class << self
+
+      def generate
+        puts '-'*58
+        puts '| JSON Parameter Generator for ' + "TCS #{ViralSeq::TCS_VERSION}".red.bold + " by " + "Shuntai Zhou".blue.bold + ' |'
+        puts '-'*58 + "\n"
+
+        param = {}
+
+        puts 'Enter the path to the directory that contains the MiSeq pair-end R1 and R2 .fastq or .fastq.gz file'
+        print '> '
+        param[:raw_sequence_dir] = gets.chomp.rstrip
+
+        puts 'Enter the estimated platform error rate (for TCS cut-off calculation), default as ' + '0.02'.red.bold
+        print '> '
+        input_error = gets.chomp.rstrip.to_f
+        if input_error == 0.0
+          param[:platform_error_rate] = 0.02
+        else
+          param[:platform_error_rate] = input_error
+        end
+
+        param[:primer_pairs] = []
+
+        loop do
+          data = {}
+          puts "Enter the name for the sequenced region: "
+          print '> '
+          data[:region] = gets.chomp.rstrip
+
+          puts "Enter the #{"cDNA".red.bold} primer sequence: "
+          print '> '
+          data[:cdna] = gets.chomp.rstrip
+
+          puts "Enter the #{"forward".blue.bold} primer sequence: "
+          print '> '
+          data[:forward] = gets.chomp.rstrip
+
+          puts "Enter supermajority cut-off (0.5 - 1.0). Default Simple Majority"
+          print '> '
+          mj = gets.chomp.rstrip.to_f
+          if (0.5..1.0).include?(mj)
+            data[:majority] = mj
+          else
+            data[:majority] = 0
+          end
+
+          print "Need end-join? Y/N \n> "
+          ej = gets.chomp.rstrip
+          if ej =~ /y|yes/i
+            data[:end_join] = true
+
+            print "End-join option? Choose from (1-4):\n
+            1: simple join, no overlap
+            2: known overlap \n
+            3: unknow overlap, use sample consensus to determine overlap, all sequence pairs have same overlap\n
+            4: unknow overlap, determine overlap by individual sequence pairs, sequence pairs can have different overlap\n
+            > "
+            ej_option = gets.chomp.rstrip
+            while ![1,2,3,4].include?(ej_option.to_i)
+              puts "Entered end-join option #{ej_option.red.bold} not valid (choose 1-4), try again"
+              ej_option = gets.chomp.rstrip.to_i
+            end
+            case ej_option.to_i
+            when 1
+              data[:end_join_option] = 1
+              data[:overlap] = 0
+            when 2
+              data[:end_join_option] = 1
+              print "overlap bases: \n> "
+              ol = gets.chomp.rstrip.to_i
+              data[:overlap] = ol
+            when 3
+              data[:end_join_option] = 3
+            when 4
+              data[:end_join_option] = 4
+            end
+
+            print "Need QC for TCS? (support for HIV-1 and SIV)? Y/N \n> "
+            qc = gets.chomp.rstrip
+            if qc =~ /y|yes/i
+              data[:TCS_QC] = true
+
+              data[:ref_genome] = get_ref
+
+              print "reference 5'end ref position or posiiton range, 0 if no need to match this end \n> "
+              data[:ref_start] = gets.chomp.rstrip.to_i
+
+              print "reference 3'end ref position or posiiton range: 0 if no need to match this end \n> "
+              data[:ref_end] = gets.chomp.rstrip.to_i
+
+              print "allow indels? (default as yes) Y/N \n> "
+              indel = gets.chomp.rstrip
+              if indel =~ /n|no/i
+                data[:indel] = false
+              else
+                data[:indel] = true
+              end
+            else
+              data[:TCS_QC] = false
+            end
+
+            print "Need trimming to a reference genome? Y/N \n> "
+            trim_option = gets.chomp.rstrip
+            if trim_option =~ /y|yes/i
+              data[:trim] = true
+              data[:trim_ref] = get_ref
+
+              print "reference 5'end ref position \n> "
+              data[:trim_ref_start] = gets.chomp.rstrip.to_i
+
+              print "reference 3'end ref position \n> "
+              data[:trim_ref_end] = gets.chomp.rstrip.to_i
+
+            else
+              data[:trim] = false
+            end
+
+          else
+            data[:end_join] = false
+          end
+
+          param[:primer_pairs] << data
+          print "Do you wish to conintue? Y/N \n> "
+          continue_sig = gets.chomp.rstrip
+          break unless continue_sig =~ /y|yes/i
+
+        end
+
+        puts "\nYour JSON string is:"
+        puts JSON.pretty_generate(param)
+
+        print "\nDo you wish to save it as a file? Y/N \n> "
+        save_option = gets.chomp.rstrip
+
+        if save_option =~ /y|yes/i
+          print "Path to save JSON file:\n> "
+          path = gets.chomp.rstrip
+          File.open(path, 'w') {|f| f.puts JSON.pretty_generate(param)}
+        end
+
+        print "\nDo you wish to execute tcs pipeline with the input params now? Y/N \n> "
+
+        rsp = gets.chomp.rstrip
+        if rsp =~ /y/i
+          return param
+        else
+          abort "Params json file generated. You can execute tcs pipeline using `tcs -p [params.json]`"
+        end
+
+      end
+
+      private
+      def get_ref
+        puts "Choose reference genome (1-3):"
+        puts "1. HIV-1 HXB2".red.bold
+        puts "2. HIV-1 NL4-3".blue.bold
+        puts "3. SIV MAC239".magenta.bold
+        print "> "
+        ref_option = gets.chomp.rstrip
+        while ![1,2,3].include?(ref_option.to_i)
+          print "Entered end-join option #{ref_option.to_s.red.bold} not valid (choose 1-3), try again\n> "
+          ref_option = gets.chomp.rstrip.to_i
+        end
+        ref = case ref_option.to_i
+              when 1
+                :HXB2
+              when 2
+                :NL43
+              when 3
+                :MAC239
+              end
+      end
+    end
+  end # end TcsJson
+end # end main module