viral_seq 1.0.4 → 1.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f54bac7de0e2ffe1b4eac99b6f325b039a3be19d832926ebf1b280154cd0ad2
-  data.tar.gz: f5c5fe62299ec5253c4b8b395816a9158e8d60f428b757161daf209da70b0c0e
+  metadata.gz: 4921d3609d6ffc7fd6fbafd7a4a86e5818d47ed855393addd68b20f28b9d214f
+  data.tar.gz: a9e18c01b287885f8f6238343d9633a52d4ae5ea061347e73bd4f3e86788b2a4
 SHA512:
-  metadata.gz: 5c9b7767cbd000c253c7a6f98432e6230adfb6736cf39ced62815443f82f1cee7f446ebabf7791c67675bf37e664f7f4ac9570350894a871ac0310c36844213e
-  data.tar.gz: '049d8b2fc82f09ab618d777a2dc6d20922dd6e4c9af8f6c47ca734b8caa46c4fcee10617247404b6ac85c3e31e3d47353c7162b7c6f1df17ec6ce9efb94829a5'
+  metadata.gz: dd21b57e17751f6c3e475f05b7a565d295ac7592b7c02f8d89ed49192834bee444f08ee9ebf48e41922c8caaf37a03651d5d0c9aa89d97ccc2edb9aad8224d5f
+  data.tar.gz: d1162424ea877d9839c179cacc330c81cd3508fcff07b64a1e753c7c706485d1dcb9a6b60aec9ce02ed33b91bbd4386ed58329c17e247ba086e7d81ed107bfd4

data/Gemfile.lock

@@ -1,15 +1,17 @@
 PATH
   remote: .
   specs:
-    viral_seq (1.0.2)
+    viral_seq (1.0.9)
+      colorize (~> 0.1)
       muscle_bio (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    colorize (0.8.1)
     diff-lcs (1.3)
     muscle_bio (0.4.0)
-    rake (10.5.0)
+    rake (13.0.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -29,9 +31,9 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 2.0)
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   viral_seq!
 
 BUNDLED WITH
-   2.0.2
+   2.1.4

data/README.md

@@ -10,66 +10,135 @@ Specifically for Primer-ID sequencing and HIV drug resistance analysis.
 
 ## Usage
 
-Load all ViralSeq classes by requiring 'viral_seq.rb'
+#### Load all ViralSeq classes by requiring 'viral_seq.rb'
+
+```ruby
+#!/usr/bin/env ruby
+require 'viral_seq'
+```
+
+#### Use executable `locator` to get the coordinates of the sequences on HIV/SIV reference genome from a FASTA file through a terminal
+
+    $ locator -i sequence.fasta -o sequence.fasta.csv
+
+
+#### Use executable `tcs` pipeline to process Primer ID MiSeq sequencing data. Parameter json file can be generated using `tcs_json_generator` or at https://tcs-dr-dept-tcs.cloudapps.unc.edu/generator.php
+
+    $ tcs params.json
+
+#### Use executable `tcs_json_generator` to generate params .json file for the `tcs` pipeline.
+
+    $ tcs_json_generator
 
-    #!/usr/bin/env ruby
-    require 'viral_seq'
 
 ## Some Examples
 
-Load nucleotide sequences from a FASTA format sequence file
+#### Load nucleotide sequences from a FASTA format sequence file
 
-    my_seqhash = ViralSeq::SeqHash.fa('my_seq_file.fasta')
+```ruby
+my_seqhash = ViralSeq::SeqHash.fa('my_seq_file.fasta')
+```
 
-Make an alignment (using MUSCLE)
+#### Make an alignment (using MUSCLE)
 
-    aligned_seqhash = my_seqhash.align
+```ruby
+aligned_seqhash = my_seqhash.align
+```
 
-Filter nucleotide sequences with the reference coordinates (HIV Protease)
+#### Filter nucleotide sequences with the reference coordinates (HIV Protease)
 
-    qc_seqhash = aligned_seqhash.hiv_seq_qc(2253, 2549, false, :HXB2)
+```ruby
+qc_seqhash = aligned_seqhash.hiv_seq_qc(2253, 2549, false, :HXB2)
+```
 
-Further filter out sequences with Apobec3g/f hypermutations
+#### Further filter out sequences with Apobec3g/f hypermutations
 
-    qc_seqhash = qc_seqhash.a3g
+```ruby
+qc_seqhash = qc_seqhash.a3g
+```
 
-Calculate nucleotide diveristy π
+#### Calculate nucleotide diveristy π
 
-    qc_seqhash.pi
+```ruby
+qc_seqhash.pi
+```
 
-Calculate cut-off for minority variants based on Poisson model
+#### Calculate cut-off for minority variants based on Poisson model
 
-    cut_off = qc_seqhash.pm
+```ruby
+cut_off = qc_seqhash.pm
+```
 
-Examine for drug resistance mutations for HIV PR region
+#### Examine for drug resistance mutations for HIV PR region
 
-    qc_seqhash.sdrm_hiv_pr(cut_off)
+```ruby
+qc_seqhash.sdrm_hiv_pr(cut_off)
+```
 
 ## Updates
 
+Version 1.0.9-07182020:
+
+  1. Change ViralSeq::SeqHash#stop_codon and ViralSeq::SeqHash#a3g_hypermut return value to hash object.
+
+  2. TCS pipeline updated to version 2.0.1. Add optional `export_raw: TRUE/FALSE` in json params. If `export_raw` is `TRUE`, raw sequence reads (have to pass quality filters) will be exported, along with TCS reads.
+
+Version 1.0.8-02282020:
+
+  1. TCS pipeline (version 2.0.0) added as executable.
+      tcs  -  main TCS pipeline script.
+      tcs_json_generator  -  step-by-step script to generate json file for tcs pipeline.
+
+  2. Methods added:
+      ViralSeq::SeqHash#trim
+
+  3. Bug fix for several methods.
+
+Version 1.0.7-01282020:
+
+  1. Several methods added, including
+      ViralSeq::SeqHash#error_table
+      ViralSeq::SeqHash#random_select
+  2. Improved performance for several functions.
+
+Version 1.0.6-07232019:
+
+  1. Several methods added to ViralSeq::SeqHash, including
+      ViralSeq::SeqHash#size
+      ViralSeq::SeqHash#+
+      ViralSeq::SeqHash#write_nt_fa
+      ViralSeq::SeqHash#mutation
+  2. Update documentations and rspec samples.
+
+Version 1.0.5-07112019:
+
+  1. Update ViralSeq::SeqHash#sequence_locator.
+     Program will try to determine the direction (`+` or `-` of the query sequence)
+  2. update executable `locator` to have a column of `direction` in output .csv file
+
 Version 1.0.4-07102019:
 
-    1. Use home directory (Dir.home) instead of the directory of the script file for temp MUSCLE file.
-    2. Fix bugs in bin `locator`
+  1. Use home directory (Dir.home) instead of the directory of the script file for temp MUSCLE file.
+  2. Fix bugs in bin `locator`
 
 Version 1.0.3-07102019:
 
-    1. Bug fix.
+  1. Bug fix.
 
 Version 1.0.2-07102019:
 
-    1. Fixed a gem loading issue.
+  1. Fixed a gem loading issue.
 
 Version 1.0.1-07102019:
 
-    1. Add keyword argument :model to ViralSeq::SeqHashPair#join2.
-    2. Add method ViralSeq::SeqHash#sequence_locator (also: #loc), a function to locate sequences on HIV/SIV reference genomes, as HIV Sequence Locator from LANL.
-    3. Add executable 'locator'. An HIV/SIV sequence locator tool similar to LANL Sequence Locator.
-    4. update documentations
+  1. Add keyword argument :model to ViralSeq::SeqHashPair#join2.
+  2. Add method ViralSeq::SeqHash#sequence_locator (also: #loc), a function to locate sequences on HIV/SIV reference genomes, as HIV Sequence Locator from LANL.
+  3. Add executable 'locator'. An HIV/SIV sequence locator tool similar to LANL Sequence Locator.
+  4. update documentations
 
 Version 1.0.0-07092019:
 
-    1. Rewrote the whole ViralSeq gem, grouping methods into modules and classes under main Module::ViralSeq
+  1. Rewrote the whole ViralSeq gem, grouping methods into modules and classes under main Module::ViralSeq
 
 ## Development
 

data/bin/locator

@@ -1,46 +1,44 @@
 #!/usr/bin/env ruby
 
+# Copyright (c) 2020 Shuntai Zhou (shuntai.zhou@gmail.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'viral_seq'
 require 'csv'
 require 'optparse'
-
-module ViralSeq
-  class SeqHash
-
-    def sequence_locator(ref_option = :HXB2)
-      out_array = []
-      dna_seq = self.dna_hash
-      title = self.title
-
-      uniq_dna = dna_seq.uniq_hash
-
-      uniq_dna.each do |seq,names|
-        s = ViralSeq::Sequence.new('',seq)
-        loc = s.locator(ref_option)
-        names.each do |name|
-          out_array << ([title, name, ref_option.to_s] + loc)
-        end
-      end
-      return out_array
-    end # end of locator
-    alias_method :loc, :sequence_locator
-  end
-end
+require 'colorize'
 
 def myparser
   options = {}
   OptionParser.new do |opts|
-    opts.banner = "Usage: locator -i [nt_sequence_fasta_file] -o [locator_info_csv_file] -r [reference_genome_option]"
+    opts.banner = "#{"Usage:".red.bold} locator #{"-i".blue.bold} [nt_sequence_fasta_file] #{"-o".blue.bold} [locator_info_csv_file] #{"-r".blue.bold} [reference_genome_option]"
 
-    opts.on('-i', '--infile FASTA_FILE', 'nt sequence file in FASTA format') do |i|
+    opts.on('-i', '--infile FASTA_FILE', "#{"nt sequence".blue.bold} file in FASTA format") do |i|
       options[:infile] = i
     end
 
-    opts.on('-o', '--outfile CSV_FILE', 'output .csv file for locator info') do |o|
+    opts.on('-o', '--outfile CSV_FILE', "output .csv file for locator info, default as \#\{infile\}.csv") do |o|
       options[:outfile] = o
     end
 
-    opts.on('-r', '--ref_option OPTION', 'reference genome option, choose from `HXB2` (default), `NL43`, `MAC239`') do |o|
+    opts.on('-r', '--ref_option OPTION', "reference genome option, choose from #{"`HXB2` (default), `NL43`, `MAC239`".blue.bold}") do |o|
       options[:ref_option] = o.to_sym
     end
 
@@ -48,13 +46,19 @@ def myparser
       puts opts
       exit
     end
+
+    opts.on("-v", "--version", "Version number of RubyGem::ViralSeq") do
+      puts opts
+      exit
+    end
+
   end.parse!
   return options
 end
 
-puts "\nSequence Locator (RubyGem::ViralSeq) #{ViralSeq::VERSION} by Shuntai Zhou"
-puts "See details at https://github.com/ViralSeq/viral_seq\n"
-puts "Resembling Sequence Locator from LANL (https://www.hiv.lanl.gov/content/sequence/LOCATE/locate.html)\n\n"
+puts "\n" + "Sequence Locator (RubyGem::ViralSeq Version #{ViralSeq::VERSION})".red.bold + " by " + "Shuntai Zhou".blue.bold
+puts "See details at " +  "https://github.com/ViralSeq/viral_seq\n".blue
+puts "Resembling" + " Sequence Locator ".magenta.bold + "from LANL" + " (https://www.hiv.lanl.gov/content/sequence/LOCATE/locate.html)\n".blue
 
 ARGV << '-h' if ARGV.size == 0
 
@@ -64,35 +68,36 @@ begin
   if options[:infile]
     seq_file = options[:infile]
   else
-    raise StandardError.new("Input file sequence file not found")
+    raise StandardError.new("Input file sequence file not found".red.bold)
   end
 
   if options[:outfile]
     csv_file = options[:outfile]
   else
-    raise StandardError.new("Please provide path to output csv file")
+    csv_file = seq_file + ".csv"
   end
 
   unless File.exist?(seq_file)
-    raise StandardError.new("Input file sequence file not found")
+    raise StandardError.new("Input file sequence file not found".red.bold)
   end
 
   seqs = ViralSeq::SeqHash.fa(seq_file)
   opt =  options[:ref_option] ? options[:ref_option] : :HXB2
 
   unless [:HXB2, :NL43, :MAC239].include? opt
-    puts "Reference option #{opt} not recognized, using `:HXB2` as the reference genome."
+    puts "Reference option `#{opt}` not recognized, using `HXB2` as the reference genome.".red.bold
     opt = :HXB2
   end
 
   locs = seqs.loc(opt)
-  head = ["title", "sequence", "ref", "start", "end", "similarity", "indel", "aligned_input", "aligned_ref"]
+  head = ["title", "sequence", "ref", "direction", "start", "end", "similarity", "indel", "aligned_input", "aligned_ref"]
   locs.unshift(head)
   data = CSV.generate do |csv|
     locs.each {|loc| csv << loc}
   end
 
   File.write(csv_file, data)
+  puts "Output file found at #{csv_file.green.bold}"
 rescue StandardError => e
   puts e.message
   puts "\n"

data/bin/tcs

@@ -0,0 +1,519 @@
+#!/usr/bin/env ruby
+
+# TCS pipeline for Primer ID sequencing data analysis.
+
+# Copyright (c) 2020 Shuntai Zhou (shuntai.zhou@gmail.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# Use JSON file as the run param
+# run tcs_json_generator.rb to generate param json file.
+
+require 'viral_seq'
+require 'json'
+require 'colorize'
+
+
+# calculate consensus cutoff
+
+def calculate_cut_off(m, error_rate = 0.02)
+  n = 0
+  case error_rate
+  when 0.005...0.015
+    if m <= 10
+      n = 2
+    else
+      n = 1.09*10**-26*m**6 + 7.82*10**-22*m**5 - 1.93*10**-16*m**4 + 1.01*10**-11*m**3 - 2.31*10**-7*m**2 + 0.00645*m + 2.872
+    end
+
+  when 0...0.005
+    if m <= 10
+      n = 2
+    else
+      n = -9.59*10**-27*m**6 + 3.27*10**-21*m**5 - 3.05*10**-16*m**4 + 1.2*10**-11*m**3 - 2.19*10**-7*m**2 + 0.004044*m + 2.273
+    end
+
+  else
+    if m <= 10
+      n = 2
+    elsif m <= 8500
+      n = -1.24*10**-21*m**6 + 3.53*10**-17*m**5 - 3.90*10**-13*m**4 + 2.12*10**-9*m**3 - 6.06*10**-6*m**2 + 1.80*10**-2*m + 3.15
+    else
+      n = 0.0079 * m + 9.4869
+    end
+  end
+
+  n = n.round
+  n = 2 if n < 3
+  return n
+end
+
+puts "\n" + '-'*50
+puts '| The TCS Pipeline ' + "Version #{ViralSeq::TCS_VERSION}".red.bold + " by " + "Shuntai Zhou".blue.bold + ' |'
+puts '-'*50 + "\n"
+
+unless ARGV[0]
+  raise "No JSON param file found. Script terminated."
+end
+
+params = JSON.parse(File.read(ARGV[0]), symbolize_names: true)
+
+indir = params[:raw_sequence_dir]
+
+unless File.exist?(indir)
+  raise "No input sequence directory found. Script terminated."
+end
+
+libname = File.basename(indir)
+
+# obtain R1 and R2 file path
+files = []
+Dir.chdir(indir) do
+  files = Dir.glob("*")
+end
+
+if files.empty?
+  raise "Input dir does not contain files. Script terminated."
+end
+
+r1_f = ""
+r2_f = ""
+
+# unzip .fasta.gz
+def unzip_r(indir, f)
+  r_file = indir + "/" + f
+  if f =~ /.gz/
+    `gzip -d #{r_file}`
+    new_f = f.sub ".gz", ""
+    r_file = File.join(indir, new_f)
+  end
+  return r_file
+end
+runtime_log_file = File.join(indir,"runtime.log")
+log = File.open(runtime_log_file, "w")
+log.puts "TSC pipeline Version " + ViralSeq::TCS_VERSION.to_s
+log.puts "viral_seq Version " + ViralSeq::VERSION.to_s
+log.puts Time.now.to_s + "\t" + "Start TCS pipeline..."
+
+
+files.each do |f|
+  t = f.split("_")
+  if t.size == 1
+    tag = f
+  else
+    tag = f.split("_")[1..-1].join("_")
+  end
+
+  if tag =~ /r1/i
+    r1_f = unzip_r(indir, f)
+  elsif tag =~ /r2/i
+    r2_f = unzip_r(indir, f)
+  end
+end
+
+
+unless File.exist?(r1_f)
+  log.puts "R1 file not found. Script terminated."
+  raise "R1 file not found. Script terminated."
+end
+
+unless File.exist?(r2_f)
+  log.puts "R2 file not found. Script terminated."
+  raise "R2 file not found. Script terminated."
+end
+
+r1_fastq_sh = ViralSeq::SeqHash.fq(r1_f)
+r2_fastq_sh = ViralSeq::SeqHash.fq(r2_f)
+
+raw_sequence_number = r1_fastq_sh.size
+log.puts Time.now.to_s + "\tRaw sequence number: #{raw_sequence_number.to_s}"
+
+if params[:platform_error_rate]
+  error_rate = params[:platform_error_rate]
+else
+  error_rate = 0.02
+end
+
+primers = params[:primer_pairs]
+if primers.empty?
+  log.puts "No primer information. Script terminated."
+  raise "No primer information. Script terminated."
+end
+
+primers.each do |primer|
+  summary_json = {}
+  summary_json[:tcs_version] = ViralSeq::TCS_VERSION
+  summary_json[:viralseq_version] = ViralSeq::VERSION
+  summary_json[:runtime] = Time.now.to_s
+
+  primer[:region] ? region = primer[:region] : region = "region"
+  summary_json[:primer_set_name] = region
+
+  cdna_primer = primer[:cdna]
+  forward_primer = primer[:forward]
+
+  export_raw = primer[:export_raw]
+
+  unless cdna_primer
+    log.puts Time.now.to_s + "\t" + region + " does not have cDNA primer sequence. #{region} skipped."
+  end
+  unless forward_primer
+    log.puts Time.now.to_s + "\t" +  region + " does not have forward primer sequence. #{region} skipped."
+  end
+  summary_json[:cdan_primer] = cdna_primer
+  summary_json[:forward_primer] = forward_primer
+
+  primer[:majority] ? majority_cut_off = primer[:majority] : majority_cut_off = 0.5
+  summary_json[:majority_cut_off] = majority_cut_off
+
+  summary_json[:total_raw_sequence] = raw_sequence_number
+
+  log.puts Time.now.to_s + "\t" +  "Porcessing #{region}..."
+
+  r1_raw = r1_fastq_sh.dna_hash
+  r2_raw = r2_fastq_sh.dna_hash
+
+  log.puts Time.now.to_s + "\t" +  "filtering R1..."
+  # obtain biological forward primer sequence
+  if forward_primer.match(/(N+)(\w+)$/)
+    forward_n = $1.size
+    forward_bio_primer = $2
+  else
+    forward_n = 0
+    forward_bio_primer = forward_primer
+  end
+  forward_bio_primer_size = forward_bio_primer.size
+  forward_starting_number = forward_n + forward_bio_primer_size
+
+  # filter R1 sequences with forward primers.
+  forward_primer_ref = forward_bio_primer.nt_parser
+  r1_passed_seq = {}
+  r1_raw.each do |name,seq|
+    next if seq[1..-2] =~ /N/ # sequences with ambiguities except the 1st and last position removed
+    next if seq =~ /A{11}/ # a string of poly-A indicates adaptor sequence
+    next if seq =~ /T{11}/ # a string of poly-T indicates adaptor sequence
+
+    primer_region_seq = seq[forward_n, forward_bio_primer_size]
+    if primer_region_seq =~ forward_primer_ref
+      r1_passed_seq[name.split("\s")[0]] = seq
+    end
+  end
+  log.puts Time.now.to_s + "\t" +  "R1 filtered: #{r1_passed_seq.size.to_s}"
+
+  summary_json[:r1_filtered_raw] = r1_passed_seq.size
+
+  log.puts Time.now.to_s + "\t" +  "filtering R2..."
+  # obtain biological reverse primer sequence
+  cdna_primer.match(/(N+)(\w+)$/)
+  pid_length = $1.size
+  cdna_bio_primer = $2
+  cdna_bio_primer_size = cdna_bio_primer.size
+  reverse_starting_number = pid_length + cdna_bio_primer_size
+
+  # filter R2 sequences with cDNA primers.
+  cdna_primer_ref = cdna_bio_primer.nt_parser
+  r2_passed_seq = {}
+  r2_raw.each do |name, seq|
+    next if seq[1..-2] =~ /N/ # sequences with ambiguities except the 1st and last position removed
+    next if seq =~ /A{11}/ # a string of poly-A indicates adaptor sequence
+    next if seq =~ /T{11}/ # a string of poly-T indicates adaptor sequence
+
+    primer_region_seq = seq[pid_length, cdna_bio_primer_size]
+    if primer_region_seq =~ cdna_primer_ref
+      r2_passed_seq[name.split("\s")[0]] = seq
+    end
+  end
+  log.puts Time.now.to_s + "\t" +  "R2 filtered: #{r2_passed_seq.size.to_s}"
+  summary_json[:r2_filtered_raw] = r2_passed_seq.size
+
+  # pair-end
+  log.puts Time.now.to_s + "\t" +  "Pairing R1 and R2 seqs..."
+  id = {} # hash for :sequence_tag => primer_id
+  bio_r2 = {} # hash for :sequence_tag => primer_trimmed_r2_sequence
+  bio_r1 = {} # hash for :sequence_tag => primer_trimmed_r1_sequence
+  common_keys = r1_passed_seq.keys & r2_passed_seq.keys
+  paired_seq_number = common_keys.size
+  log.puts Time.now.to_s + "\t" +  "Paired raw sequences are : #{paired_seq_number.to_s}"
+  summary_json[:paired_raw_sequence] = paired_seq_number
+
+  common_keys.each do |seqtag|
+    r1_seq = r1_passed_seq[seqtag]
+    r2_seq = r2_passed_seq[seqtag]
+    pid = r2_seq[0, pid_length]
+    id[seqtag] = pid
+    bio_r2[seqtag] = r2_seq[reverse_starting_number..-2]
+    bio_r1[seqtag] = r1_seq[forward_starting_number..-2]
+  end
+
+  # TCS cut-off
+  log.puts Time.now.to_s + "\t" +  "Calculate consensus cutoff...."
+
+  primer_id_list = id.values
+  primer_id_count = primer_id_list.count_freq
+  primer_id_dis = primer_id_count.values.count_freq
+
+  # calculate distinct_to_raw
+  distinct_to_raw = (primer_id_count.size/primer_id_list.size.to_f).round(3)
+  summary_json[:distinct_to_raw] = distinct_to_raw
+
+  if primer_id_dis.keys.size < 5
+    log.puts Time.now.to_s + "\t" +  "Less than 5 Primer IDs detected. Region #{region} aborted."
+    next
+  end
+
+  max_id = primer_id_dis.keys.sort[-5..-1].mean
+  consensus_cutoff = calculate_cut_off(max_id,error_rate)
+  log.puts Time.now.to_s + "\t" +  "Consensus cut-off is #{consensus_cutoff.to_s}"
+  summary_json[:consensus_cutoff] = consensus_cutoff
+  summary_json[:length_of_pid] = pid_length
+
+  log.puts Time.now.to_s + "\t" +  "Creating consensus..."
+
+  # Primer ID over the cut-off
+  primer_id_count_over_n = []
+  primer_id_count.each do |primer_id,count|
+    primer_id_count_over_n << primer_id if count > consensus_cutoff
+  end
+  pid_to_process = primer_id_count_over_n.size
+  log.puts Time.now.to_s + "\t" +  "Number of consensus to process: #{pid_to_process.to_s}"
+  summary_json[:total_tcs_with_ambiguities] = pid_to_process
+
+  # setup output path
+  out_dir_set = File.join(indir, region)
+  Dir.mkdir(out_dir_set) unless File.directory?(out_dir_set)
+  out_dir_consensus = File.join(out_dir_set, "consensus")
+  Dir.mkdir(out_dir_consensus) unless File.directory?(out_dir_consensus)
+
+  outfile_r1 = File.join(out_dir_consensus, 'r1.fasta')
+  outfile_r2 = File.join(out_dir_consensus, 'r2.fasta')
+  outfile_log = File.join(out_dir_set, 'log.json')
+
+  # if export_raw is true, create dir for raw sequence
+  if export_raw
+    out_dir_raw = File.join(out_dir_set, "raw")
+    Dir.mkdir(out_dir_raw) unless File.directory?(out_dir_raw)
+    outfile_raw_r1 = File.join(out_dir_raw, 'r1.raw.fasta')
+    outfile_raw_r2 = File.join(out_dir_raw, 'r2.raw.fasta')
+    raw_r1_f = File.open(outfile_raw_r1, 'w')
+    raw_r2_f = File.open(outfile_raw_r2, 'w')
+
+    bio_r1.keys.each do |k|
+      raw_r1_f.puts k + "_r1"
+      raw_r2_f.puts k + "_r2"
+      raw_r1_f.puts bio_r1[k]
+      raw_r2_f.puts bio_r2[k].rc
+    end
+
+    raw_r1_f.close
+    raw_r2_f.close
+  end
+
+  # create TCS
+
+  pid_seqtag_hash = {}
+  id.each do |name, pid|
+    if pid_seqtag_hash[pid]
+      pid_seqtag_hash[pid] << name
+    else
+      pid_seqtag_hash[pid] = []
+      pid_seqtag_hash[pid] << name
+    end
+  end
+
+  consensus = {}
+  r1_temp = {}
+  r2_temp = {}
+  m = 0
+  primer_id_count_over_n.each do |primer_id|
+    m += 1
+    log.puts Time.now.to_s + "\t" +  "Now processing number #{m}" if m%100 == 0
+    seq_with_same_primer_id = pid_seqtag_hash[primer_id]
+    r1_sub_seq = []
+    r2_sub_seq = []
+    seq_with_same_primer_id.each do |seq_name|
+      r1_sub_seq << bio_r1[seq_name]
+      r2_sub_seq << bio_r2[seq_name]
+    end
+
+    #consensus name including the Primer ID and number of raw sequences of that Primer ID, library name and setname.
+    consensus_name = ">" + primer_id + "_" + seq_with_same_primer_id.size.to_s + "_" + libname + "_" + region
+    r1_consensus = ViralSeq::SeqHash.array(r1_sub_seq).consensus(majority_cut_off)
+    r2_consensus = ViralSeq::SeqHash.array(r2_sub_seq).consensus(majority_cut_off)
+    next if r1_consensus =~ /[^ATCG]/
+    next if r2_consensus =~ /[^ATCG]/
+
+    # reverse complement sequence of the R2 region
+    r2_consensus = r2_consensus.rc
+    consensus[consensus_name] = [r1_consensus, r2_consensus]
+    r1_temp[consensus_name] = r1_consensus
+    r2_temp[consensus_name] = r2_consensus
+  end
+  r1_temp_sh = ViralSeq::SeqHash.new(r1_temp)
+  r2_temp_sh = ViralSeq::SeqHash.new(r2_temp)
+
+  # filter consensus sequences for residual offspring PIDs
+  consensus_filtered = {}
+  consensus_number_temp = consensus.size
+  max_pid_comb = 4**pid_length
+  if consensus_number_temp < 0.003*max_pid_comb
+    log.puts Time.now.to_s + "\t" +  "Applying PID post TCS filter..."
+    r1_consensus_filtered = r1_temp_sh.filter_similar_pid.dna_hash
+    r2_consensus_filtered = r2_temp_sh.filter_similar_pid.dna_hash
+    common_pid = r1_consensus_filtered.keys & r2_consensus_filtered.keys
+    common_pid.each do |pid|
+      consensus_filtered[pid] = [r1_consensus_filtered[pid], r2_consensus_filtered[pid]]
+    end
+  else
+    consensus_filtered = consensus
+  end
+  n_con = consensus_filtered.size
+  log.puts Time.now.to_s + "\t" +  "Number of consensus sequences: " + n_con.to_s
+  summary_json[:total_tcs] = n_con
+  summary_json[:resampling_param] = (n_con/pid_to_process.to_f).round(3)
+
+  log.puts Time.now.to_s + "\t" +  "Writing R1 and R2 files..."
+  # r1_file output
+  f1 = File.open(outfile_r1, 'w')
+  f2 = File.open(outfile_r2, 'w')
+  primer_id_in_use = {}
+  r1_seq_length = consensus_filtered.values[0][0].size
+  r2_seq_length = consensus_filtered.values[0][1].size
+  log.puts Time.now.to_s + "\t" + "R1 sequence #{r1_seq_length} bp"
+  log.puts Time.now.to_s + "\t" + "R1 sequence #{r2_seq_length} bp"
+  consensus_filtered.each do |seq_name,seq|
+    f1.print seq_name + "_r1\n" + seq[0] + "\n"
+    f2.print seq_name + "_r2\n" + seq[1] + "\n"
+    primer_id_in_use[seq_name.split("_")[0][1..-1]] = seq_name.split("_")[1].to_i
+  end
+  f1.close
+  f2.close
+
+  out_pid_json = File.join(out_dir_set, 'primer_id.json')
+  pid_json = {}
+  pid_json[:primer_id_in_use] = Hash[*(primer_id_in_use.sort_by {|k, v| [-v,k]}.flatten)]
+  pid_json[:primer_id_distribution] = Hash[*(primer_id_dis.sort_by{|k,v| k}.flatten)]
+  pid_json[:primer_id_frequency] = Hash[*(primer_id_count.sort_by {|k, v| [-v,k]}.flatten)]
+  File.open(out_pid_json, 'w') do |f|
+    f.puts JSON.pretty_generate(pid_json)
+  end
+
+  def end_join(dir, option, overlap)
+    shp = ViralSeq::SeqHashPair.fa(dir)
+    case option
+    when 1
+      joined_sh = shp.join1()
+    when 3
+      joined_sh = shp.join2
+    when 4
+      joined_sh = shp.join2(model: :indiv)
+    end
+    return joined_sh
+  end
+
+  if primer[:end_join]
+    log.puts Time.now.to_s + "\t" +  "Start end-pairing for TCS..."
+    shp = ViralSeq::SeqHashPair.fa(out_dir_consensus)
+    joined_sh = end_join(out_dir_consensus, primer[:end_join_option], primer[:overlap])
+    log.puts Time.now.to_s + "\t" + "Paired TCS number: " + joined_sh.size.to_s
+    summary_json[:combined_tcs] = joined_sh.size
+
+    if export_raw
+      joined_sh_raw = end_join(out_dir_raw, primer[:end_join_option], primer[:overlap])
+    end
+
+  else
+    File.open(outfile_log, "w") do |f|
+      f.puts JSON.pretty_generate(summary_json)
+    end
+    next
+  end
+
+  if primer[:TCS_QC]
+    ref_start = primer[:ref_start]
+    ref_end = primer[:ref_end]
+    ref_genome = primer[:ref_genome].to_sym
+    indel = primer[:indel]
+    if ref_start == 0
+      ref_start = 0..(ViralSeq::RefSeq.get(ref_genome).size - 1)
+    end
+    if ref_end == 0
+      ref_end = 0..(ViralSeq::RefSeq.get(ref_genome).size - 1)
+    end
+    if primer[:end_join_option] == 1 and primer[:overlap] == 0
+      r1_sh = ViralSeq::SeqHash.fa(outfile_r1)
+      r2_sh = ViralSeq::SeqHash.fa(outfile_r2)
+      r1_sh = r1_sh.hiv_seq_qc(ref_start, (0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), indel, ref_genome)
+      r2_sh = r2_sh.hiv_seq_qc((0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), ref_end, indel, ref_genome)
+      new_r1_seq = r1_sh.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+      new_r2_seq = r2_sh.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+      joined_seq = {}
+      new_r1_seq.each do |seq_name, seq|
+        next unless seq
+        next unless new_r2_seq[seq_name]
+        joined_seq[seq_name] = seq + new_r2_seq[seq_name]
+      end
+      joined_sh = ViralSeq::SeqHash.new(joined_seq)
+
+      if export_raw
+        r1_sh_raw = ViralSeq::SeqHash.fa(outfile_raw_r1)
+        r2_sh_raw = ViralSeq::SeqHash.fa(outfile_raw_r2)
+        r1_sh_raw = r1_sh_raw.hiv_seq_qc(ref_start, (0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), indel, ref_genome)
+        r2_sh_raw = r2_sh_raw.hiv_seq_qc((0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), ref_end, indel, ref_genome)
+        new_r1_seq_raw = r1_sh_raw.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+        new_r2_seq_raw = r2_sh_raw.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+        joined_seq_raw = {}
+        new_r1_seq_raw.each do |seq_name, seq|
+          next unless seq
+          next unless new_r2_seq_raw[seq_name]
+          joined_seq_raw[seq_name] = seq + new_r2_seq_raw[seq_name]
+        end
+        joined_sh_raw = ViralSeq::SeqHash.new(joined_seq_raw)
+      end
+    else
+      joined_sh = joined_sh.hiv_seq_qc(ref_start, ref_end, indel, ref_genome)
+
+      if export_raw
+        joined_sh_raw = joined_sh.hiv_seq_qc(ref_start, ref_end, indel, ref_genome)
+      end
+    end
+    log.puts Time.now.to_s + "\t" + "Paired TCS number after QC based on reference genome: " + joined_sh.size.to_s
+    summary_json[:combined_tcs_after_qc] = joined_sh.size
+    if primer[:trim]
+      trim_start = primer[:trim_ref_start]
+      trim_end = primer[:trim_ref_end]
+      trim_ref = primer[:trim_ref].to_sym
+      joined_sh = joined_sh.trim(trim_start, trim_end, trim_ref)
+    end
+    joined_sh.write_nt_fa(File.join(out_dir_consensus, "combined.fasta"))
+    if export_raw
+      joined_sh_raw.write_nt_fa(File.join(out_dir_raw, "combined.fasta"))
+    end
+  end
+
+  File.open(outfile_log, "w") do |f|
+    f.puts JSON.pretty_generate(summary_json)
+  end
+end
+
+log.puts Time.now.to_s + "\t" + "Removing raw sequence files..."
+File.unlink(r1_f)
+File.unlink(r2_f)
+log.puts Time.now.to_s + "\t" + "TCS pipeline successfuly exercuted."
+log.close
+puts "DONE!"