viral_seq 1.0.5 → 1.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a13ba7912ee87511c2ecf19f07256d3a129661c6d7e180d57ecd1e34978386e6
-  data.tar.gz: 61e5ed6b423f0b64c53a6bb8e8ec3801bf7e093e4d0741bd71bf9fbfa24f1b55
+  metadata.gz: 14d880e9f39b2b87892bec9d4377b358643c880cf32c81872cff51e1007bc23b
+  data.tar.gz: 6ee1c3293e2b0403a2eac033335f7575625b2d35f32127b5b57be53e94b4ec7d
 SHA512:
-  metadata.gz: f18d03220190bf1479ed29bd4d4b83777ffe5216951d38a91dd2afdc6c07b516883a8694291106b6fee2693a246b8a3c6a824786527cd03730f28f6777fa3231
-  data.tar.gz: 7fe146b081a7b633de963ed632bdcb548c71d1f401e227109d8745d23ad770d2099a2aa50bc4553a9450b260b7206892ed2a898d9154764aebe4094f38faeb44
+  metadata.gz: 951b75ced84aa21cf5650baa6970f60a617d3f29d20c14acadacefabea23d6b584f25990453c2008f30197aaef055a94edbdbb45494bb12b6343d90bc6bd45fb
+  data.tar.gz: 68ac69b4ebd5438a8f73780db823c94aa5a78c7c26d02cfd6bec979244dd1d6452c3698ade0606ddbbaccc480ad85e603171c11648dbb0110c2f5dbb3355bb35

data/Gemfile.lock

@@ -1,15 +1,17 @@
 PATH
   remote: .
   specs:
-    viral_seq (1.0.5)
+    viral_seq (1.0.10)
+      colorize (~> 0.1)
       muscle_bio (~> 0.4)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    colorize (0.8.1)
     diff-lcs (1.3)
     muscle_bio (0.4.0)
-    rake (10.5.0)
+    rake (13.0.1)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -29,9 +31,9 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 2.0)
-  rake (~> 10.0)
+  rake (~> 13.0)
   rspec (~> 3.0)
   viral_seq!
 
 BUNDLED WITH
-   2.0.2
+   2.1.4

data/README.md

@@ -4,82 +4,154 @@ A Ruby Gem containing bioinformatics tools for processing viral NGS data.
 
 Specifically for Primer-ID sequencing and HIV drug resistance analysis.
 
-## Installation
+## Install
 
+```bash
     $ gem install viral_seq
+```
 
 ## Usage
 
-Load all ViralSeq classes by requiring 'viral_seq.rb'
+### Excutables
 
-    #!/usr/bin/env ruby
-    require 'viral_seq'
+Use executable `locator` to get the coordinates of the sequences on HIV/SIV reference genome from a FASTA file through a terminal
+
+```bash
+    $ locator -i sequence.fasta -o sequence.fasta.csv
+```
+
+Use executable `tcs` pipeline to process Primer ID MiSeq sequencing data.
+
+```bash
+    $ tcs -p params.json # run TCS pipeline with params.json
+    $ tcs -j # CLI to generate params.json
+    $ tcs -h # print out the help
+```
 
 ## Some Examples
 
-### Load nucleotide sequences from a FASTA format sequence file
+Load all ViralSeq classes by requiring 'viral_seq.rb' in your Ruby scripts.
 
-    my_seqhash = ViralSeq::SeqHash.fa('my_seq_file.fasta')
+```ruby
+#!/usr/bin/env ruby
+require 'viral_seq'
+```
 
-### Make an alignment (using MUSCLE)
+Load nucleotide sequences from a FASTA format sequence file
 
-    aligned_seqhash = my_seqhash.align
+```ruby
+my_seqhash = ViralSeq::SeqHash.fa('my_seq_file.fasta')
+```
 
-### Filter nucleotide sequences with the reference coordinates (HIV Protease)
+Make an alignment (using MUSCLE)
 
-    qc_seqhash = aligned_seqhash.hiv_seq_qc(2253, 2549, false, :HXB2)
+```ruby
+aligned_seqhash = my_seqhash.align
+```
 
-### Further filter out sequences with Apobec3g/f hypermutations
+Filter nucleotide sequences with the reference coordinates (HIV Protease)
 
-    qc_seqhash = qc_seqhash.a3g
+```ruby
+qc_seqhash = aligned_seqhash.hiv_seq_qc(2253, 2549, false, :HXB2)
+```
 
-### Calculate nucleotide diveristy π
+Further filter out sequences with Apobec3g/f hypermutations
 
-    qc_seqhash.pi
+```ruby
+qc_seqhash = qc_seqhash.a3g
+```
 
-### Calculate cut-off for minority variants based on Poisson model
+Calculate nucleotide diveristy π
 
-    cut_off = qc_seqhash.pm
+```ruby
+qc_seqhash.pi
+```
 
-### Examine for drug resistance mutations for HIV PR region
+Calculate cut-off for minority variants based on Poisson model
 
-    qc_seqhash.sdrm_hiv_pr(cut_off)
+```ruby
+cut_off = qc_seqhash.pm
+```
 
-### Use executable `locator` to get the coordinates of the sequences on HIV/SIV reference genome from a FASTA file through a terminal
+Examine for drug resistance mutations for HIV PR region
 
-    $ locator -i sequence.fasta -o sequence.fasta.csv
+```ruby
+qc_seqhash.sdrm_hiv_pr(cut_off)
+```
 
 ## Updates
 
-Version 1.0.5-07112019:
+### Version 1.1.0-11112020:
+
+  1. Modularize TCS pipeline. Move key functions into /viral_seq/tcs_core.rb
+  2. `tcs_json_generator` is removed. This CLI is delivered within the `tcs` pipeline, by running `tcs -j`. The scripts are included in the /viral_seq/tcs_json.rb
+  3. consensus model now includes a true simple majority model, where no nt needs to be over 50% to be called.
+  4. a few optimizations.
+  5. TCS 2.1.0 delivered.
+  6. Tried parallel processing. Cannot achieve the goal because `parallel` gem by default can't pool data from memory of child processors and `in_threads` does not help with the speed. 
+
+### Version 1.0.9-07182020:
+
+  1. Change ViralSeq::SeqHash#stop_codon and ViralSeq::SeqHash#a3g_hypermut return value to hash object.
+
+  2. TCS pipeline updated to version 2.0.1. Add optional `export_raw: TRUE/FALSE` in json params. If `export_raw` is `TRUE`, raw sequence reads (have to pass quality filters) will be exported, along with TCS reads.
+
+### Version 1.0.8-02282020:
+
+  1. TCS pipeline (version 2.0.0) added as executable.
+      tcs  -  main TCS pipeline script.
+      tcs_json_generator  -  step-by-step script to generate json file for tcs pipeline.
+
+  2. Methods added:
+      ViralSeq::SeqHash#trim
+
+  3. Bug fix for several methods.
+
+### Version 1.0.7-01282020:
+
+  1. Several methods added, including
+      ViralSeq::SeqHash#error_table
+      ViralSeq::SeqHash#random_select
+  2. Improved performance for several functions.
+
+### Version 1.0.6-07232019:
+
+  1. Several methods added to ViralSeq::SeqHash, including
+      ViralSeq::SeqHash#size
+      ViralSeq::SeqHash#+
+      ViralSeq::SeqHash#write_nt_fa
+      ViralSeq::SeqHash#mutation
+  2. Update documentations and rspec samples.
+
+### Version 1.0.5-07112019:
 
-    1. Update ViralSeq::SeqHash#sequence_locator.
-       Program will try to determine the direction (`+` or `-` of the query sequence)
-    2. update executable `locator` to have a column of `direction` in output .csv file
+  1. Update ViralSeq::SeqHash#sequence_locator.
+     Program will try to determine the direction (`+` or `-` of the query sequence)
+  2. update executable `locator` to have a column of `direction` in output .csv file
 
-Version 1.0.4-07102019:
+### Version 1.0.4-07102019:
 
-    1. Use home directory (Dir.home) instead of the directory of the script file for temp MUSCLE file.
-    2. Fix bugs in bin `locator`
+  1. Use home directory (Dir.home) instead of the directory of the script file for temp MUSCLE file.
+  2. Fix bugs in bin `locator`
 
-Version 1.0.3-07102019:
+### Version 1.0.3-07102019:
 
-    1. Bug fix.
+  1. Bug fix.
 
-Version 1.0.2-07102019:
+### Version 1.0.2-07102019:
 
-    1. Fixed a gem loading issue.
+  1. Fixed a gem loading issue.
 
-Version 1.0.1-07102019:
+### Version 1.0.1-07102019:
 
-    1. Add keyword argument :model to ViralSeq::SeqHashPair#join2.
-    2. Add method ViralSeq::SeqHash#sequence_locator (also: #loc), a function to locate sequences on HIV/SIV reference genomes, as HIV Sequence Locator from LANL.
-    3. Add executable 'locator'. An HIV/SIV sequence locator tool similar to LANL Sequence Locator.
-    4. update documentations
+  1. Add keyword argument :model to ViralSeq::SeqHashPair#join2.
+  2. Add method ViralSeq::SeqHash#sequence_locator (also: #loc), a function to locate sequences on HIV/SIV reference genomes, as HIV Sequence Locator from LANL.
+  3. Add executable 'locator'. An HIV/SIV sequence locator tool similar to LANL Sequence Locator.
+  4. update documentations
 
-Version 1.0.0-07092019:
+### Version 1.0.0-07092019:
 
-    1. Rewrote the whole ViralSeq gem, grouping methods into modules and classes under main Module::ViralSeq
+  1. Rewrote the whole ViralSeq gem, grouping methods into modules and classes under main Module::ViralSeq
 
 ## Development
 

data/bin/locator

@@ -1,15 +1,36 @@
 #!/usr/bin/env ruby
 
+# Copyright (c) 2020 Shuntai Zhou (shuntai.zhou@gmail.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
 require 'viral_seq'
 require 'csv'
 require 'optparse'
+require 'colorize'
 
 def myparser
   options = {}
   OptionParser.new do |opts|
-    opts.banner = "Usage: locator -i [nt_sequence_fasta_file] -o [locator_info_csv_file] -r [reference_genome_option]"
+    opts.banner = "#{"Usage:".red.bold} locator #{"-i".blue.bold} [nt_sequence_fasta_file] #{"-o".blue.bold} [locator_info_csv_file] #{"-r".blue.bold} [reference_genome_option]"
 
-    opts.on('-i', '--infile FASTA_FILE', 'nt sequence file in FASTA format') do |i|
+    opts.on('-i', '--infile FASTA_FILE', "#{"nt sequence".blue.bold} file in FASTA format") do |i|
       options[:infile] = i
     end
 
@@ -17,7 +38,7 @@ def myparser
       options[:outfile] = o
     end
 
-    opts.on('-r', '--ref_option OPTION', 'reference genome option, choose from `HXB2` (default), `NL43`, `MAC239`') do |o|
+    opts.on('-r', '--ref_option OPTION', "reference genome option, choose from #{"`HXB2` (default), `NL43`, `MAC239`".blue.bold}") do |o|
       options[:ref_option] = o.to_sym
     end
 
@@ -35,9 +56,9 @@ def myparser
   return options
 end
 
-puts "\nSequence Locator (RubyGem::ViralSeq Version #{ViralSeq::VERSION}) by Shuntai Zhou"
-puts "See details at https://github.com/ViralSeq/viral_seq\n"
-puts "Resembling Sequence Locator from LANL (https://www.hiv.lanl.gov/content/sequence/LOCATE/locate.html)\n\n"
+puts "\n" + "Sequence Locator (RubyGem::ViralSeq Version #{ViralSeq::VERSION})".red.bold + " by " + "Shuntai Zhou".blue.bold
+puts "See details at " +  "https://github.com/ViralSeq/viral_seq\n".blue
+puts "Resembling" + " Sequence Locator ".magenta.bold + "from LANL" + " (https://www.hiv.lanl.gov/content/sequence/LOCATE/locate.html)\n".blue
 
 ARGV << '-h' if ARGV.size == 0
 
@@ -47,7 +68,7 @@ begin
   if options[:infile]
     seq_file = options[:infile]
   else
-    raise StandardError.new("Input file sequence file not found")
+    raise StandardError.new("Input file sequence file not found".red.bold)
   end
 
   if options[:outfile]
@@ -57,14 +78,14 @@ begin
   end
 
   unless File.exist?(seq_file)
-    raise StandardError.new("Input file sequence file not found")
+    raise StandardError.new("Input file sequence file not found".red.bold)
   end
 
   seqs = ViralSeq::SeqHash.fa(seq_file)
   opt =  options[:ref_option] ? options[:ref_option] : :HXB2
 
   unless [:HXB2, :NL43, :MAC239].include? opt
-    puts "Reference option #{opt} not recognized, using `:HXB2` as the reference genome."
+    puts "Reference option `#{opt}` not recognized, using `HXB2` as the reference genome.".red.bold
     opt = :HXB2
   end
 
@@ -76,6 +97,7 @@ begin
   end
 
   File.write(csv_file, data)
+  puts "Output file found at #{csv_file.green.bold}"
 rescue StandardError => e
   puts e.message
   puts "\n"

data/bin/tcs

@@ -0,0 +1,450 @@
+#!/usr/bin/env ruby
+
+# TCS pipeline for Primer ID sequencing data analysis.
+
+# Copyright (c) 2020 Shuntai Zhou (shuntai.zhou@gmail.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# Use JSON file as the run param
+# run tcs_json_generator.rb to generate param json file.
+
+require 'viral_seq'
+require 'json'
+require 'colorize'
+require 'OptionParser'
+
+options = {}
+
+banner = '-'*50 + "\n" +
+        '| The TCS Pipeline ' + "Version #{ViralSeq::TCS_VERSION}".red.bold + " by " + "Shuntai Zhou".blue.bold + ' |' + "\n" +
+        '-'*50 + "\n"
+
+OptionParser.new do |opts|
+  opts.banner = banner + "Usage: tcs -j"
+  opts.on "-j", "--json_generator", "Command line interfac to generate new params json file" do |j|
+    options[:json_generator] = true
+  end
+
+  opts.on("-p", "--params PARAMS_JSON", "Execute the pipeline with input params json file") do |p|
+    options[:params_json] = p
+  end
+
+  opts.on("-h", "--help", "Prints this help") do
+    puts opts
+    exit
+  end
+
+  opts.on("-v", "--version", "Version info") do
+    puts "tcs version: " + ViralSeq::TCS_VERSION.red.bold
+    puts "viral_seq version: " + ViralSeq::VERSION.red.bold
+    exit
+  end
+
+  # opts.on("--no-parallel", "toggle off parallel processing") do
+  #   options[:no_parallel] = true
+  # end
+end.parse!
+
+if options[:json_generator]
+  params = ViralSeq::TcsJson.generate
+elsif (options[:params_json] && File.exist?(options[:params_json]))
+  params = JSON.parse(File.read(options[:params_json]), symbolize_names: true)
+else
+  abort "No params JSON file found. Script terminated.".red
+end
+
+indir = params[:raw_sequence_dir]
+
+unless File.exist?(indir)
+  abort "No input sequence directory found. Script terminated.".red.bold
+end
+
+# log file
+
+runtime_log_file = File.join(indir,"runtime.log")
+log = File.open(runtime_log_file, "w")
+log.puts "TSC pipeline Version " + ViralSeq::TCS_VERSION.to_s
+log.puts "viral_seq Version " + ViralSeq::VERSION.to_s
+log.puts Time.now.to_s + "\t" + "Start TCS pipeline..."
+
+libname = File.basename indir
+
+seq_files = ViralSeq::TcsCore.r1r2 indir
+
+if seq_files[:r1_file].size > 0 and seq_files[:r2_file].size > 0
+  r1_f = seq_files[:r1_file]
+  r2_f = seq_files[:r2_file]
+elsif seq_files[:r1_file].size > 0 and seq_files[:r2_file].empty?
+  exit_sig = "Missing R2 file. Aborted."
+elsif seq_files[:r2_file].size > 0 and seq_files[:r1_file].empty?
+  exit_sig = "Missing R1 file. Aborted."
+else
+  exit_sig = "Cannot determine R1 R2 file in #{indir}. Aborted."
+end
+
+if exit_sig
+  ViralSeq::TcsCore.log_and_abort log, exit_sig
+end
+
+r1_fastq_sh = ViralSeq::SeqHash.fq(r1_f)
+r2_fastq_sh = ViralSeq::SeqHash.fq(r2_f)
+
+raw_sequence_number = r1_fastq_sh.size
+log.puts Time.now.to_s + "\tRaw sequence number: #{raw_sequence_number.to_s}"
+
+if params[:platform_error_rate]
+  error_rate = params[:platform_error_rate]
+else
+  error_rate = 0.02
+end
+
+primers = params[:primer_pairs]
+if primers.empty?
+  ViralSeq::TcsCore.log_and_abort log, "No primer information. Script terminated."
+end
+
+
+primers.each do |primer|
+  summary_json = {}
+  summary_json[:tcs_version] = ViralSeq::TCS_VERSION
+  summary_json[:viralseq_version] = ViralSeq::VERSION
+  summary_json[:runtime] = Time.now.to_s
+
+  primer[:region] ? region = primer[:region] : region = "region"
+  summary_json[:primer_set_name] = region
+
+  cdna_primer = primer[:cdna]
+  forward_primer = primer[:forward]
+
+  export_raw = primer[:export_raw]
+
+  unless cdna_primer
+    log.puts Time.now.to_s + "\t" + region + " does not have cDNA primer sequence. #{region} skipped."
+  end
+  unless forward_primer
+    log.puts Time.now.to_s + "\t" +  region + " does not have forward primer sequence. #{region} skipped."
+  end
+  summary_json[:cdan_primer] = cdna_primer
+  summary_json[:forward_primer] = forward_primer
+
+  primer[:majority] ? majority_cut_off = primer[:majority] : majority_cut_off = 0
+  summary_json[:majority_cut_off] = majority_cut_off
+
+  summary_json[:total_raw_sequence] = raw_sequence_number
+
+  log.puts Time.now.to_s + "\t" +  "Porcessing #{region}..."
+
+  # filter R1
+  log.puts Time.now.to_s + "\t" +  "filtering R1..."
+  filter_r1 = ViralSeq::TcsCore.filter_r1(r1_fastq_sh, forward_primer)
+  r1_passed_seq = filter_r1[:r1_passed_seq]
+  log.puts Time.now.to_s + "\t" +  "R1 filtered: #{r1_passed_seq.size.to_s}"
+  summary_json[:r1_filtered_raw] = r1_passed_seq.size
+
+  # filter R2
+  log.puts Time.now.to_s + "\t" +  "filtering R2..."
+  filter_r2 = ViralSeq::TcsCore.filter_r2(r2_fastq_sh, cdna_primer)
+  r2_passed_seq = filter_r2[:r2_passed_seq]
+  pid_length = filter_r2[:pid_length]
+  log.puts Time.now.to_s + "\t" +  "R2 filtered: #{r2_passed_seq.size.to_s}"
+  summary_json[:r2_filtered_raw] = r2_passed_seq.size
+
+  # pair-end
+  log.puts Time.now.to_s + "\t" +  "Pairing R1 and R2 seqs..."
+  id = {} # hash for :sequence_tag => primer_id
+  bio_r2 = {} # hash for :sequence_tag => primer_trimmed_r2_sequence
+  bio_r1 = {} # hash for :sequence_tag => primer_trimmed_r1_sequence
+  common_keys = r1_passed_seq.keys & r2_passed_seq.keys
+  paired_seq_number = common_keys.size
+  log.puts Time.now.to_s + "\t" +  "Paired raw sequences are : #{paired_seq_number.to_s}"
+  summary_json[:paired_raw_sequence] = paired_seq_number
+
+  common_keys.each do |seqtag|
+    r1_seq = r1_passed_seq[seqtag]
+    r2_seq = r2_passed_seq[seqtag]
+    pid = r2_seq[0, pid_length]
+    id[seqtag] = pid
+    bio_r2[seqtag] = r2_seq[filter_r2[:reverse_starting_number]..-2]
+    bio_r1[seqtag] = r1_seq[filter_r1[:forward_starting_number]..-2]
+  end
+
+  # TCS cut-off
+  log.puts Time.now.to_s + "\t" +  "Calculate consensus cutoff...."
+
+  primer_id_list = id.values
+  primer_id_count = primer_id_list.count_freq
+  primer_id_dis = primer_id_count.values.count_freq
+
+  # calculate distinct_to_raw
+  distinct_to_raw = (primer_id_count.size/primer_id_list.size.to_f).round(3)
+  summary_json[:distinct_to_raw] = distinct_to_raw
+
+  if primer_id_dis.keys.size < 5
+    log.puts Time.now.to_s + "\t" +  "Less than 5 Primer IDs detected. Region #{region} aborted."
+    next
+  end
+
+  max_id = primer_id_dis.keys.sort[-5..-1].mean
+  consensus_cutoff = ViralSeq::TcsCore.calculate_cut_off(max_id,error_rate)
+  log.puts Time.now.to_s + "\t" +  "Consensus cut-off is #{consensus_cutoff.to_s}"
+  summary_json[:consensus_cutoff] = consensus_cutoff
+  summary_json[:length_of_pid] = pid_length
+  log.puts Time.now.to_s + "\t" +  "Creating consensus..."
+
+  # Primer ID over the cut-off
+  primer_id_count_over_n = []
+  primer_id_count.each do |primer_id,count|
+    primer_id_count_over_n << primer_id if count > consensus_cutoff
+  end
+  pid_to_process = primer_id_count_over_n.size
+  log.puts Time.now.to_s + "\t" +  "Number of consensus to process: #{pid_to_process.to_s}"
+  summary_json[:total_tcs_with_ambiguities] = pid_to_process
+
+  # setup output path
+  out_dir_set = File.join(indir, region)
+  Dir.mkdir(out_dir_set) unless File.directory?(out_dir_set)
+  out_dir_consensus = File.join(out_dir_set, "consensus")
+  Dir.mkdir(out_dir_consensus) unless File.directory?(out_dir_consensus)
+
+  outfile_r1 = File.join(out_dir_consensus, 'r1.fasta')
+  outfile_r2 = File.join(out_dir_consensus, 'r2.fasta')
+  outfile_log = File.join(out_dir_set, 'log.json')
+
+  # if export_raw is true, create dir for raw sequence
+  if export_raw
+    out_dir_raw = File.join(out_dir_set, "raw")
+    Dir.mkdir(out_dir_raw) unless File.directory?(out_dir_raw)
+    outfile_raw_r1 = File.join(out_dir_raw, 'r1.raw.fasta')
+    outfile_raw_r2 = File.join(out_dir_raw, 'r2.raw.fasta')
+    raw_r1_f = File.open(outfile_raw_r1, 'w')
+    raw_r2_f = File.open(outfile_raw_r2, 'w')
+
+    bio_r1.keys.each do |k|
+      raw_r1_f.puts k + "_r1"
+      raw_r2_f.puts k + "_r2"
+      raw_r1_f.puts bio_r1[k]
+      raw_r2_f.puts bio_r2[k].rc
+    end
+
+    raw_r1_f.close
+    raw_r2_f.close
+  end
+
+  # create TCS
+
+  pid_seqtag_hash = {}
+  id.each do |name, pid|
+    if pid_seqtag_hash[pid]
+      pid_seqtag_hash[pid] << name
+    else
+      pid_seqtag_hash[pid] = []
+      pid_seqtag_hash[pid] << name
+    end
+  end
+
+  consensus = {}
+  r1_temp = {}
+  r2_temp = {}
+  m = 0
+  primer_id_count_over_n.each do |primer_id|
+    m += 1
+    log.puts Time.now.to_s + "\t" +  "Now processing number #{m}" if m%100 == 0
+    seq_with_same_primer_id = pid_seqtag_hash[primer_id]
+    r1_sub_seq = []
+    r2_sub_seq = []
+    seq_with_same_primer_id.each do |seq_name|
+      r1_sub_seq << bio_r1[seq_name]
+      r2_sub_seq << bio_r2[seq_name]
+    end
+
+    #consensus name including the Primer ID and number of raw sequences of that Primer ID, library name and setname.
+    consensus_name = ">" + primer_id + "_" + seq_with_same_primer_id.size.to_s + "_" + libname + "_" + region
+    r1_consensus = ViralSeq::SeqHash.array(r1_sub_seq).consensus(majority_cut_off)
+    r2_consensus = ViralSeq::SeqHash.array(r2_sub_seq).consensus(majority_cut_off)
+
+    # hide the following two lines if allowing sequence to have ambiguities.
+    next if r1_consensus =~ /[^ATCG]/
+    next if r2_consensus =~ /[^ATCG]/
+
+    # reverse complement sequence of the R2 region
+    r2_consensus = r2_consensus.rc
+    consensus[consensus_name] = [r1_consensus, r2_consensus]
+    r1_temp[consensus_name] = r1_consensus
+    r2_temp[consensus_name] = r2_consensus
+  end
+  r1_temp_sh = ViralSeq::SeqHash.new(r1_temp)
+  r2_temp_sh = ViralSeq::SeqHash.new(r2_temp)
+
+  # filter consensus sequences for residual offspring PIDs
+  consensus_filtered = {}
+  consensus_number_temp = consensus.size
+  max_pid_comb = 4**pid_length
+  if consensus_number_temp < 0.003*max_pid_comb
+    log.puts Time.now.to_s + "\t" +  "Applying PID post TCS filter..."
+    r1_consensus_filtered = r1_temp_sh.filter_similar_pid.dna_hash
+    r2_consensus_filtered = r2_temp_sh.filter_similar_pid.dna_hash
+    common_pid = r1_consensus_filtered.keys & r2_consensus_filtered.keys
+    common_pid.each do |pid|
+      consensus_filtered[pid] = [r1_consensus_filtered[pid], r2_consensus_filtered[pid]]
+    end
+  else
+    consensus_filtered = consensus
+  end
+  n_con = consensus_filtered.size
+  log.puts Time.now.to_s + "\t" +  "Number of consensus sequences: " + n_con.to_s
+  summary_json[:total_tcs] = n_con
+  summary_json[:resampling_param] = (n_con/pid_to_process.to_f).round(3)
+
+  log.puts Time.now.to_s + "\t" +  "Writing R1 and R2 files..."
+  # r1_file output
+  f1 = File.open(outfile_r1, 'w')
+  f2 = File.open(outfile_r2, 'w')
+  primer_id_in_use = {}
+  r1_seq_length = consensus_filtered.values[0][0].size
+  r2_seq_length = consensus_filtered.values[0][1].size
+  log.puts Time.now.to_s + "\t" + "R1 sequence #{r1_seq_length} bp"
+  log.puts Time.now.to_s + "\t" + "R1 sequence #{r2_seq_length} bp"
+  consensus_filtered.each do |seq_name,seq|
+    f1.print seq_name + "_r1\n" + seq[0] + "\n"
+    f2.print seq_name + "_r2\n" + seq[1] + "\n"
+    primer_id_in_use[seq_name.split("_")[0][1..-1]] = seq_name.split("_")[1].to_i
+  end
+  f1.close
+  f2.close
+
+  # Primer ID distribution in .json file
+  out_pid_json = File.join(out_dir_set, 'primer_id.json')
+  pid_json = {}
+  pid_json[:primer_id_in_use] = Hash[*(primer_id_in_use.sort_by {|k, v| [-v,k]}.flatten)]
+  pid_json[:primer_id_distribution] = Hash[*(primer_id_dis.sort_by{|k,v| k}.flatten)]
+  pid_json[:primer_id_frequency] = Hash[*(primer_id_count.sort_by {|k, v| [-v,k]}.flatten)]
+  File.open(out_pid_json, 'w') do |f|
+    f.puts JSON.pretty_generate(pid_json)
+  end
+
+  # start end-join
+  def end_join(dir, option, overlap)
+    shp = ViralSeq::SeqHashPair.fa(dir)
+    case option
+    when 1
+      joined_sh = shp.join1()
+    when 2
+      joined_sh = shp.join1(overlap)
+    when 3
+      joined_sh = shp.join2
+    when 4
+      joined_sh = shp.join2(model: :indiv)
+    end
+    return joined_sh
+  end
+
+  if primer[:end_join]
+    log.puts Time.now.to_s + "\t" +  "Start end-pairing for TCS..."
+    shp = ViralSeq::SeqHashPair.fa(out_dir_consensus)
+    joined_sh = end_join(out_dir_consensus, primer[:end_join_option], primer[:overlap])
+    log.puts Time.now.to_s + "\t" + "Paired TCS number: " + joined_sh.size.to_s
+    summary_json[:combined_tcs] = joined_sh.size
+
+    if export_raw
+      joined_sh_raw = end_join(out_dir_raw, primer[:end_join_option], primer[:overlap])
+    end
+
+  else
+    File.open(outfile_log, "w") do |f|
+      f.puts JSON.pretty_generate(summary_json)
+    end
+    next
+  end
+
+  if primer[:TCS_QC]
+    ref_start = primer[:ref_start]
+    ref_end = primer[:ref_end]
+    ref_genome = primer[:ref_genome].to_sym
+    indel = primer[:indel]
+    if ref_start == 0
+      ref_start = 0..(ViralSeq::RefSeq.get(ref_genome).size - 1)
+    end
+    if ref_end == 0
+      ref_end = 0..(ViralSeq::RefSeq.get(ref_genome).size - 1)
+    end
+    if primer[:end_join_option] == 1 and primer[:overlap] == 0
+      r1_sh = ViralSeq::SeqHash.fa(outfile_r1)
+      r2_sh = ViralSeq::SeqHash.fa(outfile_r2)
+      r1_sh = r1_sh.hiv_seq_qc(ref_start, (0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), indel, ref_genome)
+      r2_sh = r2_sh.hiv_seq_qc((0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), ref_end, indel, ref_genome)
+      new_r1_seq = r1_sh.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+      new_r2_seq = r2_sh.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+      joined_seq = {}
+      new_r1_seq.each do |seq_name, seq|
+        next unless seq
+        next unless new_r2_seq[seq_name]
+        joined_seq[seq_name] = seq + new_r2_seq[seq_name]
+      end
+      joined_sh = ViralSeq::SeqHash.new(joined_seq)
+
+      if export_raw
+        r1_sh_raw = ViralSeq::SeqHash.fa(outfile_raw_r1)
+        r2_sh_raw = ViralSeq::SeqHash.fa(outfile_raw_r2)
+        r1_sh_raw = r1_sh_raw.hiv_seq_qc(ref_start, (0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), indel, ref_genome)
+        r2_sh_raw = r2_sh_raw.hiv_seq_qc((0..(ViralSeq::RefSeq.get(ref_genome).size - 1)), ref_end, indel, ref_genome)
+        new_r1_seq_raw = r1_sh_raw.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+        new_r2_seq_raw = r2_sh_raw.dna_hash.each_with_object({}) {|(k, v), h| h[k[0..-4]] = v}
+        joined_seq_raw = {}
+        new_r1_seq_raw.each do |seq_name, seq|
+          next unless seq
+          next unless new_r2_seq_raw[seq_name]
+          joined_seq_raw[seq_name] = seq + new_r2_seq_raw[seq_name]
+        end
+        joined_sh_raw = ViralSeq::SeqHash.new(joined_seq_raw)
+      end
+    else
+      joined_sh = joined_sh.hiv_seq_qc(ref_start, ref_end, indel, ref_genome)
+
+      if export_raw
+        joined_sh_raw = joined_sh_raw.hiv_seq_qc(ref_start, ref_end, indel, ref_genome)
+      end
+    end
+
+    log.puts Time.now.to_s + "\t" + "Paired TCS number after QC based on reference genome: " + joined_sh.size.to_s
+    summary_json[:combined_tcs_after_qc] = joined_sh.size
+    if primer[:trim]
+      trim_start = primer[:trim_ref_start]
+      trim_end = primer[:trim_ref_end]
+      trim_ref = primer[:trim_ref].to_sym
+      joined_sh = joined_sh.trim(trim_start, trim_end, trim_ref)
+      joined_sh.write_nt_fa(File.join(out_dir_consensus, "combined.fasta"))
+      if export_raw
+        joined_sh_raw = joined_sh_raw.trim(trim_start, trim_end, trim_ref)
+        joined_sh_raw.write_nt_fa(File.join(out_dir_raw, "combined.raw.fasta"))
+      end
+    end
+  end
+
+  File.open(outfile_log, "w") do |f|
+    f.puts JSON.pretty_generate(summary_json)
+  end
+end
+
+log.puts Time.now.to_s + "\t" + "Removing raw sequence files..."
+File.unlink(r1_f)
+File.unlink(r2_f)
+log.puts Time.now.to_s + "\t" + "TCS pipeline successfuly exercuted."
+log.close
+puts "DONE!"